From ccc5d98bafd33433fb04806bd478172878846756 Mon Sep 17 00:00:00 2001
From: bakdata-bot <bakdata-bot@users.noreply.github.com>
Date: Fri, 29 Sep 2023 09:19:23 +0000
Subject: [PATCH] Deployed 636a85d to dev with MkDocs 1.4.2 and mike 1.1.2

---
 dev/404.html                                  |  16 +
 dev/developer/auto-generation/index.html      |   7 +-
 dev/developer/formatting/index.html           |   5 +
 dev/index.html                                |   5 +
 .../components-hierarchy/index.html           |   5 +
 dev/resources/examples/defaults/index.html    |   5 +
 dev/resources/examples/pipeline/index.html    |   5 +
 .../pipeline-components/kafka-app.yaml        |   2 +-
 .../pipeline-components/pipeline.yaml         |   6 +-
 .../pipeline-components/pipeline/index.html   |  11 +-
 .../pipeline-components/producer-app.yaml     |   2 +-
 .../sections/app-kafka-app.yaml               |   2 +-
 .../sections/app-producer-app.yaml            |   2 +-
 .../sections/app-streams-app.yaml             |   2 +-
 .../pipeline-components/streams-app.yaml      |   2 +-
 .../pipeline-defaults/defaults-kafka-app.yaml |   2 +-
 .../defaults-producer-app.yaml                |   2 +-
 .../defaults-streams-app.yaml                 |   2 +-
 dev/resources/pipeline-defaults/defaults.yaml |   6 +-
 .../pipeline-defaults/defaults/index.html     |  11 +-
 .../variables/cli_env_vars/index.html         |   5 +
 dev/resources/variables/config_env_vars.env   |  14 +-
 .../variables/config_env_vars/index.html      |  23 +-
 dev/schema/config.json                        | 168 +++--
 dev/schema/pipeline.json                      |  44 +-
 dev/search/search_index.json                  |   2 +-
 dev/sitemap.xml                               |  67 +-
 dev/sitemap.xml.gz                            | Bin 557 -> 562 bytes
 .../components/kafka-app/index.html           |   7 +-
 .../components/kafka-connector/index.html     |   5 +
 .../kafka-sink-connector/index.html           |   5 +
 .../kafka-source-connector/index.html         |   5 +
 .../components/kubernetes-app/index.html      |   5 +
 .../components/overview/index.html            |   5 +
 .../components/producer-app/index.html        |   7 +-
 .../components/streams-app/index.html         |   7 +-
 dev/user/core-concepts/config/index.html      |   5 +
 dev/user/core-concepts/defaults/index.html    |   9 +-
 .../environment_variables/index.html          |  37 +-
 .../variables/substitution/index.html         |   5 +
 .../examples/atm-fraud-pipeline/index.html    |   5 +
 .../getting-started/quick-start/index.html    |   5 +
 dev/user/getting-started/setup/index.html     |   5 +
 dev/user/getting-started/teardown/index.html  |   5 +
 dev/user/migration-guide/v1-v2/index.html     |  35 +-
 dev/user/migration-guide/v2-v3/index.html     | 641 ++++++++++++++++++
 .../ci-integration/github-actions/index.html  |  18 +-
 dev/user/references/cli-commands/index.html   |   7 +-
 .../references/editor-integration/index.html  |   5 +
 dev/user/what-is-kpops/index.html             |   5 +
 50 files changed, 1079 insertions(+), 177 deletions(-)
 create mode 100644 dev/user/migration-guide/v2-v3/index.html

diff --git a/dev/404.html b/dev/404.html
index d4fd16218..d1176f79c 100644
--- a/dev/404.html
+++ b/dev/404.html
@@ -866,6 +866,8 @@
         
           
         
+          
+        
       
       
         <label class="md-nav__link" for="__nav_2_5_1" id="__nav_2_5_1_label" tabindex="0">
@@ -894,6 +896,20 @@
 
             
           
+            
+              
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="/kpops/dev/user/migration-guide/v2-v3/" class="md-nav__link">
+        Migrate from v2 to v3
+      </a>
+    </li>
+  
+
+            
+          
         </ul>
       </nav>
     </li>
diff --git a/dev/developer/auto-generation/index.html b/dev/developer/auto-generation/index.html
index 1f551f6ac..0a0eb095a 100644
--- a/dev/developer/auto-generation/index.html
+++ b/dev/developer/auto-generation/index.html
@@ -373,6 +373,11 @@
         Migrate from v1 to v2
       </a>
 </li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../user/migration-guide/v2-v3/">
+        Migrate from v2 to v3
+      </a>
+</li>
 </ul>
 </nav>
 </li>
@@ -573,7 +578,7 @@ <h4 id="variables"><a href="https://github.com/bakdata/kpops/tree/main/docs/docs
 <ul>
 <li><code>cli_env_vars.env</code> -- All CLI environment variables in a <code>dotenv</code> file.</li>
 <li><code>cli_env_vars.md</code> -- All CLI environment variables in a table.</li>
-<li><code>config_env_vars.env</code> -- Almost all pipeline config environment variables in a <code>dotenv</code> file. The script checks for each field in <a href="https://github.com/bakdata/kpops/blob/main/kpops/cli/pipeline_config.py"><code>PipelineConfig</code></a> whether it has an <code>env</code> attribute defined. The script is currently unable to visit the classes of fields like <code>topic_name_config</code>, hence any environment variables defined there would remain unknown to it.</li>
+<li><code>config_env_vars.env</code> -- Almost all pipeline config environment variables in a <code>dotenv</code> file. The script checks for each field in <a href="https://github.com/bakdata/kpops/blob/main/kpops/cli/kpops_config.py"><code>PipelineConfig</code></a> whether it has an <code>env</code> attribute defined. The script is currently unable to visit the classes of fields like <code>topic_name_config</code>, hence any environment variables defined there would remain unknown to it.</li>
 <li><code>config_env_vars.env</code> -- Almost all pipeline config environment variables in a table.</li>
 <li><code>variable_substitution.yaml</code> -- A copy of <code>./tests/pipeline/resources/component-type-substitution/pipeline.yaml</code> used as an example of substitution.</li>
 </ul>
diff --git a/dev/developer/formatting/index.html b/dev/developer/formatting/index.html
index a0803c377..080c90587 100644
--- a/dev/developer/formatting/index.html
+++ b/dev/developer/formatting/index.html
@@ -372,6 +372,11 @@
         Migrate from v1 to v2
       </a>
 </li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../user/migration-guide/v2-v3/">
+        Migrate from v2 to v3
+      </a>
+</li>
 </ul>
 </nav>
 </li>
diff --git a/dev/index.html b/dev/index.html
index 7c7a3a17e..1a485129c 100644
--- a/dev/index.html
+++ b/dev/index.html
@@ -445,6 +445,11 @@ <h1>KPOps Documentation</h1>
         Migrate from v1 to v2
       </a>
 </li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="user/migration-guide/v2-v3/">
+        Migrate from v2 to v3
+      </a>
+</li>
 </ul>
 </nav>
 </li>
diff --git a/dev/resources/architecture/components-hierarchy/index.html b/dev/resources/architecture/components-hierarchy/index.html
index 9537bfbdb..24fa73e77 100644
--- a/dev/resources/architecture/components-hierarchy/index.html
+++ b/dev/resources/architecture/components-hierarchy/index.html
@@ -368,6 +368,11 @@
         Migrate from v1 to v2
       </a>
 </li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../../user/migration-guide/v2-v3/">
+        Migrate from v2 to v3
+      </a>
+</li>
 </ul>
 </nav>
 </li>
diff --git a/dev/resources/examples/defaults/index.html b/dev/resources/examples/defaults/index.html
index e51a32a24..eca9970ca 100644
--- a/dev/resources/examples/defaults/index.html
+++ b/dev/resources/examples/defaults/index.html
@@ -371,6 +371,11 @@
         Migrate from v1 to v2
       </a>
 </li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../../user/migration-guide/v2-v3/">
+        Migrate from v2 to v3
+      </a>
+</li>
 </ul>
 </nav>
 </li>
diff --git a/dev/resources/examples/pipeline/index.html b/dev/resources/examples/pipeline/index.html
index 05f84aa51..23c1741e4 100644
--- a/dev/resources/examples/pipeline/index.html
+++ b/dev/resources/examples/pipeline/index.html
@@ -371,6 +371,11 @@
         Migrate from v1 to v2
       </a>
 </li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../../user/migration-guide/v2-v3/">
+        Migrate from v2 to v3
+      </a>
+</li>
 </ul>
 </nav>
 </li>
diff --git a/dev/resources/pipeline-components/kafka-app.yaml b/dev/resources/pipeline-components/kafka-app.yaml
index 6d8045ad5..cdc49ef28 100644
--- a/dev/resources/pipeline-components/kafka-app.yaml
+++ b/dev/resources/pipeline-components/kafka-app.yaml
@@ -49,7 +49,7 @@
   # add the key-value pairs they need.
   app: # required
     streams: # required
-      brokers: ${brokers} # required
+      brokers: ${kafka_brokers} # required
       schemaRegistryUrl: ${schema_registry_url}
     nameOverride: override-with-this-name # kafka-app-specific
     imageTag: "1.0.0" # Example values that are shared between streams-app and producer-app
diff --git a/dev/resources/pipeline-components/pipeline.yaml b/dev/resources/pipeline-components/pipeline.yaml
index 27c5d45c1..eb7930376 100644
--- a/dev/resources/pipeline-components/pipeline.yaml
+++ b/dev/resources/pipeline-components/pipeline.yaml
@@ -49,7 +49,7 @@
   # add the key-value pairs they need.
   app: # required
     streams: # required
-      brokers: ${brokers} # required
+      brokers: ${kafka_brokers} # required
       schemaRegistryUrl: ${schema_registry_url}
     nameOverride: override-with-this-name # kafka-app-specific
     imageTag: "1.0.0" # Example values that are shared between streams-app and producer-app
@@ -275,7 +275,7 @@
   # https://github.com/bakdata/streams-bootstrap/tree/master/charts/producer-app
   app: # required
     streams: # required, producer-app-specific
-      brokers: ${brokers} # required
+      brokers: ${kafka_brokers} # required
       schemaRegistryUrl: ${schema_registry_url}
       outputTopic: output_topic
       extraOutputTopics:
@@ -346,7 +346,7 @@
   app: # required
     # Streams Bootstrap streams section
     streams: # required, streams-app-specific
-      brokers: ${brokers} # required
+      brokers: ${kafka_brokers} # required
       schemaRegistryUrl: ${schema_registry_url}
       inputTopics:
         - topic1
diff --git a/dev/resources/pipeline-components/pipeline/index.html b/dev/resources/pipeline-components/pipeline/index.html
index b690d2705..26f2d29c9 100644
--- a/dev/resources/pipeline-components/pipeline/index.html
+++ b/dev/resources/pipeline-components/pipeline/index.html
@@ -371,6 +371,11 @@
         Migrate from v1 to v2
       </a>
 </li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../../user/migration-guide/v2-v3/">
+        Migrate from v2 to v3
+      </a>
+</li>
 </ul>
 </nav>
 </li>
@@ -911,7 +916,7 @@ <h1 id="pipelineyaml">pipeline.yaml<a class="headerlink" href="#pipelineyaml" ti
 </span><span id="__span-0-49"><a id="__codelineno-0-49" name="__codelineno-0-49"></a><span class="w">  </span><span class="c1"># add the key-value pairs they need.</span>
 </span><span id="__span-0-50"><a id="__codelineno-0-50" name="__codelineno-0-50"></a><span class="w">  </span><span class="nt">app</span><span class="p">:</span><span class="w"> </span><span class="c1"># required</span>
 </span><span id="__span-0-51"><a id="__codelineno-0-51" name="__codelineno-0-51"></a><span class="w">    </span><span class="nt">streams</span><span class="p">:</span><span class="w"> </span><span class="c1"># required</span>
-</span><span id="__span-0-52"><a id="__codelineno-0-52" name="__codelineno-0-52"></a><span class="w">      </span><span class="nt">brokers</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">${brokers}</span><span class="w"> </span><span class="c1"># required</span>
+</span><span id="__span-0-52"><a id="__codelineno-0-52" name="__codelineno-0-52"></a><span class="w">      </span><span class="nt">brokers</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">${kafka_brokers}</span><span class="w"> </span><span class="c1"># required</span>
 </span><span id="__span-0-53"><a id="__codelineno-0-53" name="__codelineno-0-53"></a><span class="w">      </span><span class="nt">schemaRegistryUrl</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">${schema_registry_url}</span>
 </span><span id="__span-0-54"><a id="__codelineno-0-54" name="__codelineno-0-54"></a><span class="w">    </span><span class="nt">nameOverride</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">override-with-this-name</span><span class="w"> </span><span class="c1"># kafka-app-specific</span>
 </span><span id="__span-0-55"><a id="__codelineno-0-55" name="__codelineno-0-55"></a><span class="w">    </span><span class="nt">imageTag</span><span class="p">:</span><span class="w"> </span><span class="s">"1.0.0"</span><span class="w"> </span><span class="c1"># Example values that are shared between streams-app and producer-app</span>
@@ -1137,7 +1142,7 @@ <h1 id="pipelineyaml">pipeline.yaml<a class="headerlink" href="#pipelineyaml" ti
 </span><span id="__span-0-275"><a id="__codelineno-0-275" name="__codelineno-0-275"></a><span class="w">  </span><span class="c1"># https://github.com/bakdata/streams-bootstrap/tree/master/charts/producer-app</span>
 </span><span id="__span-0-276"><a id="__codelineno-0-276" name="__codelineno-0-276"></a><span class="w">  </span><span class="nt">app</span><span class="p">:</span><span class="w"> </span><span class="c1"># required</span>
 </span><span id="__span-0-277"><a id="__codelineno-0-277" name="__codelineno-0-277"></a><span class="w">    </span><span class="nt">streams</span><span class="p">:</span><span class="w"> </span><span class="c1"># required, producer-app-specific</span>
-</span><span id="__span-0-278"><a id="__codelineno-0-278" name="__codelineno-0-278"></a><span class="w">      </span><span class="nt">brokers</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">${brokers}</span><span class="w"> </span><span class="c1"># required</span>
+</span><span id="__span-0-278"><a id="__codelineno-0-278" name="__codelineno-0-278"></a><span class="w">      </span><span class="nt">brokers</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">${kafka_brokers}</span><span class="w"> </span><span class="c1"># required</span>
 </span><span id="__span-0-279"><a id="__codelineno-0-279" name="__codelineno-0-279"></a><span class="w">      </span><span class="nt">schemaRegistryUrl</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">${schema_registry_url}</span>
 </span><span id="__span-0-280"><a id="__codelineno-0-280" name="__codelineno-0-280"></a><span class="w">      </span><span class="nt">outputTopic</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">output_topic</span>
 </span><span id="__span-0-281"><a id="__codelineno-0-281" name="__codelineno-0-281"></a><span class="w">      </span><span class="nt">extraOutputTopics</span><span class="p">:</span>
@@ -1208,7 +1213,7 @@ <h1 id="pipelineyaml">pipeline.yaml<a class="headerlink" href="#pipelineyaml" ti
 </span><span id="__span-0-346"><a id="__codelineno-0-346" name="__codelineno-0-346"></a><span class="w">  </span><span class="nt">app</span><span class="p">:</span><span class="w"> </span><span class="c1"># required</span>
 </span><span id="__span-0-347"><a id="__codelineno-0-347" name="__codelineno-0-347"></a><span class="w">    </span><span class="c1"># Streams Bootstrap streams section</span>
 </span><span id="__span-0-348"><a id="__codelineno-0-348" name="__codelineno-0-348"></a><span class="w">    </span><span class="nt">streams</span><span class="p">:</span><span class="w"> </span><span class="c1"># required, streams-app-specific</span>
-</span><span id="__span-0-349"><a id="__codelineno-0-349" name="__codelineno-0-349"></a><span class="w">      </span><span class="nt">brokers</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">${brokers}</span><span class="w"> </span><span class="c1"># required</span>
+</span><span id="__span-0-349"><a id="__codelineno-0-349" name="__codelineno-0-349"></a><span class="w">      </span><span class="nt">brokers</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">${kafka_brokers}</span><span class="w"> </span><span class="c1"># required</span>
 </span><span id="__span-0-350"><a id="__codelineno-0-350" name="__codelineno-0-350"></a><span class="w">      </span><span class="nt">schemaRegistryUrl</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">${schema_registry_url}</span>
 </span><span id="__span-0-351"><a id="__codelineno-0-351" name="__codelineno-0-351"></a><span class="w">      </span><span class="nt">inputTopics</span><span class="p">:</span>
 </span><span id="__span-0-352"><a id="__codelineno-0-352" name="__codelineno-0-352"></a><span class="w">        </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">topic1</span>
diff --git a/dev/resources/pipeline-components/producer-app.yaml b/dev/resources/pipeline-components/producer-app.yaml
index 7a01ad24b..5be3551d8 100644
--- a/dev/resources/pipeline-components/producer-app.yaml
+++ b/dev/resources/pipeline-components/producer-app.yaml
@@ -32,7 +32,7 @@
   # https://github.com/bakdata/streams-bootstrap/tree/master/charts/producer-app
   app: # required
     streams: # required, producer-app-specific
-      brokers: ${brokers} # required
+      brokers: ${kafka_brokers} # required
       schemaRegistryUrl: ${schema_registry_url}
       outputTopic: output_topic
       extraOutputTopics:
diff --git a/dev/resources/pipeline-components/sections/app-kafka-app.yaml b/dev/resources/pipeline-components/sections/app-kafka-app.yaml
index 991e862e0..73b70c59e 100644
--- a/dev/resources/pipeline-components/sections/app-kafka-app.yaml
+++ b/dev/resources/pipeline-components/sections/app-kafka-app.yaml
@@ -2,7 +2,7 @@
   # add the key-value pairs they need.
   app: # required
     streams: # required
-      brokers: ${brokers} # required
+      brokers: ${kafka_brokers} # required
       schemaRegistryUrl: ${schema_registry_url}
     nameOverride: override-with-this-name # kafka-app-specific
     imageTag: "1.0.0" # Example values that are shared between streams-app and producer-app
diff --git a/dev/resources/pipeline-components/sections/app-producer-app.yaml b/dev/resources/pipeline-components/sections/app-producer-app.yaml
index 5cd9b000b..0cbe04ded 100644
--- a/dev/resources/pipeline-components/sections/app-producer-app.yaml
+++ b/dev/resources/pipeline-components/sections/app-producer-app.yaml
@@ -2,7 +2,7 @@
   # https://github.com/bakdata/streams-bootstrap/tree/master/charts/producer-app
   app: # required
     streams: # required, producer-app-specific
-      brokers: ${brokers} # required
+      brokers: ${kafka_brokers} # required
       schemaRegistryUrl: ${schema_registry_url}
       outputTopic: output_topic
       extraOutputTopics:
diff --git a/dev/resources/pipeline-components/sections/app-streams-app.yaml b/dev/resources/pipeline-components/sections/app-streams-app.yaml
index 44f6604aa..1c5f0849f 100644
--- a/dev/resources/pipeline-components/sections/app-streams-app.yaml
+++ b/dev/resources/pipeline-components/sections/app-streams-app.yaml
@@ -4,7 +4,7 @@
   app: # required
     # Streams Bootstrap streams section
     streams: # required, streams-app-specific
-      brokers: ${brokers} # required
+      brokers: ${kafka_brokers} # required
       schemaRegistryUrl: ${schema_registry_url}
       inputTopics:
         - topic1
diff --git a/dev/resources/pipeline-components/streams-app.yaml b/dev/resources/pipeline-components/streams-app.yaml
index 0dde5be5c..f77edf80c 100644
--- a/dev/resources/pipeline-components/streams-app.yaml
+++ b/dev/resources/pipeline-components/streams-app.yaml
@@ -51,7 +51,7 @@
   app: # required
     # Streams Bootstrap streams section
     streams: # required, streams-app-specific
-      brokers: ${brokers} # required
+      brokers: ${kafka_brokers} # required
       schemaRegistryUrl: ${schema_registry_url}
       inputTopics:
         - topic1
diff --git a/dev/resources/pipeline-defaults/defaults-kafka-app.yaml b/dev/resources/pipeline-defaults/defaults-kafka-app.yaml
index e0af3b7a7..bd6c9e2d9 100644
--- a/dev/resources/pipeline-defaults/defaults-kafka-app.yaml
+++ b/dev/resources/pipeline-defaults/defaults-kafka-app.yaml
@@ -7,7 +7,7 @@ kafka-app:
   # add the key-value pairs they need.
   app: # required
     streams: # required
-      brokers: ${brokers} # required
+      brokers: ${kafka_brokers} # required
       schemaRegistryUrl: ${schema_registry_url}
     nameOverride: override-with-this-name # kafka-app-specific
     imageTag: "1.0.0" # Example values that are shared between streams-app and producer-app
diff --git a/dev/resources/pipeline-defaults/defaults-producer-app.yaml b/dev/resources/pipeline-defaults/defaults-producer-app.yaml
index 1d81f5ced..bfa5521c4 100644
--- a/dev/resources/pipeline-defaults/defaults-producer-app.yaml
+++ b/dev/resources/pipeline-defaults/defaults-producer-app.yaml
@@ -10,7 +10,7 @@ producer-app:
   # https://github.com/bakdata/streams-bootstrap/tree/master/charts/producer-app
   app: # required
     streams: # required, producer-app-specific
-      brokers: ${brokers} # required
+      brokers: ${kafka_brokers} # required
       schemaRegistryUrl: ${schema_registry_url}
       outputTopic: output_topic
       extraOutputTopics:
diff --git a/dev/resources/pipeline-defaults/defaults-streams-app.yaml b/dev/resources/pipeline-defaults/defaults-streams-app.yaml
index 83ff13f14..ae1adab98 100644
--- a/dev/resources/pipeline-defaults/defaults-streams-app.yaml
+++ b/dev/resources/pipeline-defaults/defaults-streams-app.yaml
@@ -9,7 +9,7 @@ streams-app:
   app: # required
     # Streams Bootstrap streams section
     streams: # required, streams-app-specific
-      brokers: ${brokers} # required
+      brokers: ${kafka_brokers} # required
       schemaRegistryUrl: ${schema_registry_url}
       inputTopics:
         - topic1
diff --git a/dev/resources/pipeline-defaults/defaults.yaml b/dev/resources/pipeline-defaults/defaults.yaml
index e74272bdc..3a43d81e7 100644
--- a/dev/resources/pipeline-defaults/defaults.yaml
+++ b/dev/resources/pipeline-defaults/defaults.yaml
@@ -7,7 +7,7 @@ kafka-app:
   # add the key-value pairs they need.
   app: # required
     streams: # required
-      brokers: ${brokers} # required
+      brokers: ${kafka_brokers} # required
       schemaRegistryUrl: ${schema_registry_url}
     nameOverride: override-with-this-name # kafka-app-specific
     imageTag: "1.0.0" # Example values that are shared between streams-app and producer-app
@@ -170,7 +170,7 @@ producer-app:
   # https://github.com/bakdata/streams-bootstrap/tree/master/charts/producer-app
   app: # required
     streams: # required, producer-app-specific
-      brokers: ${brokers} # required
+      brokers: ${kafka_brokers} # required
       schemaRegistryUrl: ${schema_registry_url}
       outputTopic: output_topic
       extraOutputTopics:
@@ -188,7 +188,7 @@ streams-app:
   app: # required
     # Streams Bootstrap streams section
     streams: # required, streams-app-specific
-      brokers: ${brokers} # required
+      brokers: ${kafka_brokers} # required
       schemaRegistryUrl: ${schema_registry_url}
       inputTopics:
         - topic1
diff --git a/dev/resources/pipeline-defaults/defaults/index.html b/dev/resources/pipeline-defaults/defaults/index.html
index 02804976e..508752f43 100644
--- a/dev/resources/pipeline-defaults/defaults/index.html
+++ b/dev/resources/pipeline-defaults/defaults/index.html
@@ -371,6 +371,11 @@
         Migrate from v1 to v2
       </a>
 </li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../../user/migration-guide/v2-v3/">
+        Migrate from v2 to v3
+      </a>
+</li>
 </ul>
 </nav>
 </li>
@@ -700,7 +705,7 @@ <h1 id="defaultsyaml">defaults.yaml<a class="headerlink" href="#defaultsyaml" ti
 </span><span id="__span-0-7"><a id="__codelineno-0-7" name="__codelineno-0-7"></a><span class="w">  </span><span class="c1"># add the key-value pairs they need.</span>
 </span><span id="__span-0-8"><a id="__codelineno-0-8" name="__codelineno-0-8"></a><span class="w">  </span><span class="nt">app</span><span class="p">:</span><span class="w"> </span><span class="c1"># required</span>
 </span><span id="__span-0-9"><a id="__codelineno-0-9" name="__codelineno-0-9"></a><span class="w">    </span><span class="nt">streams</span><span class="p">:</span><span class="w"> </span><span class="c1"># required</span>
-</span><span id="__span-0-10"><a id="__codelineno-0-10" name="__codelineno-0-10"></a><span class="w">      </span><span class="nt">brokers</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">${brokers}</span><span class="w"> </span><span class="c1"># required</span>
+</span><span id="__span-0-10"><a id="__codelineno-0-10" name="__codelineno-0-10"></a><span class="w">      </span><span class="nt">brokers</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">${kafka_brokers}</span><span class="w"> </span><span class="c1"># required</span>
 </span><span id="__span-0-11"><a id="__codelineno-0-11" name="__codelineno-0-11"></a><span class="w">      </span><span class="nt">schemaRegistryUrl</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">${schema_registry_url}</span>
 </span><span id="__span-0-12"><a id="__codelineno-0-12" name="__codelineno-0-12"></a><span class="w">    </span><span class="nt">nameOverride</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">override-with-this-name</span><span class="w"> </span><span class="c1"># kafka-app-specific</span>
 </span><span id="__span-0-13"><a id="__codelineno-0-13" name="__codelineno-0-13"></a><span class="w">    </span><span class="nt">imageTag</span><span class="p">:</span><span class="w"> </span><span class="s">"1.0.0"</span><span class="w"> </span><span class="c1"># Example values that are shared between streams-app and producer-app</span>
@@ -863,7 +868,7 @@ <h1 id="defaultsyaml">defaults.yaml<a class="headerlink" href="#defaultsyaml" ti
 </span><span id="__span-0-170"><a id="__codelineno-0-170" name="__codelineno-0-170"></a><span class="w">  </span><span class="c1"># https://github.com/bakdata/streams-bootstrap/tree/master/charts/producer-app</span>
 </span><span id="__span-0-171"><a id="__codelineno-0-171" name="__codelineno-0-171"></a><span class="w">  </span><span class="nt">app</span><span class="p">:</span><span class="w"> </span><span class="c1"># required</span>
 </span><span id="__span-0-172"><a id="__codelineno-0-172" name="__codelineno-0-172"></a><span class="w">    </span><span class="nt">streams</span><span class="p">:</span><span class="w"> </span><span class="c1"># required, producer-app-specific</span>
-</span><span id="__span-0-173"><a id="__codelineno-0-173" name="__codelineno-0-173"></a><span class="w">      </span><span class="nt">brokers</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">${brokers}</span><span class="w"> </span><span class="c1"># required</span>
+</span><span id="__span-0-173"><a id="__codelineno-0-173" name="__codelineno-0-173"></a><span class="w">      </span><span class="nt">brokers</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">${kafka_brokers}</span><span class="w"> </span><span class="c1"># required</span>
 </span><span id="__span-0-174"><a id="__codelineno-0-174" name="__codelineno-0-174"></a><span class="w">      </span><span class="nt">schemaRegistryUrl</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">${schema_registry_url}</span>
 </span><span id="__span-0-175"><a id="__codelineno-0-175" name="__codelineno-0-175"></a><span class="w">      </span><span class="nt">outputTopic</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">output_topic</span>
 </span><span id="__span-0-176"><a id="__codelineno-0-176" name="__codelineno-0-176"></a><span class="w">      </span><span class="nt">extraOutputTopics</span><span class="p">:</span>
@@ -881,7 +886,7 @@ <h1 id="defaultsyaml">defaults.yaml<a class="headerlink" href="#defaultsyaml" ti
 </span><span id="__span-0-188"><a id="__codelineno-0-188" name="__codelineno-0-188"></a><span class="w">  </span><span class="nt">app</span><span class="p">:</span><span class="w"> </span><span class="c1"># required</span>
 </span><span id="__span-0-189"><a id="__codelineno-0-189" name="__codelineno-0-189"></a><span class="w">    </span><span class="c1"># Streams Bootstrap streams section</span>
 </span><span id="__span-0-190"><a id="__codelineno-0-190" name="__codelineno-0-190"></a><span class="w">    </span><span class="nt">streams</span><span class="p">:</span><span class="w"> </span><span class="c1"># required, streams-app-specific</span>
-</span><span id="__span-0-191"><a id="__codelineno-0-191" name="__codelineno-0-191"></a><span class="w">      </span><span class="nt">brokers</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">${brokers}</span><span class="w"> </span><span class="c1"># required</span>
+</span><span id="__span-0-191"><a id="__codelineno-0-191" name="__codelineno-0-191"></a><span class="w">      </span><span class="nt">brokers</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">${kafka_brokers}</span><span class="w"> </span><span class="c1"># required</span>
 </span><span id="__span-0-192"><a id="__codelineno-0-192" name="__codelineno-0-192"></a><span class="w">      </span><span class="nt">schemaRegistryUrl</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">${schema_registry_url}</span>
 </span><span id="__span-0-193"><a id="__codelineno-0-193" name="__codelineno-0-193"></a><span class="w">      </span><span class="nt">inputTopics</span><span class="p">:</span>
 </span><span id="__span-0-194"><a id="__codelineno-0-194" name="__codelineno-0-194"></a><span class="w">        </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">topic1</span>
diff --git a/dev/resources/variables/cli_env_vars/index.html b/dev/resources/variables/cli_env_vars/index.html
index d2bcb5498..1f2cb8435 100644
--- a/dev/resources/variables/cli_env_vars/index.html
+++ b/dev/resources/variables/cli_env_vars/index.html
@@ -368,6 +368,11 @@
         Migrate from v1 to v2
       </a>
 </li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../../user/migration-guide/v2-v3/">
+        Migrate from v2 to v3
+      </a>
+</li>
 </ul>
 </nav>
 </li>
diff --git a/dev/resources/variables/config_env_vars.env b/dev/resources/variables/config_env_vars.env
index 308fb6334..00bef6a4c 100644
--- a/dev/resources/variables/config_env_vars.env
+++ b/dev/resources/variables/config_env_vars.env
@@ -9,18 +9,18 @@
 # Suffix your environment files with this value (e.g.
 # defaults_development.yaml for environment=development).
 KPOPS_ENVIRONMENT # No default value, required
-# brokers
+# kafka_brokers
 # The comma separated Kafka brokers address.
 KPOPS_KAFKA_BROKERS # No default value, required
-# schema_registry_url
+# url
 # Address of the Schema Registry.
-KPOPS_SCHEMA_REGISTRY_URL # No default value, not required
-# kafka_rest_host
+KPOPS_SCHEMA_REGISTRY_URL=http://localhost:8081
+# url
 # Address of the Kafka REST Proxy.
-KPOPS_REST_PROXY_HOST # No default value, not required
-# kafka_connect_host
+KPOPS_KAFKA_REST_URL=http://localhost:8082
+# url
 # Address of Kafka Connect.
-KPOPS_CONNECT_HOST # No default value, not required
+KPOPS_KAFKA_CONNECT_URL=http://localhost:8083
 # timeout
 # The timeout in seconds that specifies when actions like deletion or
 # deploy timeout.
diff --git a/dev/resources/variables/config_env_vars/index.html b/dev/resources/variables/config_env_vars/index.html
index dbdbbf6ac..635de0758 100644
--- a/dev/resources/variables/config_env_vars/index.html
+++ b/dev/resources/variables/config_env_vars/index.html
@@ -368,6 +368,11 @@
         Migrate from v1 to v2
       </a>
 </li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../../user/migration-guide/v2-v3/">
+        Migrate from v2 to v3
+      </a>
+</li>
 </ul>
 </nav>
 </li>
@@ -472,28 +477,28 @@ <h1>Config env vars</h1>
 <td></td>
 <td>True</td>
 <td>The comma separated Kafka brokers address.</td>
-<td>brokers</td>
+<td>kafka_brokers</td>
 </tr>
 <tr>
 <td>KPOPS_SCHEMA_REGISTRY_URL</td>
-<td></td>
+<td>http://localhost:8081</td>
 <td>False</td>
 <td>Address of the Schema Registry.</td>
-<td>schema_registry_url</td>
+<td>url</td>
 </tr>
 <tr>
-<td>KPOPS_REST_PROXY_HOST</td>
-<td></td>
+<td>KPOPS_KAFKA_REST_URL</td>
+<td>http://localhost:8082</td>
 <td>False</td>
 <td>Address of the Kafka REST Proxy.</td>
-<td>kafka_rest_host</td>
+<td>url</td>
 </tr>
 <tr>
-<td>KPOPS_CONNECT_HOST</td>
-<td></td>
+<td>KPOPS_KAFKA_CONNECT_URL</td>
+<td>http://localhost:8083</td>
 <td>False</td>
 <td>Address of Kafka Connect.</td>
-<td>kafka_connect_host</td>
+<td>url</td>
 </tr>
 <tr>
 <td>KPOPS_TIMEOUT</td>
diff --git a/dev/schema/config.json b/dev/schema/config.json
index a2f18eb6b..a0841dae3 100644
--- a/dev/schema/config.json
+++ b/dev/schema/config.json
@@ -1,8 +1,8 @@
 {
-    "$ref": "#/definitions/PipelineConfig",
+    "$ref": "#/definitions/KpopsConfig",
     "definitions": {
         "HelmConfig": {
-            "description": "Global Helm configuration.",
+            "description": "Global Helm configuration",
             "properties": {
                 "api_version": {
                     "description": "Kubernetes API version used for Capabilities.APIVersions",
@@ -41,25 +41,57 @@
             "title": "HelmDiffConfig",
             "type": "object"
         },
-        "PipelineConfig": {
+        "KafkaConnectConfig": {
             "additionalProperties": false,
-            "description": "Pipeline configuration unrelated to the components.",
+            "description": "Configuration for Kafka Connect.",
             "properties": {
-                "brokers": {
-                    "description": "The comma separated Kafka brokers address.",
-                    "env": "KPOPS_KAFKA_BROKERS",
+                "url": {
+                    "default": "http://localhost:8083",
+                    "description": "Address of Kafka Connect.",
+                    "env": "KPOPS_KAFKA_CONNECT_URL",
                     "env_names": [
-                        "kpops_kafka_brokers"
+                        "kpops_kafka_connect_url"
                     ],
-                    "example": "broker1:9092,broker2:9092,broker3:9092",
-                    "title": "Brokers",
+                    "format": "uri",
+                    "maxLength": 65536,
+                    "minLength": 1,
+                    "title": "Url",
                     "type": "string"
-                },
+                }
+            },
+            "title": "KafkaConnectConfig",
+            "type": "object"
+        },
+        "KafkaRestConfig": {
+            "additionalProperties": false,
+            "description": "Configuration for Kafka REST Proxy.",
+            "properties": {
+                "url": {
+                    "default": "http://localhost:8082",
+                    "description": "Address of the Kafka REST Proxy.",
+                    "env": "KPOPS_KAFKA_REST_URL",
+                    "env_names": [
+                        "kpops_kafka_rest_url"
+                    ],
+                    "format": "uri",
+                    "maxLength": 65536,
+                    "minLength": 1,
+                    "title": "Url",
+                    "type": "string"
+                }
+            },
+            "title": "KafkaRestConfig",
+            "type": "object"
+        },
+        "KpopsConfig": {
+            "additionalProperties": false,
+            "description": "Pipeline configuration unrelated to the components.",
+            "properties": {
                 "create_namespace": {
                     "default": false,
                     "description": "Flag for `helm upgrade --install`. Create the release namespace if not present.",
                     "env_names": [
-                        "create_namespace"
+                        "kpops_create_namespace"
                     ],
                     "title": "Create Namespace",
                     "type": "boolean"
@@ -68,7 +100,7 @@
                     "default": "defaults",
                     "description": "The name of the defaults file and the prefix of the defaults environment file.",
                     "env_names": [
-                        "defaults_filename_prefix"
+                        "kpops_defaults_filename_prefix"
                     ],
                     "title": "Defaults Filename Prefix",
                     "type": "string"
@@ -77,7 +109,7 @@
                     "default": ".",
                     "description": "The path to the folder containing the defaults.yaml file and the environment defaults files. Paths can either be absolute or relative to `config.yaml`",
                     "env_names": [
-                        "defaults_path"
+                        "kpops_defaults_path"
                     ],
                     "example": "defaults",
                     "format": "path",
@@ -107,7 +139,7 @@
                     },
                     "description": "Global flags for Helm.",
                     "env_names": [
-                        "helm_config"
+                        "kpops_helm_config"
                     ],
                     "title": "Helm Config"
                 },
@@ -122,29 +154,49 @@
                     },
                     "description": "Configure Helm Diff.",
                     "env_names": [
-                        "helm_diff_config"
+                        "kpops_helm_diff_config"
                     ],
                     "title": "Helm Diff Config"
                 },
-                "kafka_connect_host": {
-                    "description": "Address of Kafka Connect.",
-                    "env": "KPOPS_CONNECT_HOST",
+                "kafka_brokers": {
+                    "description": "The comma separated Kafka brokers address.",
+                    "env": "KPOPS_KAFKA_BROKERS",
                     "env_names": [
-                        "kpops_connect_host"
+                        "kpops_kafka_brokers"
                     ],
-                    "example": "http://localhost:8083",
-                    "title": "Kafka Connect Host",
+                    "example": "broker1:9092,broker2:9092,broker3:9092",
+                    "title": "Kafka Brokers",
                     "type": "string"
                 },
-                "kafka_rest_host": {
-                    "description": "Address of the Kafka REST Proxy.",
-                    "env": "KPOPS_REST_PROXY_HOST",
+                "kafka_connect": {
+                    "allOf": [
+                        {
+                            "$ref": "#/definitions/KafkaConnectConfig"
+                        }
+                    ],
+                    "default": {
+                        "url": "http://localhost:8083"
+                    },
+                    "description": "Configuration for Kafka Connect.",
                     "env_names": [
-                        "kpops_rest_proxy_host"
+                        "kpops_kafka_connect"
                     ],
-                    "example": "http://localhost:8082",
-                    "title": "Kafka Rest Host",
-                    "type": "string"
+                    "title": "Kafka Connect"
+                },
+                "kafka_rest": {
+                    "allOf": [
+                        {
+                            "$ref": "#/definitions/KafkaRestConfig"
+                        }
+                    ],
+                    "default": {
+                        "url": "http://localhost:8082"
+                    },
+                    "description": "Configuration for Kafka REST Proxy.",
+                    "env_names": [
+                        "kpops_kafka_rest"
+                    ],
+                    "title": "Kafka Rest"
                 },
                 "retain_clean_jobs": {
                     "default": false,
@@ -156,15 +208,21 @@
                     "title": "Retain Clean Jobs",
                     "type": "boolean"
                 },
-                "schema_registry_url": {
-                    "description": "Address of the Schema Registry.",
-                    "env": "KPOPS_SCHEMA_REGISTRY_URL",
+                "schema_registry": {
+                    "allOf": [
+                        {
+                            "$ref": "#/definitions/SchemaRegistryConfig"
+                        }
+                    ],
+                    "default": {
+                        "enabled": false,
+                        "url": "http://localhost:8081"
+                    },
+                    "description": "Configuration for Schema Registry.",
                     "env_names": [
-                        "kpops_schema_registry_url"
+                        "kpops_schema_registry"
                     ],
-                    "example": "http://localhost:8081",
-                    "title": "Schema Registry Url",
-                    "type": "string"
+                    "title": "Schema Registry"
                 },
                 "timeout": {
                     "default": 300,
@@ -188,21 +246,51 @@
                     },
                     "description": "Configure the topic name variables you can use in the pipeline definition.",
                     "env_names": [
-                        "topic_name_config"
+                        "kpops_topic_name_config"
                     ],
                     "title": "Topic Name Config"
                 }
             },
             "required": [
                 "environment",
-                "brokers"
+                "kafka_brokers"
             ],
-            "title": "PipelineConfig",
+            "title": "KpopsConfig",
+            "type": "object"
+        },
+        "SchemaRegistryConfig": {
+            "additionalProperties": false,
+            "description": "Configuration for Schema Registry.",
+            "properties": {
+                "enabled": {
+                    "default": false,
+                    "description": "Whether the Schema Registry handler should be initialized.",
+                    "env_names": [
+                        "enabled"
+                    ],
+                    "title": "Enabled",
+                    "type": "boolean"
+                },
+                "url": {
+                    "default": "http://localhost:8081",
+                    "description": "Address of the Schema Registry.",
+                    "env": "KPOPS_SCHEMA_REGISTRY_URL",
+                    "env_names": [
+                        "kpops_schema_registry_url"
+                    ],
+                    "format": "uri",
+                    "maxLength": 65536,
+                    "minLength": 1,
+                    "title": "Url",
+                    "type": "string"
+                }
+            },
+            "title": "SchemaRegistryConfig",
             "type": "object"
         },
         "TopicNameConfig": {
             "additionalProperties": false,
-            "description": "Configures topic names.",
+            "description": "Configure the topic name variables you can use in the pipeline definition.",
             "properties": {
                 "default_error_topic_name": {
                     "default": "${pipeline_name}-${component_name}-error",
diff --git a/dev/schema/pipeline.json b/dev/schema/pipeline.json
index 7e77b0ddd..9695ec9a2 100644
--- a/dev/schema/pipeline.json
+++ b/dev/schema/pipeline.json
@@ -2,7 +2,7 @@
     "definitions": {
         "FromSection": {
             "additionalProperties": false,
-            "description": "Holds multiple input topics.",
+            "description": "Holds multiple input topics",
             "properties": {
                 "components": {
                     "additionalProperties": {
@@ -28,7 +28,7 @@
         },
         "FromTopic": {
             "additionalProperties": false,
-            "description": "Input topic.",
+            "description": "Input topic",
             "properties": {
                 "role": {
                     "description": "Custom identifier belonging to a topic; define only if `type` is `pattern` or `None`",
@@ -48,7 +48,7 @@
             "type": "object"
         },
         "HelmRepoConfig": {
-            "description": "Helm repository configuration.",
+            "description": "Helm repository configuration",
             "properties": {
                 "repo_auth_flags": {
                     "allOf": [
@@ -85,7 +85,7 @@
             "type": "object"
         },
         "InputTopicTypes": {
-            "description": "Input topic types.\n\nINPUT (input topic), PATTERN (extra-topic-pattern or input-topic-pattern)",
+            "description": "Input topic types\n\nINPUT (input topic), PATTERN (extra-topic-pattern or input-topic-pattern)",
             "enum": [
                 "input",
                 "pattern"
@@ -97,7 +97,7 @@
             "additionalProperties": {
                 "type": "string"
             },
-            "description": "Settings specific to Kafka Connectors.",
+            "description": "Settings specific to Kafka Connectors",
             "properties": {
                 "connector.class": {
                     "title": "Connector.Class",
@@ -111,7 +111,7 @@
             "type": "object"
         },
         "KafkaSinkConnector": {
-            "description": "Kafka sink connector model.",
+            "description": "Kafka sink connector model",
             "properties": {
                 "app": {
                     "allOf": [
@@ -183,7 +183,7 @@
                 },
                 "type": {
                     "default": "kafka-sink-connector",
-                    "description": "Kafka sink connector model.",
+                    "description": "Kafka sink connector model",
                     "enum": [
                         "kafka-sink-connector"
                     ],
@@ -206,7 +206,7 @@
             "type": "object"
         },
         "KafkaSourceConnector": {
-            "description": "Kafka source connector model.",
+            "description": "Kafka source connector model",
             "properties": {
                 "app": {
                     "allOf": [
@@ -283,7 +283,7 @@
                 },
                 "type": {
                     "default": "kafka-source-connector",
-                    "description": "Kafka source connector model.",
+                    "description": "Kafka source connector model",
                     "enum": [
                         "kafka-source-connector"
                     ],
@@ -384,13 +384,13 @@
             "type": "object"
         },
         "KubernetesAppConfig": {
-            "description": "Settings specific to Kubernetes Apps.",
+            "description": "Settings specific to Kubernetes Apps",
             "properties": {},
             "title": "KubernetesAppConfig",
             "type": "object"
         },
         "OutputTopicTypes": {
-            "description": "Types of output topic.\n\nOUTPUT (output topic), ERROR (error topic)",
+            "description": "Types of output topic\n\nOUTPUT (output topic), ERROR (error topic)",
             "enum": [
                 "output",
                 "error"
@@ -399,7 +399,7 @@
             "type": "string"
         },
         "ProducerApp": {
-            "description": "Producer component.\nThis producer holds configuration to use as values for the streams bootstrap producer helm chart. Note that the producer does not support error topics.",
+            "description": "Producer component\nThis producer holds configuration to use as values for the streams bootstrap producer helm chart. Note that the producer does not support error topics.",
             "properties": {
                 "app": {
                     "allOf": [
@@ -462,7 +462,7 @@
                 },
                 "type": {
                     "default": "producer-app",
-                    "description": "Producer component.\nThis producer holds configuration to use as values for the streams bootstrap producer helm chart. Note that the producer does not support error topics.",
+                    "description": "Producer component\nThis producer holds configuration to use as values for the streams bootstrap producer helm chart. Note that the producer does not support error topics.",
                     "enum": [
                         "producer-app"
                     ],
@@ -485,7 +485,7 @@
             "type": "object"
         },
         "ProducerStreamsConfig": {
-            "description": "Kafka Streams settings specific to Producer.",
+            "description": "Kafka Streams settings specific to Producer",
             "properties": {
                 "brokers": {
                     "description": "Brokers",
@@ -519,7 +519,7 @@
             "type": "object"
         },
         "ProducerValues": {
-            "description": "Settings specific to producers.",
+            "description": "Settings specific to producers",
             "properties": {
                 "nameOverride": {
                     "description": "Override name with this value",
@@ -543,7 +543,7 @@
             "type": "object"
         },
         "RepoAuthFlags": {
-            "description": "Authorisation-related flags for `helm repo`.",
+            "description": "Authorisation-related flags for `helm repo`",
             "properties": {
                 "ca_file": {
                     "description": "Path to CA bundle file to verify certificates of HTTPS-enabled servers",
@@ -578,7 +578,7 @@
             "type": "object"
         },
         "StreamsApp": {
-            "description": "StreamsApp component that configures a streams bootstrap app.",
+            "description": "StreamsApp component that configures a streams bootstrap app",
             "properties": {
                 "app": {
                     "allOf": [
@@ -645,7 +645,7 @@
                 },
                 "type": {
                     "default": "streams-app",
-                    "description": "StreamsApp component that configures a streams bootstrap app.",
+                    "description": "StreamsApp component that configures a streams bootstrap app",
                     "enum": [
                         "streams-app"
                     ],
@@ -668,7 +668,7 @@
             "type": "object"
         },
         "StreamsAppAutoScaling": {
-            "description": "Kubernetes Event-driven Autoscaling config.",
+            "description": "Kubernetes Event-driven Autoscaling config",
             "properties": {
                 "consumerGroup": {
                     "description": "Name of the consumer group used for checking the offset on the topic and processing the related lag.",
@@ -771,7 +771,7 @@
             "type": "object"
         },
         "StreamsConfig": {
-            "description": "Streams Bootstrap streams section.",
+            "description": "Streams Bootstrap streams section",
             "properties": {
                 "brokers": {
                     "description": "Brokers",
@@ -854,7 +854,7 @@
             "type": "object"
         },
         "ToSection": {
-            "description": "Holds multiple output topics.",
+            "description": "Holds multiple output topics",
             "properties": {
                 "models": {
                     "additionalProperties": {
@@ -880,7 +880,7 @@
         },
         "TopicConfig": {
             "additionalProperties": false,
-            "description": "Configure an output topic.",
+            "description": "Configure an output topic",
             "properties": {
                 "configs": {
                     "additionalProperties": {
diff --git a/dev/search/search_index.json b/dev/search/search_index.json
index c4be88c36..f0098b40b 100644
--- a/dev/search/search_index.json
+++ b/dev/search/search_index.json
@@ -1 +1 @@
-{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"developer/auto-generation/","title":"Auto generation","text":"<p>Auto generation happens mostly with <code>pre-commit</code> hooks. You can find the pre-commit configuration here. These pre-commit hooks call different Python scripts to auto generate code for the documentation.</p>"},{"location":"developer/auto-generation/#generation-scripts-and-their-respective-files","title":"Generation scripts and their respective files","text":""},{"location":"developer/auto-generation/#documentation","title":"Documentation","text":""},{"location":"developer/auto-generation/#variables","title":"Variables","text":"<ul> <li><code>cli_env_vars.env</code> -- All CLI environment variables in a <code>dotenv</code> file.</li> <li><code>cli_env_vars.md</code> -- All CLI environment variables in a table.</li> <li><code>config_env_vars.env</code> -- Almost all pipeline config environment variables in a <code>dotenv</code> file. The script checks for each field in <code>PipelineConfig</code> whether it has an <code>env</code> attribute defined. The script is currently unable to visit the classes of fields like <code>topic_name_config</code>, hence any environment variables defined there would remain unknown to it.</li> <li><code>config_env_vars.env</code> -- Almost all pipeline config environment variables in a table.</li> <li><code>variable_substitution.yaml</code> -- A copy of <code>./tests/pipeline/resources/component-type-substitution/pipeline.yaml</code> used as an example of substitution.</li> </ul>"},{"location":"developer/auto-generation/#cli-commands","title":"CLI commands","text":"<p>Generated by <code>typer-cli</code> from the code in <code>main.py</code>. It is called with Python's <code>subprocess</code> module.</p>"},{"location":"developer/auto-generation/#pipeline-and-defaults-example-definitions","title":"Pipeline and defaults example definitions","text":"<p>Generates example <code>pipeline.yaml</code> and <code>defaults.yaml</code> for each individual component, stores them and also concatenates them into 1 big pipeline definition and 1 big pipeline defaults definition.</p> <p>User input</p> <ul> <li><code>headers/*\\.yaml</code> -- The top of each example. Includes a description comment, <code>type</code> and <code>name</code>. The headers for <code>pipeline.yaml</code> reside in the <code>pipeline-components</code> dir and the <code>defaults.yaml</code> headers reside in the <code>pipeline-defaults</code> dir. The names of the files must be equal to the respective component <code>type</code>.</li> <li><code>sections/*\\.yaml</code> -- Each YAML file contains a single section (component attribute) definition. The intention is to keep the minimal set of definitions there from which any component definition can be built. The names of the files must be equal to the respective component <code>type</code> and the attribute name. The sections are used for both <code>defaults.yaml</code> and <code>pipeline.yaml</code> generation and reside in the <code>pipeline-components</code> dir.</li> </ul> <p>Generated</p> <ul> <li><code>pipeline-components/dependencies/*</code>   Cached information about KPOps components</li> <li><code>pipeline_component_dependencies.yaml</code> -- Specifies per component which files in the <code>sections</code> dir should be used for the <code>pipeline.yaml</code> generation.</li> <li><code>defaults_pipeline_component_dependencies.yaml</code> -- Specifies per component which files in the <code>sections</code> dir should be used for the <code>defaults.yaml</code> generation.</li> <li><code>kpops_structure.yaml</code> -- Specifies the inheritance hierarchy of the components and what sections exist in each component.</li> <li><code>pipeline-components/*\\.yaml</code> -- All single-component pipeline definitions and one big (complete) <code>pipeline.yaml</code> that contains all of them.</li> <li><code>pipeline-defaults/*\\.yaml</code> -- All single-component defaults definitions and one big (complete) <code>defaults.yaml</code> that contains all of them.</li> </ul>"},{"location":"developer/auto-generation/#editor-integration","title":"Editor integration","text":""},{"location":"developer/auto-generation/#schemas","title":"Schemas","text":"<ul> <li>config.json</li> <li>pipeline.json</li> </ul>"},{"location":"developer/formatting/","title":"Formatting","text":""},{"location":"developer/formatting/#markdown","title":"Markdown","text":"<p>To ensure a consistent markdown style, we use dprint to check and reformat.</p> <pre><code>dprint fmt\n</code></pre> <p>Use the official documentation to set up dprint. The configuration can be found here.</p>"},{"location":"resources/architecture/components-hierarchy/","title":"Components hierarchy","text":"<pre><code>flowchart BT\n    KubernetesApp --&gt; PipelineComponent\n    KafkaConnector --&gt; PipelineComponent\n    KafkaApp --&gt; KubernetesApp\n    StreamsApp --&gt; KafkaApp\n    ProducerApp --&gt; KafkaApp\n    KafkaSourceConnector --&gt; KafkaConnector\n    KafkaSinkConnector --&gt; KafkaConnector\n    \n    click KubernetesApp \"../kubernetes-app\"\n    click KafkaApp \"../kafka-app\"\n    click StreamsApp \"../streams-app\"\n    click ProducerApp \"../producer-app\"\n    click KafkaConnector \"../kafka-connector\"\n    click KafkaSourceConnector \"../kafka-source-connector\"\n    click KafkaSinkConnector \"../kafka-sink-connector\"</code></pre> <p>KPOps component hierarchy</p>"},{"location":"resources/examples/defaults/","title":"Example <code>defaults.yaml</code> files","text":""},{"location":"resources/examples/defaults/#atm-fraud-pipeline","title":"ATM Fraud Pipeline","text":"defaults.yaml <pre><code>pipeline-component:\nprefix: \"\"\nkubernetes-app:\nnamespace: ${NAMESPACE}\nkafka-connector:\nnamespace: ${NAMESPACE}\nkafka-app:\napp:\nstreams:\nbrokers: ${brokers}\nschemaRegistryUrl: ${schema_registry_url}\noptimizeLeaveGroupBehavior: false\nproducer-app:\nto:\ntopics:\n${output_topic_name}:\npartitions_count: 3\nstreams-app:\napp:\nlabels:\npipeline: ${pipeline_name}\nstreams:\noptimizeLeaveGroupBehavior: false\nto:\ntopics:\n${error_topic_name}:\ntype: error\npartitions_count: 1\n${output_topic_name}:\npartitions_count: 3\n</code></pre>"},{"location":"resources/examples/defaults/#word-count-pipeline","title":"Word-count Pipeline","text":"defaults.yaml <pre><code>pipeline-component:\nprefix: word-count-\nnamespace: kpops\nkafka-app:\napp:\nstreams:\nbrokers: ${brokers}\nprometheus:\njmx:\nenabled: false\ndebug: true\nproducer-app:\nto:\ntopics:\n${output_topic_name}:\ntype: output\npartitions_count: 1\nstreams-app:\napp:\nlabels:\npipeline: ${pipeline_name}\ncommandLine:\nPRODUCTIVE: false\nto:\ntopics:\n${error_topic_name}:\ntype: error\npartitions_count: 1\n${output_topic_name}:\ntype: output\npartitions_count: 1\n</code></pre>"},{"location":"resources/examples/pipeline/","title":"Example <code>pipeline.yaml</code> files","text":""},{"location":"resources/examples/pipeline/#atm-fraud-pipeline","title":"ATM Fraud Pipeline","text":"pipeline.yaml <pre><code> - type: producer-app\nname: account-producer\napp:\nreplicaCount: 1\nimage: ${DOCKER_REGISTRY}/atm-demo-accountproducer\nimageTag: 1.0.0\nschedule: 0 12 * * *\nsuspend: true\nprometheus:\njmx:\nenabled: false\ndebug: true\n- type: producer-app\nname: transaction-avro-producer\napp:\nreplicaCount: 1\nimage: ${DOCKER_REGISTRY}/atm-demo-transactionavroproducer\nimageTag: 1.0.0\ncommandLine:\nREAL_TX: 19\nITERATION: 20\nschedule: 0 12 * * *\nsuspend: true\nprometheus:\njmx:\nenabled: false\ndebug: true\n- type: streams-app\nname: transaction-joiner\napp:\nreplicaCount: 1\nimage: ${DOCKER_REGISTRY}/atm-demo-transactionjoiner\nimageTag: 1.0.0\ncommandLine:\nPRODUCTIVE: false\nannotations:\nconsumerGroup: atm-transactionjoiner-atm-fraud-joinedtransactions-topic\nprometheus:\njmx:\nenabled: false\ndebug: true\n- type: streams-app\nname: fraud-detector\napp:\nreplicaCount: 1\nimage: ${DOCKER_REGISTRY}/atm-demo-frauddetector\nimageTag: 1.0.0\ncommandLine:\nPRODUCTIVE: false\nannotations:\nconsumerGroup: atm-frauddetector-atm-fraud-possiblefraudtransactions-topic\nprometheus:\njmx:\nenabled: false\ndebug: true\n- type: streams-app\nname: account-linker\nfrom:\ncomponents:\nfraud-detector:\ntype: input\naccount-producer:\nrole: accounts\napp:\nreplicaCount: 1\nimage: ${DOCKER_REGISTRY}/atm-demo-accountlinker\nimageTag: 1.0.0\ncommandLine:\nPRODUCTIVE: false\nannotations:\nconsumerGroup: atm-accountlinker-atm-fraud-output-topic\nprometheus:\njmx:\nenabled: false\ndebug: true\n- type: kafka-sink-connector\nname: postgresql-connector\napp:\nconnector.class: io.confluent.connect.jdbc.JdbcSinkConnector\ntasks.max: 1\ntopics: ${pipeline_name}-account-linker-topic\nconnection.url: jdbc:postgresql://postgresql-dev.kpops.svc.cluster.local:5432/app_db\nconnection.user: app1\nconnection.password: AppPassword\nconnection.ds.pool.size: 5\ninsert.mode: insert\ninsert.mode.databaselevel: true\nvalue.converter: io.confluent.connect.avro.AvroConverter\nvalue.converter.schema.registry.url: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081\nkey.converter: org.apache.kafka.connect.storage.StringConverter\ntransforms: flatten\ntransforms.flatten.type: org.apache.kafka.connect.transforms.Flatten$Value\nauto.create: true\ntable.name.format: fraud_transactions\nerrors.deadletterqueue.context.headers.enable: true\nerrors.deadletterqueue.topic.name: postgres-request-sink-dead-letters\nerrors.deadletterqueue.topic.replication.factor: 1\nerrors.tolerance: all\npk.mode: record_value\n</code></pre>"},{"location":"resources/examples/pipeline/#word-count-pipeline","title":"Word-count Pipeline","text":"pipeline.yaml <pre><code>- type: producer-app\nname: data-producer\napp:\nimage: bakdata/kpops-demo-sentence-producer\nimageTag: \"1.0.0\"\n- type: streams-app\nname: word-counter\nto:\ntopics:\n${output_topic_name}:\ntype: output\nconfigs:\ncleanup.policy: compact\napp:\nreplicaCount: 1\nimage: bakdata/kpops-demo-word-count-app\nimageTag: \"1.0.0\"\n- type: kafka-sink-connector\nname: redis-sink-connector\napp:\nconnector.class: com.github.jcustenborder.kafka.connect.redis.RedisSinkConnector\nredis.hosts: redis-headless:6379\nredis.database: 0\ntasks.max: 1\nkey.converter: org.apache.kafka.connect.storage.StringConverter\nvalue.converter: org.apache.kafka.connect.storage.StringConverter\n</code></pre>"},{"location":"resources/pipeline-components/pipeline/","title":"pipeline.yaml","text":"<pre><code># Base component for Kafka-based components.\n# Producer or streaming apps should inherit from this class.\n- type: kafka-app # required\nname: kafka-app # required\n# Pipeline prefix that will prefix every component name. If you wish to not\n# have any prefix you can specify an empty string.\nprefix: ${pipeline_name}-\nfrom: # Must not be null\ntopics: # read from topic\n${pipeline_name}-input-topic:\ntype: input # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra\n${pipeline_name}-input-pattern-topic:\ntype: pattern # Implied to be an input pattern if `role` is undefined\n${pipeline_name}-extra-pattern-topic:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\ncomponents: # read from specific component\naccount-producer:\ntype: output # Implied when role is NOT specified\nother-producer:\nrole: some-role # Implies `type` to be extra\ncomponent-as-input-pattern:\ntype: pattern # Implied to be an input pattern if `role` is undefined\ncomponent-as-extra-pattern:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\n# Topic(s) into which the component will write output\nto:\ntopics:\n${pipeline_name}-output-topic:\ntype: output # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined\n${pipeline_name}-error-topic:\ntype: error\n# Currently KPOps supports Avro and JSON schemas.\nkey_schema: key-schema # must implement SchemaProvider to use\nvalue_schema: value-schema\npartitions_count: 1\nreplication_factor: 1\nconfigs: # https://kafka.apache.org/documentation/#topicconfigs\ncleanup.policy: compact\nmodels: # SchemaProvider is initiated with the values given here\nmodel: model\nnamespace: namespace # required\n# `app` can contain application-specific settings, hence  the user is free to\n# add the key-value pairs they need.\napp: # required\nstreams: # required\nbrokers: ${brokers} # required\nschemaRegistryUrl: ${schema_registry_url}\nnameOverride: override-with-this-name # kafka-app-specific\nimageTag: \"1.0.0\" # Example values that are shared between streams-app and producer-app\n# Helm repository configuration (optional)\n# If not set the helm repo add will not be called. Useful when using local Helm charts\nrepo_config:\nrepository_name: bakdata-streams-bootstrap # required\nurl: https://bakdata.github.io/streams-bootstrap/ # required\nrepo_auth_flags:\nusername: user\npassword: pass\nca_file: /home/user/path/to/ca-file\ninsecure_skip_tls_verify: false\nversion: \"2.12.0\" # Helm chart version\n# Kafka sink connector\n- type: kafka-sink-connector\nname: kafka-sink-connector # required\n# Pipeline prefix that will prefix every component name. If you wish to not\n# have any prefix you can specify an empty string.\nprefix: ${pipeline_name}-\nfrom: # Must not be null\ntopics: # read from topic\n${pipeline_name}-input-topic:\ntype: input # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra\n${pipeline_name}-input-pattern-topic:\ntype: pattern # Implied to be an input pattern if `role` is undefined\n${pipeline_name}-extra-pattern-topic:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\ncomponents: # read from specific component\naccount-producer:\ntype: output # Implied when role is NOT specified\nother-producer:\nrole: some-role # Implies `type` to be extra\ncomponent-as-input-pattern:\ntype: pattern # Implied to be an input pattern if `role` is undefined\ncomponent-as-extra-pattern:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\n# Topic(s) into which the component will write output\nto:\ntopics:\n${pipeline_name}-output-topic:\ntype: output # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined\n${pipeline_name}-error-topic:\ntype: error\n# Currently KPOps supports Avro and JSON schemas.\nkey_schema: key-schema # must implement SchemaProvider to use\nvalue_schema: value-schema\npartitions_count: 1\nreplication_factor: 1\nconfigs: # https://kafka.apache.org/documentation/#topicconfigs\ncleanup.policy: compact\nmodels: # SchemaProvider is initiated with the values given here\nmodel: model\nnamespace: namespace # required\n# `app` contains application-specific settings, hence it does not have a rigid\n# structure. The fields below are just an example. Extensive documentation on\n# connectors: https://kafka.apache.org/documentation/#connectconfigs\napp: # required\ntasks.max: 1\n# Helm repository configuration for resetter\nrepo_config:\nrepository_name: my-repo # required\nurl: https://bakdata.github.io/kafka-connect-resetter/ # required\nrepo_auth_flags:\nusername: user\npassword: pass\nca_file: /home/user/path/to/ca-file\ninsecure_skip_tls_verify: false\nversion: \"1.0.6\" # Helm chart version\n# Overriding Kafka Connect Resetter Helm values. E.g. to override the\n# Image Tag etc.\nresetter_values:\nimageTag: \"1.2.3\"\n# Kafka source connector\n- type: kafka-source-connector # required\nname: kafka-source-connector # required\n# Pipeline prefix that will prefix every component name. If you wish to not\n# have any prefix you can specify an empty string.\nprefix: ${pipeline_name}-\n# The source connector has no `from` section\n# from:\n# Topic(s) into which the component will write output\nto:\ntopics:\n${pipeline_name}-output-topic:\ntype: output # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined\n${pipeline_name}-error-topic:\ntype: error\n# Currently KPOps supports Avro and JSON schemas.\nkey_schema: key-schema # must implement SchemaProvider to use\nvalue_schema: value-schema\npartitions_count: 1\nreplication_factor: 1\nconfigs: # https://kafka.apache.org/documentation/#topicconfigs\ncleanup.policy: compact\nmodels: # SchemaProvider is initiated with the values given here\nmodel: model\nnamespace: namespace # required\n# `app` contains application-specific settings, hence it does not have a rigid\n# structure. The fields below are just an example. Extensive documentation on\n# connectors: https://kafka.apache.org/documentation/#connectconfigs\napp: # required\ntasks.max: 1\n# Helm repository configuration for resetter\nrepo_config:\nrepository_name: my-repo # required\nurl: https://bakdata.github.io/kafka-connect-resetter/ # required\nrepo_auth_flags:\nusername: user\npassword: pass\nca_file: /home/user/path/to/ca-file\ninsecure_skip_tls_verify: false\nversion: \"1.0.6\" # Helm chart version\n# Overriding Kafka Connect Resetter Helm values. E.g. to override the\n# Image Tag etc.\nresetter_values:\nimageTag: \"1.2.3\"\n# offset.storage.topic\n# https://kafka.apache.org/documentation/#connect_running\noffset_topic: offset_topic\n# Base Kubernetes App\n- type: kubernetes-app\nname: kubernetes-app # required\n# Pipeline prefix that will prefix every component name. If you wish to not\n# have any prefix you can specify an empty string.\nprefix: ${pipeline_name}-\nfrom: # Must not be null\ntopics: # read from topic\n${pipeline_name}-input-topic:\ntype: input # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra\n${pipeline_name}-input-pattern-topic:\ntype: pattern # Implied to be an input pattern if `role` is undefined\n${pipeline_name}-extra-pattern-topic:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\ncomponents: # read from specific component\naccount-producer:\ntype: output # Implied when role is NOT specified\nother-producer:\nrole: some-role # Implies `type` to be extra\ncomponent-as-input-pattern:\ntype: pattern # Implied to be an input pattern if `role` is undefined\ncomponent-as-extra-pattern:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\n# Topic(s) into which the component will write output\nto:\ntopics:\n${pipeline_name}-output-topic:\ntype: output # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined\n${pipeline_name}-error-topic:\ntype: error\n# Currently KPOps supports Avro and JSON schemas.\nkey_schema: key-schema # must implement SchemaProvider to use\nvalue_schema: value-schema\npartitions_count: 1\nreplication_factor: 1\nconfigs: # https://kafka.apache.org/documentation/#topicconfigs\ncleanup.policy: compact\nmodels: # SchemaProvider is initiated with the values given here\nmodel: model\nnamespace: namespace # required\n# `app` contains application-specific settings, hence it does not have a rigid\n# structure. The fields below are just an example.\napp: # required\nimage: exampleImage # Example\ndebug: false # Example\ncommandLine: {} # Example\n# Helm repository configuration (optional)\n# If not set the helm repo add will not be called. Useful when using local Helm charts\nrepo_config:\nrepository_name: bakdata-streams-bootstrap # required\nurl: https://bakdata.github.io/streams-bootstrap/ # required\nrepo_auth_flags:\nusername: user\npassword: pass\nca_file: /home/user/path/to/ca-file\ninsecure_skip_tls_verify: false\nversion: \"1.0.0\" # Helm chart version\n# Holds configuration to use as values for the streams bootstrap producer-app Helm\n# chart.\n# More documentation on ProducerApp:\n# https://github.com/bakdata/streams-bootstrap\n- type: producer-app\nname: producer-app # required\n# Pipeline prefix that will prefix every component name. If you wish to not\n# have any prefix you can specify an empty string.\nprefix: ${pipeline_name}-\n# from: # While the producer-app does inherit from kafka-app, it does not need a\n# `from` section, hence it does not support it.\n# Topic(s) into which the component will write output\nto:\ntopics:\n${pipeline_name}-output-topic:\ntype: output # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined\n${pipeline_name}-error-topic:\ntype: error\n# Currently KPOps supports Avro and JSON schemas.\nkey_schema: key-schema # must implement SchemaProvider to use\nvalue_schema: value-schema\npartitions_count: 1\nreplication_factor: 1\nconfigs: # https://kafka.apache.org/documentation/#topicconfigs\ncleanup.policy: compact\nmodels: # SchemaProvider is initiated with the values given here\nmodel: model\nnamespace: namespace # required\n# Allowed configs:\n# https://github.com/bakdata/streams-bootstrap/tree/master/charts/producer-app\napp: # required\nstreams: # required, producer-app-specific\nbrokers: ${brokers} # required\nschemaRegistryUrl: ${schema_registry_url}\noutputTopic: output_topic\nextraOutputTopics:\noutput_role1: output_topic1\noutput_role2: output_topic2\nnameOverride: override-with-this-name # kafka-app-specific\n# Helm repository configuration (optional)\n# If not set the helm repo add will not be called. Useful when using local Helm charts\nrepo_config:\nrepository_name: bakdata-streams-bootstrap # required\nurl: https://bakdata.github.io/streams-bootstrap/ # required\nrepo_auth_flags:\nusername: user\npassword: pass\nca_file: /home/user/path/to/ca-file\ninsecure_skip_tls_verify: false\nversion: \"2.12.0\" # Helm chart version\n# StreamsApp component that configures a streams bootstrap app.\n# More documentation on StreamsApp: https://github.com/bakdata/streams-bootstrap\n- type: streams-app # required\nname: streams-app # required\n# Pipeline prefix that will prefix every component name. If you wish to not\n# have any prefix you can specify an empty string.\nprefix: ${pipeline_name}-\nfrom: # Must not be null\ntopics: # read from topic\n${pipeline_name}-input-topic:\ntype: input # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra\n${pipeline_name}-input-pattern-topic:\ntype: pattern # Implied to be an input pattern if `role` is undefined\n${pipeline_name}-extra-pattern-topic:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\ncomponents: # read from specific component\naccount-producer:\ntype: output # Implied when role is NOT specified\nother-producer:\nrole: some-role # Implies `type` to be extra\ncomponent-as-input-pattern:\ntype: pattern # Implied to be an input pattern if `role` is undefined\ncomponent-as-extra-pattern:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\n# Topic(s) into which the component will write output\nto:\ntopics:\n${pipeline_name}-output-topic:\ntype: output # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined\n${pipeline_name}-error-topic:\ntype: error\n# Currently KPOps supports Avro and JSON schemas.\nkey_schema: key-schema # must implement SchemaProvider to use\nvalue_schema: value-schema\npartitions_count: 1\nreplication_factor: 1\nconfigs: # https://kafka.apache.org/documentation/#topicconfigs\ncleanup.policy: compact\nmodels: # SchemaProvider is initiated with the values given here\nmodel: model\nnamespace: namespace # required\n# No arbitrary keys are allowed under `app`here\n# Allowed configs:\n# https://github.com/bakdata/streams-bootstrap/tree/master/charts/streams-app\napp: # required\n# Streams Bootstrap streams section\nstreams: # required, streams-app-specific\nbrokers: ${brokers} # required\nschemaRegistryUrl: ${schema_registry_url}\ninputTopics:\n- topic1\n- topic2\noutputTopic: output-topic\ninputPattern: input-pattern\nextraInputTopics:\ninput_role1:\n- input_topic1\n- input_topic2\ninput_role2:\n- input_topic3\n- input_topic4\nextraInputPatterns:\npattern_role1: input_pattern1\nextraOutputTopics:\noutput_role1: output_topic1\noutput_role2: output_topic2\nerrorTopic: error-topic\nconfig:\nmy.streams.config: my.value\nnameOverride: override-with-this-name # streams-app-specific\nautoscaling: # streams-app-specific\nconsumerGroup: consumer-group # required\nlagThreshold: 0 # Average target value to trigger scaling actions.\nenabled: false # Whether to enable auto-scaling using KEDA.\n# This is the interval to check each trigger on.\n# https://keda.sh/docs/2.9/concepts/scaling-deployments/#pollinginterval\npollingInterval: 30\n# The period to wait after the last trigger reported active before scaling\n#  the resource back to 0. https://keda.sh/docs/2.9/concepts/scaling-deployments/#cooldownperiod\ncooldownPeriod: 300\n# The offset reset policy for the consumer if the the consumer group is\n# not yet subscribed to a partition.\noffsetResetPolicy: earliest\n# This setting is passed to the HPA definition that KEDA will create for a\n# given resource and holds the maximum number of replicas of the target resouce.\n# https://keda.sh/docs/2.9/concepts/scaling-deployments/#maxreplicacount\nmaxReplicas: 1\n# Minimum number of replicas KEDA will scale the resource down to.\n# https://keda.sh/docs/2.7/concepts/scaling-deployments/#minreplicacount\nminReplicas: 0\n# If this property is set, KEDA will scale the resource down to this\n# number of replicas.\n# https://keda.sh/docs/2.9/concepts/scaling-deployments/#idlereplicacount\nidleReplicas: 0\ntopics: # List of auto-generated Kafka Streams topics used by the streams app.\n- topic1\n- topic2\n# Helm repository configuration (optional)\n# If not set the helm repo add will not be called. Useful when using local Helm charts\nrepo_config:\nrepository_name: bakdata-streams-bootstrap # required\nurl: https://bakdata.github.io/streams-bootstrap/ # required\nrepo_auth_flags:\nusername: user\npassword: pass\nca_file: /home/user/path/to/ca-file\ninsecure_skip_tls_verify: false\nversion: \"2.12.0\" # Helm chart version\n</code></pre>"},{"location":"resources/pipeline-defaults/defaults/","title":"defaults.yaml","text":"<pre><code># Base component for Kafka-based components.\n#\n# Parent of: ProducerApp, StreamsApp\n# Child of: KubernetesApp\nkafka-app:\n# `app` can contain application-specific settings, hence  the user is free to\n# add the key-value pairs they need.\napp: # required\nstreams: # required\nbrokers: ${brokers} # required\nschemaRegistryUrl: ${schema_registry_url}\nnameOverride: override-with-this-name # kafka-app-specific\nimageTag: \"1.0.0\" # Example values that are shared between streams-app and producer-app\nversion: \"2.12.0\" # Helm chart version\n# Kafka connector\n#\n# Parent of: KafkaSinkConnector, KafkaSourceConnector\n# Child of: PipelineComponent\nkafka-connector:\n# Pipeline prefix that will prefix every component name. If you wish to not\n# have any prefix you can specify an empty string.\nprefix: ${pipeline_name}-\nfrom: # Must not be null\ntopics: # read from topic\n${pipeline_name}-input-topic:\ntype: input # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra\n${pipeline_name}-input-pattern-topic:\ntype: pattern # Implied to be an input pattern if `role` is undefined\n${pipeline_name}-extra-pattern-topic:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\ncomponents: # read from specific component\naccount-producer:\ntype: output # Implied when role is NOT specified\nother-producer:\nrole: some-role # Implies `type` to be extra\ncomponent-as-input-pattern:\ntype: pattern # Implied to be an input pattern if `role` is undefined\ncomponent-as-extra-pattern:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\n# Topic(s) into which the component will write output\nto:\ntopics:\n${pipeline_name}-output-topic:\ntype: output # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined\n${pipeline_name}-error-topic:\ntype: error\n# Currently KPOps supports Avro and JSON schemas.\nkey_schema: key-schema # must implement SchemaProvider to use\nvalue_schema: value-schema\npartitions_count: 1\nreplication_factor: 1\nconfigs: # https://kafka.apache.org/documentation/#topicconfigs\ncleanup.policy: compact\nmodels: # SchemaProvider is initiated with the values given here\nmodel: model\nnamespace: namespace # required\n# `app` contains application-specific settings, hence it does not have a rigid\n# structure. The fields below are just an example. Extensive documentation on\n# connectors: https://kafka.apache.org/documentation/#connectconfigs\napp: # required\ntasks.max: 1\n# Helm repository configuration for resetter\nrepo_config:\nrepository_name: my-repo # required\nurl: https://bakdata.github.io/kafka-connect-resetter/ # required\nrepo_auth_flags:\nusername: user\npassword: pass\nca_file: /home/user/path/to/ca-file\ninsecure_skip_tls_verify: false\nversion: \"1.0.6\" # Helm chart version\n# Overriding Kafka Connect Resetter Helm values. E.g. to override the\n# Image Tag etc.\nresetter_values:\nimageTag: \"1.2.3\"\n# Kafka sink connector\n#\n# Child of: KafkaConnector\nkafka-sink-connector:\n# No settings differ from `kafka-connector`\n# Kafka source connector\n#\n# Child of: KafkaConnector\nkafka-source-connector:\n# The source connector has no `from` section\n# from:\n# offset.storage.topic\n# https://kafka.apache.org/documentation/#connect_running\noffset_topic: offset_topic\n# Base Kubernetes App\n#\n# Parent of: KafkaApp\n# Child of: PipelineComponent\nkubernetes-app:\n# Pipeline prefix that will prefix every component name. If you wish to not\n# have any prefix you can specify an empty string.\nprefix: ${pipeline_name}-\nfrom: # Must not be null\ntopics: # read from topic\n${pipeline_name}-input-topic:\ntype: input # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra\n${pipeline_name}-input-pattern-topic:\ntype: pattern # Implied to be an input pattern if `role` is undefined\n${pipeline_name}-extra-pattern-topic:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\ncomponents: # read from specific component\naccount-producer:\ntype: output # Implied when role is NOT specified\nother-producer:\nrole: some-role # Implies `type` to be extra\ncomponent-as-input-pattern:\ntype: pattern # Implied to be an input pattern if `role` is undefined\ncomponent-as-extra-pattern:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\n# Topic(s) into which the component will write output\nto:\ntopics:\n${pipeline_name}-output-topic:\ntype: output # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined\n${pipeline_name}-error-topic:\ntype: error\n# Currently KPOps supports Avro and JSON schemas.\nkey_schema: key-schema # must implement SchemaProvider to use\nvalue_schema: value-schema\npartitions_count: 1\nreplication_factor: 1\nconfigs: # https://kafka.apache.org/documentation/#topicconfigs\ncleanup.policy: compact\nmodels: # SchemaProvider is initiated with the values given here\nmodel: model\nnamespace: namespace # required\n# `app` contains application-specific settings, hence it does not have a rigid\n# structure. The fields below are just an example.\napp: # required\nimage: exampleImage # Example\ndebug: false # Example\ncommandLine: {} # Example\n# Helm repository configuration (optional)\n# If not set the helm repo add will not be called. Useful when using local Helm charts\nrepo_config:\nrepository_name: bakdata-streams-bootstrap # required\nurl: https://bakdata.github.io/streams-bootstrap/ # required\nrepo_auth_flags:\nusername: user\npassword: pass\nca_file: /home/user/path/to/ca-file\ninsecure_skip_tls_verify: false\nversion: \"1.0.0\" # Helm chart version\n# Holds configuration to use as values for the streams bootstrap producer-app Helm\n# chart.\n#\n# Child of: KafkaApp\n# More documentation on ProducerApp: https://github.com/bakdata/streams-bootstrap\nproducer-app:\n# from: # While the producer-app does inherit from kafka-app, it does not need a\n# `from` section, hence it does not support it.\n# Allowed configs:\n# https://github.com/bakdata/streams-bootstrap/tree/master/charts/producer-app\napp: # required\nstreams: # required, producer-app-specific\nbrokers: ${brokers} # required\nschemaRegistryUrl: ${schema_registry_url}\noutputTopic: output_topic\nextraOutputTopics:\noutput_role1: output_topic1\noutput_role2: output_topic2\nnameOverride: override-with-this-name # kafka-app-specific\n# StreamsApp component that configures a streams bootstrap app.\n#\n# Child of: KafkaApp\n# More documentation on StreamsApp: https://github.com/bakdata/streams-bootstrap\nstreams-app:\n# No arbitrary keys are allowed under `app`here\n# Allowed configs:\n# https://github.com/bakdata/streams-bootstrap/tree/master/charts/streams-app\napp: # required\n# Streams Bootstrap streams section\nstreams: # required, streams-app-specific\nbrokers: ${brokers} # required\nschemaRegistryUrl: ${schema_registry_url}\ninputTopics:\n- topic1\n- topic2\noutputTopic: output-topic\ninputPattern: input-pattern\nextraInputTopics:\ninput_role1:\n- input_topic1\n- input_topic2\ninput_role2:\n- input_topic3\n- input_topic4\nextraInputPatterns:\npattern_role1: input_pattern1\nextraOutputTopics:\noutput_role1: output_topic1\noutput_role2: output_topic2\nerrorTopic: error-topic\nconfig:\nmy.streams.config: my.value\nnameOverride: override-with-this-name # streams-app-specific\nautoscaling: # streams-app-specific\nconsumerGroup: consumer-group # required\nlagThreshold: 0 # Average target value to trigger scaling actions.\nenabled: false # Whether to enable auto-scaling using KEDA.\n# This is the interval to check each trigger on.\n# https://keda.sh/docs/2.9/concepts/scaling-deployments/#pollinginterval\npollingInterval: 30\n# The period to wait after the last trigger reported active before scaling\n#  the resource back to 0. https://keda.sh/docs/2.9/concepts/scaling-deployments/#cooldownperiod\ncooldownPeriod: 300\n# The offset reset policy for the consumer if the the consumer group is\n# not yet subscribed to a partition.\noffsetResetPolicy: earliest\n# This setting is passed to the HPA definition that KEDA will create for a\n# given resource and holds the maximum number of replicas of the target resouce.\n# https://keda.sh/docs/2.9/concepts/scaling-deployments/#maxreplicacount\nmaxReplicas: 1\n# Minimum number of replicas KEDA will scale the resource down to.\n# https://keda.sh/docs/2.7/concepts/scaling-deployments/#minreplicacount\nminReplicas: 0\n# If this property is set, KEDA will scale the resource down to this\n# number of replicas.\n# https://keda.sh/docs/2.9/concepts/scaling-deployments/#idlereplicacount\nidleReplicas: 0\ntopics: # List of auto-generated Kafka Streams topics used by the streams app.\n- topic1\n- topic2\n</code></pre>"},{"location":"resources/variables/cli_env_vars/","title":"Cli env vars","text":"<p>These variables are a lower priority alternative to the commands' flags. If a variable is set, the corresponding flag does not have to be specified in commands. Variables marked as required can instead be set as flags.</p> Name Default Value Required Description KPOPS_PIPELINE_BASE_DIR . False Base directory to the pipelines (default is current working directory) KPOPS_CONFIG_PATH config.yaml False Path to the config.yaml file KPOPS_DEFAULT_PATH False Path to defaults folder KPOPS_PIPELINE_PATH True Path to YAML with pipeline definition KPOPS_PIPELINE_STEPS False Comma separated list of steps to apply the command on"},{"location":"resources/variables/config_env_vars/","title":"Config env vars","text":"<p>These variables are a lower priority alternative to the settings in <code>config.yaml</code>. Variables marked as required can instead be set in the pipeline config.</p> Name Default Value Required Description Setting name KPOPS_ENVIRONMENT True The environment you want to generate and deploy the pipeline to. Suffix your environment files with this value (e.g. defaults_development.yaml for environment=development). environment KPOPS_KAFKA_BROKERS True The comma separated Kafka brokers address. brokers KPOPS_SCHEMA_REGISTRY_URL False Address of the Schema Registry. schema_registry_url KPOPS_REST_PROXY_HOST False Address of the Kafka REST Proxy. kafka_rest_host KPOPS_CONNECT_HOST False Address of Kafka Connect. kafka_connect_host KPOPS_TIMEOUT 300 False The timeout in seconds that specifies when actions like deletion or deploy timeout. timeout KPOPS_RETAIN_CLEAN_JOBS False False Whether to retain clean up jobs in the cluster or uninstall the, after completion. retain_clean_jobs"},{"location":"user/what-is-kpops/","title":"What is KPOps?","text":"<p>With a couple of easy commands in the shell and a <code>pipeline.yaml</code> of under 30 lines, KPOps can not only <code>deploy</code> a Kafka pipeline1 to a Kubernetes cluster, but also <code>reset</code>, <code>clean</code> or <code>destroy</code> it!</p>"},{"location":"user/what-is-kpops/#key-features","title":"Key features","text":"<ul> <li>Deploy Kafka apps to Kubernetes: KPOps allows to deploy consecutive Kafka Streams applications and producers using an easy-to-read and -write pipeline definition.</li> <li>Manage Kafka Connectors: KPOps connects with your Kafka Connect cluster and deploys, validates, and deletes your connectors.</li> <li>Configure multiple pipelines and steps: KPOps has various abstractions that simplify configuring multiple pipelines and steps within pipelines by sharing common configuration between different components, such as producers or streaming applications.</li> <li>Handle your topics and schemas: KPOps not only creates and deletes your topics but also registers and deletes your schemas.</li> <li>Clean termination of Kafka components: KPOps removes your pipeline components (i.e., Kafka Streams applications) from the Kubernetes cluster and cleans up the component-related states (i.e., removing/resetting offset of Kafka consumer groups).</li> <li>Preview your pipeline changes: With the KPOps dry-run, you can ensure your pipeline definition is set up correctly. This helps to minimize downtime and prevent potential errors or issues that could impact your production environment.</li> </ul>"},{"location":"user/what-is-kpops/#example","title":"Example","text":"An overview of Word-count pipeline shown in Streams Explorer Word-count pipeline.yaml<pre><code>- type: producer-app\nname: data-producer\napp:\nimage: bakdata/kpops-demo-sentence-producer\nimageTag: \"1.0.0\"\n- type: streams-app\nname: word-counter\nto:\ntopics:\n${output_topic_name}:\ntype: output\nconfigs:\ncleanup.policy: compact\napp:\nreplicaCount: 1\nimage: bakdata/kpops-demo-word-count-app\nimageTag: \"1.0.0\"\n- type: kafka-sink-connector\nname: redis-sink-connector\napp:\nconnector.class: com.github.jcustenborder.kafka.connect.redis.RedisSinkConnector\nredis.hosts: redis-headless:6379\nredis.database: 0\ntasks.max: 1\nkey.converter: org.apache.kafka.connect.storage.StringConverter\nvalue.converter: org.apache.kafka.connect.storage.StringConverter\n</code></pre> <ol> <li> <p>A Kafka pipeline can consist of consecutive streaming applications, producers, and connectors.\u00a0\u21a9</p> </li> </ol>"},{"location":"user/core-concepts/config/","title":"Configuration","text":"<p>KPOps reads its global configuration that is unrelated to a pipeline's components from <code>config.yaml</code>.</p> <p>Consider enabling KPOps' editor integration feature to enjoy the benefits of autocompletion and validation when configuring your pipeline.</p> <p>To learn about any of the available settings, take a look at the example below.</p> <code>config.yaml</code> <pre><code># CONFIGURATION\n#\n# The path to the folder containing the defaults.yaml file and the environment\n# defaults files.\ndefaults_path: .\n# The environment you want to generate and deploy the pipeline to. Suffix your\n# environment files with this value (e.g. defaults_development.yaml and\n# pipeline_development.yaml for environment=development).\n# REQUIRED\nenvironment: development\n# The Kafka brokers address.\n# REQUIRED\nbrokers: \"http://broker1:9092,http://broker2:9092\"\n# The name of the defaults file and the prefix of the defaults environment file.\ndefaults_filename_prefix: defaults\n# Configures topic names.\ntopic_name_config: # Configures the value for the variable ${output_topic_name}\ndefault_output_topic_name: ${pipeline_name}-${component_name}\n# Configures the value for the variable ${error_topic_name}\ndefault_error_topic_name: ${pipeline_name}-${component_name}-error\n# Address of the Schema Registry\nschema_registry_url: \"http://localhost:8081\"\n# Address of the Kafka REST Proxy.\nkafka_rest_host: \"http://localhost:8082\"\n# Address of Kafka Connect.\nkafka_connect_host: \"http://localhost:8083\"\n# The timeout in seconds that specifies when actions like deletion or deploy\n# timeout.\ntimeout: 300\n# Flag for `helm upgrade --install`.\n# Create the release namespace if not present.\ncreate_namespace: false\n# Global flags for Helm.\nhelm_config:\n# Set the name of the kubeconfig context. (--kube-context)\ncontext: name\n# Run Helm in Debug mode.\ndebug: false\n# Configure Helm Diff.\nhelm_diff_config: # Set of keys that should not be checked.\nignore: - name\n- imageTag\n# Whether to retain clean up jobs in the cluster or uninstall the, after\n# completion.\nretain_clean_jobs: false\n</code></pre> <p>Environment-specific pipeline definitions</p> <p>Similarly to defaults, it is possible to have an unlimited amount of additional environment-specific pipeline definitions. The naming convention is the same: add a suffix of the form <code>_{environment}</code> to the filename.</p>"},{"location":"user/core-concepts/defaults/","title":"Defaults","text":"<p>KPOps has a very efficient way of dealing with repeating settings which manifests as <code>defaults.yaml</code>. This file provides the user with the power to set defaults for any and all components, thus omitting the need to repeat the same settings in <code>pipeline.yaml</code>.</p> <p>See real-world examples for <code>defaults</code>.</p>"},{"location":"user/core-concepts/defaults/#features","title":"Features","text":""},{"location":"user/core-concepts/defaults/#inheritance","title":"Inheritance","text":"<p>An important mechanic of KPOps is that <code>defaults</code> set for a component apply to all components that inherit from it.</p> <p>It is possible, although not recommended, to add settings that are specific to a component's subclass. An example would be configuring <code>offset_topic</code> under <code>kafka-connector</code> instead of <code>kafka-source-connector</code>.</p>"},{"location":"user/core-concepts/defaults/#configuration","title":"Configuration","text":"<p>It is possible to set specific <code>defaults</code> for each <code>environment</code> by adding files called <code>defaults_{environment}.yaml</code> to the defaults folder at <code>defaults_path</code>. The defaults are loaded based on the currently set environment.</p> <p>It is important to note that <code>defaults_{environment}.yaml</code> overrides only the settings that are explicitly set to be different from the ones in the base <code>defaults</code> file.</p> <p>Tip</p> <p><code>defaults</code> is the default value of <code>defaults_filename_prefix</code>. Together with <code>defaults_path</code> and <code>environment</code> it can be changed in <code>config.yaml</code></p>"},{"location":"user/core-concepts/defaults/#components","title":"Components","text":"<p>The <code>defaults</code> codeblocks in this section contain the full set of settings that are specific to the component. If a setting already exists in a parent config, it will not be included in the child's.</p>"},{"location":"user/core-concepts/defaults/#kubernetesapp","title":"KubernetesApp","text":"<code>defaults.yaml</code> <pre><code># Base Kubernetes App\n#\n# Parent of: KafkaApp\n# Child of: PipelineComponent\nkubernetes-app:\n# Pipeline prefix that will prefix every component name. If you wish to not\n# have any prefix you can specify an empty string.\nprefix: ${pipeline_name}-\nfrom: # Must not be null\ntopics: # read from topic\n${pipeline_name}-input-topic:\ntype: input # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra\n${pipeline_name}-input-pattern-topic:\ntype: pattern # Implied to be an input pattern if `role` is undefined\n${pipeline_name}-extra-pattern-topic:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\ncomponents: # read from specific component\naccount-producer:\ntype: output # Implied when role is NOT specified\nother-producer:\nrole: some-role # Implies `type` to be extra\ncomponent-as-input-pattern:\ntype: pattern # Implied to be an input pattern if `role` is undefined\ncomponent-as-extra-pattern:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\n# Topic(s) into which the component will write output\nto:\ntopics:\n${pipeline_name}-output-topic:\ntype: output # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined\n${pipeline_name}-error-topic:\ntype: error\n# Currently KPOps supports Avro and JSON schemas.\nkey_schema: key-schema # must implement SchemaProvider to use\nvalue_schema: value-schema\npartitions_count: 1\nreplication_factor: 1\nconfigs: # https://kafka.apache.org/documentation/#topicconfigs\ncleanup.policy: compact\nmodels: # SchemaProvider is initiated with the values given here\nmodel: model\nnamespace: namespace # required\n# `app` contains application-specific settings, hence it does not have a rigid\n# structure. The fields below are just an example.\napp: # required\nimage: exampleImage # Example\ndebug: false # Example\ncommandLine: {} # Example\n# Helm repository configuration (optional)\n# If not set the helm repo add will not be called. Useful when using local Helm charts\nrepo_config:\nrepository_name: bakdata-streams-bootstrap # required\nurl: https://bakdata.github.io/streams-bootstrap/ # required\nrepo_auth_flags:\nusername: user\npassword: pass\nca_file: /home/user/path/to/ca-file\ninsecure_skip_tls_verify: false\nversion: \"1.0.0\" # Helm chart version\n</code></pre>"},{"location":"user/core-concepts/defaults/#kafkaapp","title":"KafkaApp","text":"<code>defaults.yaml</code> <pre><code># Base component for Kafka-based components.\n#\n# Parent of: ProducerApp, StreamsApp\n# Child of: KubernetesApp\nkafka-app:\n# `app` can contain application-specific settings, hence  the user is free to\n# add the key-value pairs they need.\napp: # required\nstreams: # required\nbrokers: ${brokers} # required\nschemaRegistryUrl: ${schema_registry_url}\nnameOverride: override-with-this-name # kafka-app-specific\nimageTag: \"1.0.0\" # Example values that are shared between streams-app and producer-app\nversion: \"2.12.0\" # Helm chart version\n</code></pre>"},{"location":"user/core-concepts/defaults/#streamsapp","title":"StreamsApp","text":"<code>defaults.yaml</code> <pre><code># StreamsApp component that configures a streams bootstrap app.\n#\n# Child of: KafkaApp\n# More documentation on StreamsApp: https://github.com/bakdata/streams-bootstrap\nstreams-app:\n# No arbitrary keys are allowed under `app`here\n# Allowed configs:\n# https://github.com/bakdata/streams-bootstrap/tree/master/charts/streams-app\napp: # required\n# Streams Bootstrap streams section\nstreams: # required, streams-app-specific\nbrokers: ${brokers} # required\nschemaRegistryUrl: ${schema_registry_url}\ninputTopics:\n- topic1\n- topic2\noutputTopic: output-topic\ninputPattern: input-pattern\nextraInputTopics:\ninput_role1:\n- input_topic1\n- input_topic2\ninput_role2:\n- input_topic3\n- input_topic4\nextraInputPatterns:\npattern_role1: input_pattern1\nextraOutputTopics:\noutput_role1: output_topic1\noutput_role2: output_topic2\nerrorTopic: error-topic\nconfig:\nmy.streams.config: my.value\nnameOverride: override-with-this-name # streams-app-specific\nautoscaling: # streams-app-specific\nconsumerGroup: consumer-group # required\nlagThreshold: 0 # Average target value to trigger scaling actions.\nenabled: false # Whether to enable auto-scaling using KEDA.\n# This is the interval to check each trigger on.\n# https://keda.sh/docs/2.9/concepts/scaling-deployments/#pollinginterval\npollingInterval: 30\n# The period to wait after the last trigger reported active before scaling\n#  the resource back to 0. https://keda.sh/docs/2.9/concepts/scaling-deployments/#cooldownperiod\ncooldownPeriod: 300\n# The offset reset policy for the consumer if the the consumer group is\n# not yet subscribed to a partition.\noffsetResetPolicy: earliest\n# This setting is passed to the HPA definition that KEDA will create for a\n# given resource and holds the maximum number of replicas of the target resouce.\n# https://keda.sh/docs/2.9/concepts/scaling-deployments/#maxreplicacount\nmaxReplicas: 1\n# Minimum number of replicas KEDA will scale the resource down to.\n# https://keda.sh/docs/2.7/concepts/scaling-deployments/#minreplicacount\nminReplicas: 0\n# If this property is set, KEDA will scale the resource down to this\n# number of replicas.\n# https://keda.sh/docs/2.9/concepts/scaling-deployments/#idlereplicacount\nidleReplicas: 0\ntopics: # List of auto-generated Kafka Streams topics used by the streams app.\n- topic1\n- topic2\n</code></pre>"},{"location":"user/core-concepts/defaults/#producerapp","title":"ProducerApp","text":"<code>defaults.yaml</code> <pre><code>\n</code></pre>"},{"location":"user/core-concepts/defaults/#kafkaconnector","title":"KafkaConnector","text":"<code>defaults.yaml</code> <pre><code># Kafka connector\n#\n# Parent of: KafkaSinkConnector, KafkaSourceConnector\n# Child of: PipelineComponent\nkafka-connector:\n# Pipeline prefix that will prefix every component name. If you wish to not\n# have any prefix you can specify an empty string.\nprefix: ${pipeline_name}-\nfrom: # Must not be null\ntopics: # read from topic\n${pipeline_name}-input-topic:\ntype: input # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra\n${pipeline_name}-input-pattern-topic:\ntype: pattern # Implied to be an input pattern if `role` is undefined\n${pipeline_name}-extra-pattern-topic:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\ncomponents: # read from specific component\naccount-producer:\ntype: output # Implied when role is NOT specified\nother-producer:\nrole: some-role # Implies `type` to be extra\ncomponent-as-input-pattern:\ntype: pattern # Implied to be an input pattern if `role` is undefined\ncomponent-as-extra-pattern:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\n# Topic(s) into which the component will write output\nto:\ntopics:\n${pipeline_name}-output-topic:\ntype: output # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined\n${pipeline_name}-error-topic:\ntype: error\n# Currently KPOps supports Avro and JSON schemas.\nkey_schema: key-schema # must implement SchemaProvider to use\nvalue_schema: value-schema\npartitions_count: 1\nreplication_factor: 1\nconfigs: # https://kafka.apache.org/documentation/#topicconfigs\ncleanup.policy: compact\nmodels: # SchemaProvider is initiated with the values given here\nmodel: model\nnamespace: namespace # required\n# `app` contains application-specific settings, hence it does not have a rigid\n# structure. The fields below are just an example. Extensive documentation on\n# connectors: https://kafka.apache.org/documentation/#connectconfigs\napp: # required\ntasks.max: 1\n# Helm repository configuration for resetter\nrepo_config:\nrepository_name: my-repo # required\nurl: https://bakdata.github.io/kafka-connect-resetter/ # required\nrepo_auth_flags:\nusername: user\npassword: pass\nca_file: /home/user/path/to/ca-file\ninsecure_skip_tls_verify: false\nversion: \"1.0.6\" # Helm chart version\n# Overriding Kafka Connect Resetter Helm values. E.g. to override the\n# Image Tag etc.\nresetter_values:\nimageTag: \"1.2.3\"\n</code></pre>"},{"location":"user/core-concepts/defaults/#kafkasourceconnector","title":"KafkaSourceConnector","text":"<code>defaults.yaml</code> <pre><code># Kafka source connector\n#\n# Child of: KafkaConnector\nkafka-source-connector:\n# The source connector has no `from` section\n# from:\n# offset.storage.topic\n# https://kafka.apache.org/documentation/#connect_running\noffset_topic: offset_topic\n</code></pre>"},{"location":"user/core-concepts/defaults/#kafkasinkconnector","title":"KafkaSinkConnector","text":"<code>defaults.yaml</code> <pre><code># Kafka sink connector\n#\n# Child of: KafkaConnector\nkafka-sink-connector:\n# No settings differ from `kafka-connector`\n</code></pre>"},{"location":"user/core-concepts/components/kafka-app/","title":"KafkaApp","text":"<p>Subclass of KubernetesApp.</p>"},{"location":"user/core-concepts/components/kafka-app/#usage","title":"Usage","text":"<ul> <li>Defines a streams-bootstrap component</li> <li>Should not be used in <code>pipeline.yaml</code> as the component can be defined as either a StreamsApp or a ProducerApp</li> <li>Often used in <code>defaults.yaml</code></li> </ul>"},{"location":"user/core-concepts/components/kafka-app/#configuration","title":"Configuration","text":"<code>pipeline.yaml</code> <pre><code># Base component for Kafka-based components.\n# Producer or streaming apps should inherit from this class.\n- type: kafka-app # required\nname: kafka-app # required\n# Pipeline prefix that will prefix every component name. If you wish to not\n# have any prefix you can specify an empty string.\nprefix: ${pipeline_name}-\nfrom: # Must not be null\ntopics: # read from topic\n${pipeline_name}-input-topic:\ntype: input # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra\n${pipeline_name}-input-pattern-topic:\ntype: pattern # Implied to be an input pattern if `role` is undefined\n${pipeline_name}-extra-pattern-topic:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\ncomponents: # read from specific component\naccount-producer:\ntype: output # Implied when role is NOT specified\nother-producer:\nrole: some-role # Implies `type` to be extra\ncomponent-as-input-pattern:\ntype: pattern # Implied to be an input pattern if `role` is undefined\ncomponent-as-extra-pattern:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\n# Topic(s) into which the component will write output\nto:\ntopics:\n${pipeline_name}-output-topic:\ntype: output # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined\n${pipeline_name}-error-topic:\ntype: error\n# Currently KPOps supports Avro and JSON schemas.\nkey_schema: key-schema # must implement SchemaProvider to use\nvalue_schema: value-schema\npartitions_count: 1\nreplication_factor: 1\nconfigs: # https://kafka.apache.org/documentation/#topicconfigs\ncleanup.policy: compact\nmodels: # SchemaProvider is initiated with the values given here\nmodel: model\nnamespace: namespace # required\n# `app` can contain application-specific settings, hence  the user is free to\n# add the key-value pairs they need.\napp: # required\nstreams: # required\nbrokers: ${brokers} # required\nschemaRegistryUrl: ${schema_registry_url}\nnameOverride: override-with-this-name # kafka-app-specific\nimageTag: \"1.0.0\" # Example values that are shared between streams-app and producer-app\n# Helm repository configuration (optional)\n# If not set the helm repo add will not be called. Useful when using local Helm charts\nrepo_config:\nrepository_name: bakdata-streams-bootstrap # required\nurl: https://bakdata.github.io/streams-bootstrap/ # required\nrepo_auth_flags:\nusername: user\npassword: pass\nca_file: /home/user/path/to/ca-file\ninsecure_skip_tls_verify: false\nversion: \"2.12.0\" # Helm chart version\n</code></pre>"},{"location":"user/core-concepts/components/kafka-app/#operations","title":"Operations","text":""},{"location":"user/core-concepts/components/kafka-app/#deploy","title":"deploy","text":"<p>In addition to KubernetesApp's <code>deploy</code>:</p> <ul> <li>Create topics if provided (optional)</li> <li>Submit Avro schemas to the registry if provided (optional)</li> </ul>"},{"location":"user/core-concepts/components/kafka-app/#destroy","title":"destroy","text":"<p>Uninstall Helm release.</p>"},{"location":"user/core-concepts/components/kafka-app/#reset","title":"reset","text":"<p>Do nothing.</p>"},{"location":"user/core-concepts/components/kafka-app/#clean","title":"clean","text":"<p>Do nothing.</p>"},{"location":"user/core-concepts/components/kafka-connector/","title":"KafkaConnector","text":"<p><code>KafkaConnector</code> is a component that deploys Kafka Connectors. Since a connector cannot be different from sink or source it is not recommended to use <code>KafkaConnector</code> for deployment in <code>pipeline.yaml</code>. Instead, <code>KafkaConnector</code> should be used in <code>defaults.yaml</code> to set defaults for all connectors in the pipeline as they can share some common settings.</p>"},{"location":"user/core-concepts/components/kafka-sink-connector/","title":"KafkaSinkConnector","text":"<p>Subclass of KafkaConnector.</p>"},{"location":"user/core-concepts/components/kafka-sink-connector/#usage","title":"Usage","text":"<p>Lets other systems pull data from Apache Kafka.</p>"},{"location":"user/core-concepts/components/kafka-sink-connector/#configuration","title":"Configuration","text":"<code>pipeline.yaml</code> <pre><code># Kafka sink connector\n- type: kafka-sink-connector\nname: kafka-sink-connector # required\n# Pipeline prefix that will prefix every component name. If you wish to not\n# have any prefix you can specify an empty string.\nprefix: ${pipeline_name}-\nfrom: # Must not be null\ntopics: # read from topic\n${pipeline_name}-input-topic:\ntype: input # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra\n${pipeline_name}-input-pattern-topic:\ntype: pattern # Implied to be an input pattern if `role` is undefined\n${pipeline_name}-extra-pattern-topic:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\ncomponents: # read from specific component\naccount-producer:\ntype: output # Implied when role is NOT specified\nother-producer:\nrole: some-role # Implies `type` to be extra\ncomponent-as-input-pattern:\ntype: pattern # Implied to be an input pattern if `role` is undefined\ncomponent-as-extra-pattern:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\n# Topic(s) into which the component will write output\nto:\ntopics:\n${pipeline_name}-output-topic:\ntype: output # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined\n${pipeline_name}-error-topic:\ntype: error\n# Currently KPOps supports Avro and JSON schemas.\nkey_schema: key-schema # must implement SchemaProvider to use\nvalue_schema: value-schema\npartitions_count: 1\nreplication_factor: 1\nconfigs: # https://kafka.apache.org/documentation/#topicconfigs\ncleanup.policy: compact\nmodels: # SchemaProvider is initiated with the values given here\nmodel: model\nnamespace: namespace # required\n# `app` contains application-specific settings, hence it does not have a rigid\n# structure. The fields below are just an example. Extensive documentation on\n# connectors: https://kafka.apache.org/documentation/#connectconfigs\napp: # required\ntasks.max: 1\n# Helm repository configuration for resetter\nrepo_config:\nrepository_name: my-repo # required\nurl: https://bakdata.github.io/kafka-connect-resetter/ # required\nrepo_auth_flags:\nusername: user\npassword: pass\nca_file: /home/user/path/to/ca-file\ninsecure_skip_tls_verify: false\nversion: \"1.0.6\" # Helm chart version\n# Overriding Kafka Connect Resetter Helm values. E.g. to override the\n# Image Tag etc.\nresetter_values:\nimageTag: \"1.2.3\"\n</code></pre>"},{"location":"user/core-concepts/components/kafka-sink-connector/#operations","title":"Operations","text":""},{"location":"user/core-concepts/components/kafka-sink-connector/#deploy","title":"deploy","text":"<ul> <li>Add the sink connector to the Kafka Connect cluster</li> <li>Create the output topics if provided (optional)</li> <li>Register schemas in the Schema Registry if provided (optional)</li> </ul>"},{"location":"user/core-concepts/components/kafka-sink-connector/#destroy","title":"destroy","text":"<p>The associated sink connector is removed from the Kafka Connect cluster.</p>"},{"location":"user/core-concepts/components/kafka-sink-connector/#reset","title":"reset","text":"<p>Reset the consumer group offsets using bakdata's sink resetter.</p>"},{"location":"user/core-concepts/components/kafka-sink-connector/#clean","title":"clean","text":"<ul> <li>Delete associated consumer group</li> <li>Delete configured error topics</li> </ul>"},{"location":"user/core-concepts/components/kafka-source-connector/","title":"KafkaSourceConnector","text":"<p>Subclass of KafkaConnector.</p>"},{"location":"user/core-concepts/components/kafka-source-connector/#usage","title":"Usage","text":"<p>Manages source connectors in your Kafka Connect cluster.</p>"},{"location":"user/core-concepts/components/kafka-source-connector/#configuration","title":"Configuration","text":"<code>pipeline.yaml</code> <pre><code># Kafka source connector\n- type: kafka-source-connector # required\nname: kafka-source-connector # required\n# Pipeline prefix that will prefix every component name. If you wish to not\n# have any prefix you can specify an empty string.\nprefix: ${pipeline_name}-\n# The source connector has no `from` section\n# from:\n# Topic(s) into which the component will write output\nto:\ntopics:\n${pipeline_name}-output-topic:\ntype: output # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined\n${pipeline_name}-error-topic:\ntype: error\n# Currently KPOps supports Avro and JSON schemas.\nkey_schema: key-schema # must implement SchemaProvider to use\nvalue_schema: value-schema\npartitions_count: 1\nreplication_factor: 1\nconfigs: # https://kafka.apache.org/documentation/#topicconfigs\ncleanup.policy: compact\nmodels: # SchemaProvider is initiated with the values given here\nmodel: model\nnamespace: namespace # required\n# `app` contains application-specific settings, hence it does not have a rigid\n# structure. The fields below are just an example. Extensive documentation on\n# connectors: https://kafka.apache.org/documentation/#connectconfigs\napp: # required\ntasks.max: 1\n# Helm repository configuration for resetter\nrepo_config:\nrepository_name: my-repo # required\nurl: https://bakdata.github.io/kafka-connect-resetter/ # required\nrepo_auth_flags:\nusername: user\npassword: pass\nca_file: /home/user/path/to/ca-file\ninsecure_skip_tls_verify: false\nversion: \"1.0.6\" # Helm chart version\n# Overriding Kafka Connect Resetter Helm values. E.g. to override the\n# Image Tag etc.\nresetter_values:\nimageTag: \"1.2.3\"\n# offset.storage.topic\n# https://kafka.apache.org/documentation/#connect_running\noffset_topic: offset_topic\n</code></pre>"},{"location":"user/core-concepts/components/kafka-source-connector/#operations","title":"Operations","text":""},{"location":"user/core-concepts/components/kafka-source-connector/#deploy","title":"deploy","text":"<ul> <li>Add the source connector to the Kafka Connect cluster</li> <li>Create the output topics if provided (optional)</li> <li>Register schemas in the Schema registry if provided (optional)</li> </ul>"},{"location":"user/core-concepts/components/kafka-source-connector/#destroy","title":"destroy","text":"<p>Remove the source connector from the Kafka Connect cluster.</p>"},{"location":"user/core-concepts/components/kafka-source-connector/#reset","title":"reset","text":"<p>Delete state associated with the connector using bakdata's sink resetter.</p>"},{"location":"user/core-concepts/components/kafka-source-connector/#clean","title":"clean","text":"<ul> <li>Delete all associated output topics</li> <li>Delete all associated schemas in the Schema Registry</li> <li>Delete state associated with the connector</li> </ul>"},{"location":"user/core-concepts/components/kubernetes-app/","title":"KubernetesApp","text":""},{"location":"user/core-concepts/components/kubernetes-app/#usage","title":"Usage","text":"<p>Can be used to deploy any app in Kubernetes using Helm, for example, a REST service that serves Kafka data.</p>"},{"location":"user/core-concepts/components/kubernetes-app/#configuration","title":"Configuration","text":"<code>pipeline.yaml</code> <pre><code># Base Kubernetes App\n- type: kubernetes-app\nname: kubernetes-app # required\n# Pipeline prefix that will prefix every component name. If you wish to not\n# have any prefix you can specify an empty string.\nprefix: ${pipeline_name}-\nfrom: # Must not be null\ntopics: # read from topic\n${pipeline_name}-input-topic:\ntype: input # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra\n${pipeline_name}-input-pattern-topic:\ntype: pattern # Implied to be an input pattern if `role` is undefined\n${pipeline_name}-extra-pattern-topic:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\ncomponents: # read from specific component\naccount-producer:\ntype: output # Implied when role is NOT specified\nother-producer:\nrole: some-role # Implies `type` to be extra\ncomponent-as-input-pattern:\ntype: pattern # Implied to be an input pattern if `role` is undefined\ncomponent-as-extra-pattern:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\n# Topic(s) into which the component will write output\nto:\ntopics:\n${pipeline_name}-output-topic:\ntype: output # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined\n${pipeline_name}-error-topic:\ntype: error\n# Currently KPOps supports Avro and JSON schemas.\nkey_schema: key-schema # must implement SchemaProvider to use\nvalue_schema: value-schema\npartitions_count: 1\nreplication_factor: 1\nconfigs: # https://kafka.apache.org/documentation/#topicconfigs\ncleanup.policy: compact\nmodels: # SchemaProvider is initiated with the values given here\nmodel: model\nnamespace: namespace # required\n# `app` contains application-specific settings, hence it does not have a rigid\n# structure. The fields below are just an example.\napp: # required\nimage: exampleImage # Example\ndebug: false # Example\ncommandLine: {} # Example\n# Helm repository configuration (optional)\n# If not set the helm repo add will not be called. Useful when using local Helm charts\nrepo_config:\nrepository_name: bakdata-streams-bootstrap # required\nurl: https://bakdata.github.io/streams-bootstrap/ # required\nrepo_auth_flags:\nusername: user\npassword: pass\nca_file: /home/user/path/to/ca-file\ninsecure_skip_tls_verify: false\nversion: \"1.0.0\" # Helm chart version\n</code></pre>"},{"location":"user/core-concepts/components/kubernetes-app/#operations","title":"Operations","text":""},{"location":"user/core-concepts/components/kubernetes-app/#deploy","title":"deploy","text":"<p>Deploy using Helm.</p>"},{"location":"user/core-concepts/components/kubernetes-app/#destroy","title":"destroy","text":"<p>Uninstall Helm release.</p>"},{"location":"user/core-concepts/components/kubernetes-app/#reset","title":"reset","text":"<p>Do nothing.</p>"},{"location":"user/core-concepts/components/kubernetes-app/#clean","title":"clean","text":"<p>Do nothing.</p>"},{"location":"user/core-concepts/components/overview/","title":"Overview","text":"<p>This section explains the different components of KPOps, their usage and configuration in the pipeline definition <code>pipeline.yaml</code>.</p> <pre><code>flowchart BT\n    KubernetesApp --&gt; PipelineComponent\n    KafkaConnector --&gt; PipelineComponent\n    KafkaApp --&gt; KubernetesApp\n    StreamsApp --&gt; KafkaApp\n    ProducerApp --&gt; KafkaApp\n    KafkaSourceConnector --&gt; KafkaConnector\n    KafkaSinkConnector --&gt; KafkaConnector\n    \n    click KubernetesApp \"../kubernetes-app\"\n    click KafkaApp \"../kafka-app\"\n    click StreamsApp \"../streams-app\"\n    click ProducerApp \"../producer-app\"\n    click KafkaConnector \"../kafka-connector\"\n    click KafkaSourceConnector \"../kafka-source-connector\"\n    click KafkaSinkConnector \"../kafka-sink-connector\"</code></pre> <p>KPOps component hierarchy</p>"},{"location":"user/core-concepts/components/producer-app/","title":"ProducerApp","text":"<p>Subclass of KafkaApp.</p>"},{"location":"user/core-concepts/components/producer-app/#usage","title":"Usage","text":"<p>Configures a streams-bootstrap Kafka producer app</p>"},{"location":"user/core-concepts/components/producer-app/#configuration","title":"Configuration","text":"<code>pipeline.yaml</code> <pre><code># Holds configuration to use as values for the streams bootstrap producer-app Helm\n# chart.\n# More documentation on ProducerApp:\n# https://github.com/bakdata/streams-bootstrap\n- type: producer-app\nname: producer-app # required\n# Pipeline prefix that will prefix every component name. If you wish to not\n# have any prefix you can specify an empty string.\nprefix: ${pipeline_name}-\n# from: # While the producer-app does inherit from kafka-app, it does not need a\n# `from` section, hence it does not support it.\n# Topic(s) into which the component will write output\nto:\ntopics:\n${pipeline_name}-output-topic:\ntype: output # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined\n${pipeline_name}-error-topic:\ntype: error\n# Currently KPOps supports Avro and JSON schemas.\nkey_schema: key-schema # must implement SchemaProvider to use\nvalue_schema: value-schema\npartitions_count: 1\nreplication_factor: 1\nconfigs: # https://kafka.apache.org/documentation/#topicconfigs\ncleanup.policy: compact\nmodels: # SchemaProvider is initiated with the values given here\nmodel: model\nnamespace: namespace # required\n# Allowed configs:\n# https://github.com/bakdata/streams-bootstrap/tree/master/charts/producer-app\napp: # required\nstreams: # required, producer-app-specific\nbrokers: ${brokers} # required\nschemaRegistryUrl: ${schema_registry_url}\noutputTopic: output_topic\nextraOutputTopics:\noutput_role1: output_topic1\noutput_role2: output_topic2\nnameOverride: override-with-this-name # kafka-app-specific\n# Helm repository configuration (optional)\n# If not set the helm repo add will not be called. Useful when using local Helm charts\nrepo_config:\nrepository_name: bakdata-streams-bootstrap # required\nurl: https://bakdata.github.io/streams-bootstrap/ # required\nrepo_auth_flags:\nusername: user\npassword: pass\nca_file: /home/user/path/to/ca-file\ninsecure_skip_tls_verify: false\nversion: \"2.12.0\" # Helm chart version\n</code></pre>"},{"location":"user/core-concepts/components/producer-app/#operations","title":"Operations","text":""},{"location":"user/core-concepts/components/producer-app/#deploy","title":"deploy","text":"<p>In addition to KubernetesApp's <code>deploy</code>:</p> <ul> <li>Create topics if provided (optional)</li> <li>Submit Avro schemas to the registry if provided (optional)</li> </ul>"},{"location":"user/core-concepts/components/producer-app/#destroy","title":"destroy","text":"<p>Uninstall Helm release.</p>"},{"location":"user/core-concepts/components/producer-app/#reset","title":"reset","text":"<p>Do nothing, producers are stateless.</p>"},{"location":"user/core-concepts/components/producer-app/#clean","title":"clean","text":"<ul> <li>Delete the output topics of the Kafka producer</li> <li>Delete all associated schemas in the Schema Registry</li> </ul>"},{"location":"user/core-concepts/components/streams-app/","title":"StreamsApp","text":"<p>Subclass of KafkaApp.</p>"},{"location":"user/core-concepts/components/streams-app/#usage","title":"Usage","text":"<p>Configures a streams-bootstrap Kafka Streams app</p>"},{"location":"user/core-concepts/components/streams-app/#configuration","title":"Configuration","text":"<code>pipeline.yaml</code> <pre><code># StreamsApp component that configures a streams bootstrap app.\n# More documentation on StreamsApp: https://github.com/bakdata/streams-bootstrap\n- type: streams-app # required\nname: streams-app # required\n# Pipeline prefix that will prefix every component name. If you wish to not\n# have any prefix you can specify an empty string.\nprefix: ${pipeline_name}-\nfrom: # Must not be null\ntopics: # read from topic\n${pipeline_name}-input-topic:\ntype: input # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra\n${pipeline_name}-input-pattern-topic:\ntype: pattern # Implied to be an input pattern if `role` is undefined\n${pipeline_name}-extra-pattern-topic:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\ncomponents: # read from specific component\naccount-producer:\ntype: output # Implied when role is NOT specified\nother-producer:\nrole: some-role # Implies `type` to be extra\ncomponent-as-input-pattern:\ntype: pattern # Implied to be an input pattern if `role` is undefined\ncomponent-as-extra-pattern:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\n# Topic(s) into which the component will write output\nto:\ntopics:\n${pipeline_name}-output-topic:\ntype: output # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined\n${pipeline_name}-error-topic:\ntype: error\n# Currently KPOps supports Avro and JSON schemas.\nkey_schema: key-schema # must implement SchemaProvider to use\nvalue_schema: value-schema\npartitions_count: 1\nreplication_factor: 1\nconfigs: # https://kafka.apache.org/documentation/#topicconfigs\ncleanup.policy: compact\nmodels: # SchemaProvider is initiated with the values given here\nmodel: model\nnamespace: namespace # required\n# No arbitrary keys are allowed under `app`here\n# Allowed configs:\n# https://github.com/bakdata/streams-bootstrap/tree/master/charts/streams-app\napp: # required\n# Streams Bootstrap streams section\nstreams: # required, streams-app-specific\nbrokers: ${brokers} # required\nschemaRegistryUrl: ${schema_registry_url}\ninputTopics:\n- topic1\n- topic2\noutputTopic: output-topic\ninputPattern: input-pattern\nextraInputTopics:\ninput_role1:\n- input_topic1\n- input_topic2\ninput_role2:\n- input_topic3\n- input_topic4\nextraInputPatterns:\npattern_role1: input_pattern1\nextraOutputTopics:\noutput_role1: output_topic1\noutput_role2: output_topic2\nerrorTopic: error-topic\nconfig:\nmy.streams.config: my.value\nnameOverride: override-with-this-name # streams-app-specific\nautoscaling: # streams-app-specific\nconsumerGroup: consumer-group # required\nlagThreshold: 0 # Average target value to trigger scaling actions.\nenabled: false # Whether to enable auto-scaling using KEDA.\n# This is the interval to check each trigger on.\n# https://keda.sh/docs/2.9/concepts/scaling-deployments/#pollinginterval\npollingInterval: 30\n# The period to wait after the last trigger reported active before scaling\n#  the resource back to 0. https://keda.sh/docs/2.9/concepts/scaling-deployments/#cooldownperiod\ncooldownPeriod: 300\n# The offset reset policy for the consumer if the the consumer group is\n# not yet subscribed to a partition.\noffsetResetPolicy: earliest\n# This setting is passed to the HPA definition that KEDA will create for a\n# given resource and holds the maximum number of replicas of the target resouce.\n# https://keda.sh/docs/2.9/concepts/scaling-deployments/#maxreplicacount\nmaxReplicas: 1\n# Minimum number of replicas KEDA will scale the resource down to.\n# https://keda.sh/docs/2.7/concepts/scaling-deployments/#minreplicacount\nminReplicas: 0\n# If this property is set, KEDA will scale the resource down to this\n# number of replicas.\n# https://keda.sh/docs/2.9/concepts/scaling-deployments/#idlereplicacount\nidleReplicas: 0\ntopics: # List of auto-generated Kafka Streams topics used by the streams app.\n- topic1\n- topic2\n# Helm repository configuration (optional)\n# If not set the helm repo add will not be called. Useful when using local Helm charts\nrepo_config:\nrepository_name: bakdata-streams-bootstrap # required\nurl: https://bakdata.github.io/streams-bootstrap/ # required\nrepo_auth_flags:\nusername: user\npassword: pass\nca_file: /home/user/path/to/ca-file\ninsecure_skip_tls_verify: false\nversion: \"2.12.0\" # Helm chart version\n</code></pre>"},{"location":"user/core-concepts/components/streams-app/#operations","title":"Operations","text":""},{"location":"user/core-concepts/components/streams-app/#deploy","title":"deploy","text":"<p>In addition to KubernetesApp's <code>deploy</code>:</p> <ul> <li>Create topics if provided (optional)</li> <li>Submit Avro schemas to the registry if provided (optional)</li> </ul>"},{"location":"user/core-concepts/components/streams-app/#destroy","title":"destroy","text":"<p>Uninstall Helm release.</p>"},{"location":"user/core-concepts/components/streams-app/#reset","title":"reset","text":"<ul> <li>Delete the consumer group offsets</li> <li>Delete Kafka Streams state</li> </ul>"},{"location":"user/core-concepts/components/streams-app/#clean","title":"clean","text":"<p>Similar to <code>reset</code> with to additional steps:</p> <ul> <li>Delete the app's output topics</li> <li>Delete all associated schemas in the Schema Registry</li> </ul>"},{"location":"user/core-concepts/variables/environment_variables/","title":"Environment variables","text":"<p>Environment variables can be set by using the export command in Linux or the set command in Windows.</p> <p>dotenv files</p> <p>Support for <code>.env</code> files is on the roadmap,  but not implemented in KPOps yet. One of the possible ways to still  use one and export the contents manually is with the following command: <code>export $(xargs &lt; .env)</code>. This would work in <code>bash</code> suppose there are no spaces inside the values.</p>"},{"location":"user/core-concepts/variables/environment_variables/#config","title":"Config","text":"<p>These variables are a lower priority alternative to the settings in <code>config.yaml</code>. Variables marked as required can instead be set in the pipeline config.</p> Name Default Value Required Description Setting name KPOPS_ENVIRONMENT True The environment you want to generate and deploy the pipeline to. Suffix your environment files with this value (e.g. defaults_development.yaml for environment=development). environment KPOPS_KAFKA_BROKERS True The comma separated Kafka brokers address. brokers KPOPS_SCHEMA_REGISTRY_URL False Address of the Schema Registry. schema_registry_url KPOPS_REST_PROXY_HOST False Address of the Kafka REST Proxy. kafka_rest_host KPOPS_CONNECT_HOST False Address of Kafka Connect. kafka_connect_host KPOPS_TIMEOUT 300 False The timeout in seconds that specifies when actions like deletion or deploy timeout. timeout KPOPS_RETAIN_CLEAN_JOBS False False Whether to retain clean up jobs in the cluster or uninstall the, after completion. retain_clean_jobs config_env_vars.env Exhaustive list of all config-related environment variables<pre><code># Pipeline config environment variables\n#\n# The default setup is shown. These variables are a lower priority\n# alternative to the settings in `config.yaml`. Variables marked as\n# required can instead be set in the pipeline config.\n#\n# environment\n# The environment you want to generate and deploy the pipeline to.\n# Suffix your environment files with this value (e.g.\n# defaults_development.yaml for environment=development).\nKPOPS_ENVIRONMENT # No default value, required\n# brokers\n# The comma separated Kafka brokers address.\nKPOPS_KAFKA_BROKERS # No default value, required\n# schema_registry_url\n# Address of the Schema Registry.\nKPOPS_SCHEMA_REGISTRY_URL # No default value, not required\n# kafka_rest_host\n# Address of the Kafka REST Proxy.\nKPOPS_REST_PROXY_HOST # No default value, not required\n# kafka_connect_host\n# Address of Kafka Connect.\nKPOPS_CONNECT_HOST # No default value, not required\n# timeout\n# The timeout in seconds that specifies when actions like deletion or\n# deploy timeout.\nKPOPS_TIMEOUT=300\n# retain_clean_jobs\n# Whether to retain clean up jobs in the cluster or uninstall the,\n# after completion.\nKPOPS_RETAIN_CLEAN_JOBS=False\n</code></pre>"},{"location":"user/core-concepts/variables/environment_variables/#cli","title":"CLI","text":"<p>These variables are a lower priority alternative to the commands' flags. If a variable is set, the corresponding flag does not have to be specified in commands. Variables marked as required can instead be set as flags.</p> Name Default Value Required Description KPOPS_PIPELINE_BASE_DIR . False Base directory to the pipelines (default is current working directory) KPOPS_CONFIG_PATH config.yaml False Path to the config.yaml file KPOPS_DEFAULT_PATH False Path to defaults folder KPOPS_PIPELINE_PATH True Path to YAML with pipeline definition KPOPS_PIPELINE_STEPS False Comma separated list of steps to apply the command on cli_env_vars.env Exhaustive list of all cli-related environment variables<pre><code># CLI Environment variables\n#\n# The default setup is shown. These variables are a lower priority\n# alternative to the commands' flags. If a variable is set, the\n# corresponding flag does not have to be specified in commands.\n# Variables marked as required can instead be set as flags.\n#\n# Base directory to the pipelines (default is current working\n# directory)\nKPOPS_PIPELINE_BASE_DIR=.\n# Path to the config.yaml file\nKPOPS_CONFIG_PATH=config.yaml\n# Path to defaults folder\nKPOPS_DEFAULT_PATH # No default value, not required\n# Path to YAML with pipeline definition\nKPOPS_PIPELINE_PATH # No default value, required\n# Comma separated list of steps to apply the command on\nKPOPS_PIPELINE_STEPS # No default value, not required\n</code></pre>"},{"location":"user/core-concepts/variables/substitution/","title":"Substitution","text":"<p>KPOps supports the usage of placeholders and environment variables in pipeline definition and defaults.</p>"},{"location":"user/core-concepts/variables/substitution/#component-specific-variables","title":"Component-specific variables","text":"<p>These variables can be used in a component's definition to refer to any of its attributes, including ones that the user has defined in the defaults.</p> <p>All of them are prefixed with <code>component_</code> and follow the following form: <code>component_{attribute_name}</code>. If the attribute itself contains attributes, they can be referred to like this: <code>component_{attribute_name}_{subattribute_name}</code>.</p> Example <pre><code>- type: scheduled-producer\napp:\nlabels:\napp_type: \"${component_type}\"\napp_name: \"${component_name}\"\napp_schedule: \"${component_app_schedule}\"\ncommandLine:\nFAKE_ARG: \"fake-arg-value\"\nschedule: \"30 3/8 * * *\"\n- type: converter\napp:\ncommandLine:\nCONVERT_XML: true\nresources:\nlimits:\nmemory: 2G\nrequests:\nmemory: 2G\n- type: filter\nname: \"filter-app\"\napp:\nlabels:\napp_type: \"${component_type}\"\napp_name: \"${component_name}\"\napp_resources_requests_memory: \"${component_app_resources_requests_memory}\"\n${component_type}: \"${component_app_labels_app_name}-${component_app_labels_app_type}\"\ntest_placeholder_in_placeholder: \"${component_app_labels_${component_type}}\"\ncommandLine:\nTYPE: \"nothing\"\nresources:\nrequests:\nmemory: 3G\nreplicaCount: 4\nautoscaling:\nminReplicas: 4\nmaxReplicas: 4\n</code></pre>"},{"location":"user/core-concepts/variables/substitution/#pipeline-config-specific-variables","title":"Pipeline-config-specific variables","text":"<p>These variables include all fields in the config and refer to the pipeline configuration that is independent of the components.</p> <p>Info</p> <p><code>error_topic_name</code> is an alias for <code>topic_name_config_default_error_topic_name</code> <code>output_topic_name</code> is an alias for <code>topic_name_config_default_output_topic_name</code></p>"},{"location":"user/core-concepts/variables/substitution/#environment-variables","title":"Environment variables","text":"<p>Environment variables such as <code>$PATH</code> can be used in the pipeline definition and defaults without any transformation following the form <code>${ENV_VAR_NAME}</code>. This, of course, includes variables like the ones relevant to the KPOps cli that are exported by the user.</p> <p>See all KPOps environment variables</p>"},{"location":"user/core-concepts/variables/substitution/#pipeline-name-variables","title":"Pipeline name variables","text":"<p>These are special variables that refer to the name and path of a pipeline.</p> <ul> <li> <p><code>${pipeline_name}</code>: Concatenated path of the parent directory where pipeline.yaml is defined in.   For instance, <code>./data/pipelines/v1/pipeline.yaml</code>, here the value for the variable would be <code>data-pipelines-v1</code>.</p> </li> <li> <p><code>${pipeline_name_&lt;level&gt;}</code>: Similar to the previous variable, each <code>&lt;level&gt;</code> contains a part of the path to the <code>pipeline.yaml</code> file.   Consider the previous example, <code>${pipeline_name_0}</code> would be <code>data</code>, <code>${pipeline_name_1}</code> would be <code>pipelines</code>, and <code>${pipeline_name_2}</code> equals to <code>v1</code>.</p> </li> </ul>"},{"location":"user/core-concepts/variables/substitution/#advanced-use-cases","title":"Advanced use cases","text":"<ol> <li>Refer to default component field values: As long as a value is assigned to a component attribute, it is possible to refer to it with a placeholder. To see all component fields, take a look at the pipeline schema.</li> <li>Chaining variables: It is possible to chain any number of variables, see the example above.</li> <li>Cross-component substitution: YAML is quite an intricate language and with some of its magic one could write cross-component references.</li> </ol>"},{"location":"user/examples/atm-fraud-pipeline/","title":"ATM fraud detection pipeline","text":"<p>ATM fraud is a demo pipeline for ATM fraud detection. The original by Confluent is written in KSQL and outlined in this blogpost. The one used in this example is re-built from scratch using bakdata's <code>streams-bootstrap</code> library.</p>"},{"location":"user/examples/atm-fraud-pipeline/#what-this-will-demonstrate","title":"What this will demonstrate","text":"<ul> <li>Deploying a PostgreSQL database using Helm</li> <li>Deploying a pipeline using KPOps</li> <li>Destroying a pipeline using KPOps</li> </ul>"},{"location":"user/examples/atm-fraud-pipeline/#prerequisites","title":"Prerequisites","text":"<p>Completed all steps in the setup.</p>"},{"location":"user/examples/atm-fraud-pipeline/#setup-and-deployment","title":"Setup and deployment","text":""},{"location":"user/examples/atm-fraud-pipeline/#postgresql","title":"PostgreSQL","text":"<p>Deploy PostgreSQL using the Bitnami Helm chart: Add the helm repository:</p> <pre><code>helm repo add bitnami https://charts.bitnami.com/bitnami &amp;&amp; \\\nhelm repo update\n</code></pre> <p>Install the PostgreSQL with helm:</p> <pre><code>helm upgrade --install -f ./postgresql.yaml \\\n--namespace kpops \\\npostgresql bitnami/postgresql\n</code></pre> PostgreSQL Example Helm chart values (<code>postgresql.yaml</code>) <pre><code>auth:\ndatabase: app_db  enablePostgresUser: true\npassword: AppPassword\npostgresPassword: StrongPassword\nusername: app1\nprimary:\npersistence:\nenabled: false\nexistingClaim: postgresql-data-claim\nvolumePermissions:\nenabled: true\n</code></pre>"},{"location":"user/examples/atm-fraud-pipeline/#atm-fraud-detection-example-pipeline-setup","title":"ATM fraud detection example pipeline setup","text":""},{"location":"user/examples/atm-fraud-pipeline/#port-forwarding","title":"Port forwarding","text":"<p>Before we deploy the pipeline, we need to forward the ports of <code>kafka-rest-proxy</code> and <code>kafka-connect</code>. Run the following commands in two different terminals.</p> <pre><code>kubectl port-forward --namespace kpops service/k8kafka-cp-rest 8082:8082\n</code></pre> <pre><code>kubectl port-forward --namespace kpops service/k8kafka-cp-kafka-connect 8083:8083\n</code></pre>"},{"location":"user/examples/atm-fraud-pipeline/#deploying-the-atm-fraud-detection-pipeline","title":"Deploying the ATM fraud detection pipeline","text":"<ol> <li>Export environment variables in your terminal:</li> </ol> <pre><code>export DOCKER_REGISTRY=bakdata &amp;&amp; \\\nexport NAMESPACE=kpops\n</code></pre> <ol> <li>Deploy the pipeline</li> </ol> <pre><code>poetry run kpops deploy ./examples/bakdata/atm-fraud-detection/pipeline.yaml \\\n--pipeline-base-dir ./examples \\\n--config ./examples/bakdata/atm-fraud-detection/config.yaml \\\n--execute\n</code></pre> <p>Note</p> <p>You can use the <code>--dry-run</code> flag instead of the <code>--execute</code> flag and check the logs if your pipeline will be deployed correctly.</p>"},{"location":"user/examples/atm-fraud-pipeline/#check-if-the-deployment-is-successful","title":"Check if the deployment is successful","text":"<p>You can use the Streams Explorer to see the deployed pipeline. To do so, port-forward the service in a separate terminal session using the command below:</p> <pre><code>kubectl port-forward -n kpops service/streams-explorer 8080:8080\n</code></pre> <p>After that open http://localhost:8080 in your browser. You should be able to see pipeline shown in the image below:</p> <p> </p> An overview of ATM fraud pipeline shown in Streams Explorer <p>Attention</p> <p>Kafka Connect needs some time to set up the connector.  Moreover, Streams Explorer needs a while to scrape the information from Kafka connect. Therefore, it might take a bit until you see the whole graph.</p>"},{"location":"user/examples/atm-fraud-pipeline/#teardown-resources","title":"Teardown resources","text":""},{"location":"user/examples/atm-fraud-pipeline/#postrgresql","title":"PostrgreSQL","text":"<p>PostgreSQL can be uninstalled by running the following command:</p> <pre><code>helm --namespace kpops uninstall postgresql\n</code></pre>"},{"location":"user/examples/atm-fraud-pipeline/#atm-fraud-pipeline","title":"ATM fraud pipeline","text":"<ol> <li> <p>Export environment variables in your terminal.</p> <pre><code>export DOCKER_REGISTRY=bakdata &amp;&amp; \\\nexport NAMESPACE=kpops\n</code></pre> </li> <li> <p>Remove the pipeline</p> <pre><code>poetry run kpops clean ./examples/bakdata/atm-fraud-detection/pipeline.yaml \\\n--pipeline-base-dir ./examples \\\n--config ./examples/bakdata/atm-fraud-detection/config.yaml \\\n--verbose \\\n--execute\n</code></pre> </li> </ol> <p>Note</p> <p>You can use the <code>--dry-run</code> flag instead of the <code>--execute</code> flag and check the logs if your pipeline will be destroyed correctly.</p> <p>Attention</p> <p>If you face any issues destroying this example see Teardown for manual deletion.</p>"},{"location":"user/examples/atm-fraud-pipeline/#common-errors","title":"Common errors","text":"<ul> <li><code>deploy</code> fails:</li> <li>Read the error message.</li> <li>Try to correct the mistakes if there were any. Likely the configuration is not correct or the port-forwarding is not working as intended.</li> <li>Run <code>clean</code>.</li> <li>Run <code>deploy --dry-run</code> to avoid havig to <code>clean</code> again. If an error is dropped, start over from step 1.</li> <li>If the dry-run is succesful, run <code>deploy</code>.</li> <li><code>clean</code> fails:</li> <li>Read the error message.</li> <li>Try to correct the indicated mistakes if there were any. Likely the configuration is not correct or the port-forwarding is not working as intended.</li> <li>Run <code>clean</code>.</li> <li>If <code>clean</code> fails, follow the steps in teardown.</li> </ul>"},{"location":"user/getting-started/quick-start/","title":"Quick start","text":""},{"location":"user/getting-started/quick-start/#word-count","title":"Word-count","text":"<p>Word-count is a demo pipeline which consists of a producer producing words to Kafka, a Kafka streams app counting the number of times each word occurs and finally a Redis database into which the words are exported.</p>"},{"location":"user/getting-started/quick-start/#what-this-will-demonstrate","title":"What this will demonstrate","text":"<ul> <li>Deploying a Redis database using Helm</li> <li>Deploying a pipeline using KPOps</li> <li>Destroying a pipeline using KPOps</li> </ul>"},{"location":"user/getting-started/quick-start/#prerequisites","title":"Prerequisites","text":"<p>Completed all steps in the setup.</p>"},{"location":"user/getting-started/quick-start/#setup-and-deployment","title":"Setup and deployment","text":""},{"location":"user/getting-started/quick-start/#redis","title":"Redis","text":"<p>Deploy Redis using the Bitnami Helm chart: Add the Helm repository:</p> <pre><code>helm repo add bitnami https://charts.bitnami.com/bitnami &amp;&amp; \\\nhelm repo update\n</code></pre> <p>Install Redis with Helm:</p> <pre><code>helm upgrade --install -f ./values-redis.yaml \\\n--namespace kpops \\\nredis bitnami/redis\n</code></pre> Redis example Helm chart values (<code>values-redis.yaml</code>) <pre><code>architecture: standalone\nauth:\nenabled: false\nmaster:\ncount: 1\nconfiguration: \"databases 1\"\nimage:\ntag: 7.0.8\n</code></pre>"},{"location":"user/getting-started/quick-start/#word-count-example-pipeline-setup","title":"Word-count example pipeline setup","text":""},{"location":"user/getting-started/quick-start/#port-forwarding","title":"Port forwarding","text":"<p>Before we deploy the pipeline, we need to forward the ports of <code>kafka-rest-proxy</code> and <code>kafka-connect</code>. Run the following commands in two different terminals.</p> <pre><code>kubectl port-forward --namespace kpops service/k8kafka-cp-rest 8082:8082\n</code></pre> <pre><code>kubectl port-forward --namespace kpops service/k8kafka-cp-kafka-connect 8083:8083\n</code></pre>"},{"location":"user/getting-started/quick-start/#deploying-the-word-count-pipeline","title":"Deploying the Word-count pipeline","text":"<ol> <li>Copy the configuration from the kpops-examples repository into <code>kpops&gt;examples&gt;bakdata&gt;word-count</code> like so:</li> </ol> <pre><code>kpops\n\u251c\u2500\u2500 examples\n|   \u251c\u2500\u2500 bakdata\n|   |   \u251c\u2500\u2500 word-count\n|   |   |   \u251c\u2500\u2500 config.yaml\n|   |   |   \u251c\u2500\u2500 defaults\n|   |   |   \u2502\u00a0\u00a0 \u2514\u2500\u2500 defaults.yaml\n|   |   |   \u2514\u2500\u2500 pipeline.yaml\n|   |   |\n</code></pre> <ol> <li>Export environment variables in your terminal:</li> </ol> <pre><code>export DOCKER_REGISTRY=bakdata &amp;&amp; \\\nexport NAMESPACE=kpops\n</code></pre> <ol> <li>Deploy the pipeline</li> </ol> <pre><code>kpops deploy ./examples/bakdata/word-count/pipeline.yaml \\\n--pipeline-base-dir ./examples \\\n--config ./examples/bakdata/word-count/config.yaml \\\n--execute\n</code></pre> <p>Note</p> <p>You can use the <code>--dry-run</code> flag instead of the <code>--execute</code> flag and check the logs if your pipeline will be deployed correctly.</p>"},{"location":"user/getting-started/quick-start/#check-if-the-deployment-is-successful","title":"Check if the deployment is successful","text":"<p>You can use the Streams Explorer to inspect the deployed pipeline. To do so, port-forward the service in a separate terminal session using the command below:</p> <pre><code>kubectl port-forward -n kpops service/streams-explorer 8080:8080\n</code></pre> <p>After that open http://localhost:8080 in your browser.</p> <p>You should be able to see pipeline shown in the image below:</p> <p> </p> An overview of Word-count pipeline shown in Streams Explorer <p>Attention</p> <p>Kafka Connect needs some time to set up the connector. Moreover, Streams Explorer needs a while to scrape the information from Kafka Connect. Therefore, it might take a bit until you see the whole graph.</p>"},{"location":"user/getting-started/quick-start/#teardown-resources","title":"Teardown resources","text":""},{"location":"user/getting-started/quick-start/#redis_1","title":"Redis","text":"<p>Redis can be uninstalled by running the following command:</p> <pre><code>helm --namespace kpops uninstall redis\n</code></pre>"},{"location":"user/getting-started/quick-start/#word-count-pipeline","title":"Word-count pipeline","text":"<ol> <li>Export environment variables in your terminal.</li> </ol> <pre><code>export DOCKER_REGISTRY=bakdata &amp;&amp; \\\nexport NAMESPACE=kpops\n</code></pre> <ol> <li>Remove the pipeline</li> </ol> <pre><code>kpops clean ./examples/bakdata/word-count/pipeline.yaml \\\n--pipeline-base-dir ./examples \\\n--config ./examples/bakdata/word-count/config.yaml \\\n--verbose \\\n--execute\n</code></pre> <p>Note</p> <p>You can use the <code>--dry-run</code> flag instead of the <code>--execute</code> flag and check the logs if your pipeline will be destroyed correctly.</p> <p>Attention</p> <p>If you face any issues destroying this example see Teardown for manual deletion.</p>"},{"location":"user/getting-started/quick-start/#common-errors","title":"Common errors","text":"<ul> <li><code>deploy</code> fails:</li> <li>Read the error message.</li> <li>Try to correct the mistakes if there were any. Likely the configuration is not correct or the port-forwarding is not working as intended.</li> <li>Run <code>clean</code>.</li> <li>Run <code>deploy --dry-run</code> to avoid having to <code>clean</code> again. If an error is dropped, start over from step 1.</li> <li>If the dry-run is successful, run <code>deploy</code>.</li> <li><code>clean</code> fails:</li> <li>Read the error message.</li> <li>Try to correct the indicated mistakes if there were any. Likely the configuration is not correct or the port-forwarding is not working as intended.</li> <li>Run <code>clean</code>.</li> <li>If <code>clean</code> fails, follow the steps in teardown.</li> </ul>"},{"location":"user/getting-started/setup/","title":"Setup KPOps","text":"<p>In this part, you will set up KPOps. This includes:</p> <ul> <li>optionally creating a local Kubernetes cluster</li> <li>running Apache Kafka and Confluent's Schema Registry</li> <li>installing KPOps</li> </ul>"},{"location":"user/getting-started/setup/#prerequisites","title":"Prerequisites","text":"<ul> <li>k3d (Version 5.4.6+) and Docker (Version &gt;= v20.10.5) or an existing Kubernetes cluster (&gt;= 1.21.0)</li> <li>kubectl (Compatible with server version 1.21.0)</li> <li>Helm (Version 3.8.0+)</li> </ul>"},{"location":"user/getting-started/setup/#setup-kubernetes-with-k3d","title":"Setup Kubernetes with k3d","text":"<p>If you don't have access to an existing Kubernetes cluster, this section will guide you through creating a local cluster. We recommend the lightweight Kubernetes distribution k3s for this. k3d is a wrapper around k3s in Docker that lets you get started fast.</p> <ol> <li>You can install k3d with its installation script:</li> </ol> <pre><code>wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/v5.4.6/install.sh | bash\n</code></pre> <p>For other ways of installing k3d, you can have a look at their installation guide.</p> <ol> <li>The Kafka deployment needs a modified Docker image. In that case the image is built and pushed to a Docker registry that holds it. If you do not have access to an existing Docker registry, you can use k3d's Docker registry:</li> </ol> <pre><code>k3d registry create kpops-registry.localhost --port 12345\n</code></pre> <ol> <li>Now you can create a new cluster called <code>kpops</code> that uses the previously created Docker registry:</li> </ol> <pre><code>k3d cluster create kpops --k3s-arg \"--no-deploy=traefik@server:*\" --registry-use k3d-kpops-registry.localhost:12345\n</code></pre> <p>Note</p> <p>Creating a new k3d cluster automatically configures <code>kubectl</code> to connect to the local cluster by modifying your <code>~/.kube/config</code>. In case you manually set the <code>KUBECONFIG</code> variable or don't want k3d to modify your config, k3d offers many other options.</p> <p>You can check the cluster status with <code>kubectl get pods -n kube-system</code>. If all returned elements have a <code>STATUS</code> of <code>Running</code> or <code>Completed</code>, then the cluster is up and running.</p>"},{"location":"user/getting-started/setup/#deploy-kafka","title":"Deploy Kafka","text":"<p>Kafka is an open-source data streaming platform. More information about Kafka can be found in the documentation. To deploy Kafka, this guide uses Confluent's Helm chart.</p> <ol> <li> <p>To allow connectivity to other systems Kafka Connect needs to be extended with drivers. You can install a JDBC driver for Kafka Connect by creating a new Docker image:</p> </li> <li> <p>Create a <code>Dockerfile</code> with the following content:</p> <pre><code>FROM confluentinc/cp-kafka-connect:7.1.3\nRUN confluent-hub install --no-prompt confluentinc/kafka-connect-jdbc:10.6.0\n</code></pre> </li> <li> <p>Build and push the modified image to your private Docker registry:</p> <pre><code>docker build . --tag localhost:12345/kafka-connect-jdbc:7.1.3 &amp;&amp; \\\ndocker push localhost:12345/kafka-connect-jdbc:7.1.3\n</code></pre> </li> </ol> <p>Detailed instructions on building, tagging and pushing a docker image can be found in Docker docs.</p> <ol> <li>Add Confluent's Helm chart repository and update the index:</li> </ol> <pre><code>helm repo add confluentinc https://confluentinc.github.io/cp-helm-charts/ &amp;&amp;  helm repo update\n</code></pre> <ol> <li>Install Kafka, Zookeeper, Confluent's Schema Registry, Kafka Rest Proxy, and Kafka Connect. A single Helm chart installs all five components. Below you can find an example for the <code>--values ./kafka.yaml</code> file configuring the deployment accordingly. Deploy the services:</li> </ol> <pre><code>helm upgrade \\\n--install \\\n--version 0.6.1 \\\n--values ./kafka.yaml \\\n--namespace kpops \\\n--create-namespace \\\n--wait \\\nk8kafka confluentinc/cp-helm-charts\n</code></pre> Kafka Helm chart values (<code>kafka.yaml</code>) <p>An example value configuration for Confluent's Helm chart. This configuration deploys a single Kafka Broker, a Schema Registry, Zookeeper, Kafka Rest Proxy, and Kafka Connect with minimal resources.</p> <pre><code>cp-zookeeper:\nenabled: true\nservers: 1\nimageTag: 7.1.3\nheapOptions: \"-Xms124M -Xmx124M\"\noverrideGroupId: k8kafka\nfullnameOverride: \"k8kafka-cp-zookeeper\"\nresources:\nrequests:\ncpu: 50m\nmemory: 0.2G\nlimits:\ncpu: 250m\nmemory: 0.2G\nprometheus:\njmx:\nenabled: false\ncp-kafka:\nenabled: true\nbrokers: 1\nimageTag: 7.1.3\npodManagementPolicy: Parallel\nconfigurationOverrides:\n\"auto.create.topics.enable\": false\n\"offsets.topic.replication.factor\": 1\n\"transaction.state.log.replication.factor\": 1\n\"transaction.state.log.min.isr\": 1\n\"confluent.metrics.reporter.topic.replicas\": 1\nresources:\nrequests:\ncpu: 50m\nmemory: 0.5G\nlimits:\ncpu: 250m\nmemory: 0.5G\nprometheus:\njmx:\nenabled: false\npersistence:\nenabled: false\ncp-schema-registry:\nenabled: true\nimageTag: 7.1.3\nfullnameOverride: \"k8kafka-cp-schema-registry\"\noverrideGroupId: k8kafka\nkafka:\nbootstrapServers: \"PLAINTEXT://k8kafka-cp-kafka-headless:9092\"\nresources:\nrequests:\ncpu: 50m\nmemory: 0.25G\nlimits:\ncpu: 250m\nmemory: 0.25G\nprometheus:\njmx:\nenabled: false\ncp-kafka-connect:\nenabled: true\nreplicaCount: 1\nimage: k3d-kpops-registry.localhost:12345/kafka-connect-jdbc\nimageTag: 7.1.3\nfullnameOverride: \"k8kafka-cp-kafka-connect\"\noverrideGroupId: k8kafka\nkafka:\nbootstrapServers: \"PLAINTEXT://k8kafka-cp-kafka-headless:9092\"\nheapOptions: \"-Xms256M -Xmx256M\"\nresources:\nrequests:\ncpu: 500m\nmemory: 0.25G\nlimits:\ncpu: 500m\nmemory: 0.25G\nconfigurationOverrides:\n\"consumer.max.poll.records\": \"10\"\n\"consumer.max.poll.interval.ms\": \"900000\"\n\"config.storage.replication.factor\": \"1\"\n\"offset.storage.replication.factor\": \"1\"\n\"status.storage.replication.factor\": \"1\"\ncp-schema-registry:\nurl: http://k8kafka-cp-schema-registry:8081\nprometheus:\njmx:\nenabled: false\ncp-kafka-rest:\nenabled: true\nimageTag: 7.1.3\nfullnameOverride: \"k8kafka-cp-rest\"\nheapOptions: \"-Xms256M -Xmx256M\"\nresources:\nrequests:\ncpu: 50m\nmemory: 0.25G\nlimits:\ncpu: 250m\nmemory: 0.5G\nprometheus:\njmx:\nenabled: false\ncp-ksql-server:\nenabled: false\ncp-control-center:\nenabled: false\n</code></pre>"},{"location":"user/getting-started/setup/#deploy-streams-explorer","title":"Deploy Streams Explorer","text":"<p>Streams Explorer allows examining Apache Kafka data pipelines in a Kubernetes cluster including the inspection of schemas and monitoring of metrics. First, add the Helm repository:</p> <pre><code>helm repo add streams-explorer https://bakdata.github.io/streams-explorer &amp;&amp; \\\nhelm repo update\n</code></pre> <p>Below you can find an example for the <code>--values ./streams-explorer.yaml</code> file configuring the deployment accordingly. Now, deploy the service:</p> <pre><code>helm upgrade \\\n--install \\\n--version 0.2.3 \\\n--values ./streams-explorer.yaml \\\n--namespace kpops \\\nstreams-explorer streams-explorer/streams-explorer\n</code></pre> Streams Explorer Helm chart values (<code>streams-explorer.yaml</code>) <p>An example value configuration for Steams Explorer Helm chart.</p> <pre><code>imageTag: \"v2.1.2\"\nconfig:\nK8S__deployment__cluster: true\nSCHEMAREGISTRY__url: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081\nKAFKACONNECT__url: http://k8kafka-cp-kafka-connect.kpops.svc.cluster.local:8083\nresources:\nrequests:\ncpu: 200m\nmemory: 300Mi\nlimits:\ncpu: 200m\nmemory: 300Mi\n</code></pre>"},{"location":"user/getting-started/setup/#check-the-status-of-your-deployments","title":"Check the status of your deployments","text":"<p>Now we will check if all the pods are running in our namespace. You can list all pods in the namespace with this command:</p> <pre><code>kubectl --namespace kpops get pods\n</code></pre> <p>Then you should see the following output in your terminal:</p> <pre><code>NAME                                          READY   STATUS    RESTARTS   AGE\nk8kafka-cp-kafka-connect-8fc7d544f-8pjnt      1/1     Running   0          15m\nk8kafka-cp-zookeeper-0                        1/1     Running   0          15m\nk8kafka-cp-kafka-0                            1/1     Running   0          15m\nk8kafka-cp-schema-registry-588f8c65db-jdwbq   1/1     Running   0          15m\nk8kafka-cp-rest-6bbfd7b645-nwkf8              1/1     Running   0          15m\nstreams-explorer-54db878c67-s8wbz             1/1     Running   0          15m\n</code></pre> <p>Pay attention to the <code>STATUS</code> row. The pods should have a status of <code>Running</code>.</p>"},{"location":"user/getting-started/setup/#install-kpops","title":"Install KPOps","text":"<p>KPOps comes as a PyPI package. You can install it with <code>pip</code>:</p> <pre><code>pip install kpops\n</code></pre>"},{"location":"user/getting-started/teardown/","title":"Teardown resources","text":""},{"location":"user/getting-started/teardown/#kpops-teardown-commands","title":"KPOps teardown commands","text":"<ul> <li><code>destroy</code>: Removes Kubernetes resources.</li> <li><code>reset</code>: Runs <code>destroy</code>, resets the states of Kafka Streams apps and resets offsets to zero.</li> <li><code>clean</code>: Runs <code>reset</code> and removes all Kafka resources.</li> </ul>"},{"location":"user/getting-started/teardown/#kpops-deployed-pipeline","title":"KPOps-deployed pipeline","text":"<p>The <code>kpops</code> CLI can be used to destroy a pipeline that was previously deployed with KPOps. In case that doesn't work, the pipeline can always be taken down manually with <code>helm</code> (see section Infrastructure).</p> <ol> <li>Export environment variables.</li> </ol> <pre><code>export DOCKER_REGISTRY=bakdata &amp;&amp; \\\nexport NAMESPACE=kpops\n</code></pre> <ol> <li> <p>Navigate to the <code>examples</code> folder.    Replace the <code>&lt;name-of-the-example-directory&gt;</code> with the example you want to tear down.    For example the <code>atm-fraud-detection</code>.</p> </li> <li> <p>Remove the pipeline</p> </li> </ol> <pre><code># Uncomment 1 line to either destroy, reset or clean.\n# poetry run kpops destroy &lt;name-of-the-example-directory&gt;/pipeline.yaml \\\n# poetry run kpops reset &lt;name-of-the-example-directory&gt;/pipeline.yaml \\\n# poetry run kpops clean &lt;name-of-the-example-directory&gt;/pipeline.yaml \\\n--config &lt;name-of-the-example-directory&gt;/config.yaml \\\n--execute\n</code></pre>"},{"location":"user/getting-started/teardown/#infrastructure","title":"Infrastructure","text":"<p>Delete namespace:</p> <pre><code>kubectl delete namespace kpops\n</code></pre> <p>Note</p> <p>In case <code>kpops destroy</code> is not working one can uninstall the pipeline services one by one. This is equivalent to running <code>kpops destroy</code>. In case a clean uninstall (like the one <code>kpops clean</code> does)  is needed, one needs to also delete the topics and schemas created by deployment of the pipeline.</p>"},{"location":"user/getting-started/teardown/#local-cluster","title":"Local cluster","text":"<p>Delete local cluster:</p> <pre><code>k3d cluster delete kpops\n</code></pre>"},{"location":"user/getting-started/teardown/#local-image-registry","title":"Local image registry","text":"<p>Delete local registry:</p> <pre><code>k3d registry delete k3d-kpops-registry.localhost\n</code></pre>"},{"location":"user/migration-guide/v1-v2/","title":"Migrate from V1 to V2","text":""},{"location":"user/migration-guide/v1-v2/#derive-component-type-automatically-from-class-name","title":"Derive component type automatically from class name","text":"<p>KPOps automatically infers the component <code>type</code> from the class name. Therefore, the <code>type</code> and <code>schema_type</code> attributes can be removed from your custom components. By convention the <code>type</code> would be the lower, and kebab cased name of the class.</p> <pre><code>class MyCoolStreamApp(StreamsApp):\n-    type = \"my-cool-stream-app\"\n+    ...\n</code></pre> <p>Because of this new convention <code>producer</code> has been renamed to <code>producer-app</code>. This must be addressed in your <code>pipeline.yaml</code> and <code>defaults.yaml</code>.</p> <pre><code>- producer:\n+ producer-app:\n   app:\n        streams:\n        outputTopic: output_topic\n        extraOutputTopics: \n            output_role1: output_topic1\n            output_role2: output_topic2\n</code></pre>"},{"location":"user/migration-guide/v1-v2/#refactor-inputoutput-types","title":"Refactor input/output types","text":""},{"location":"user/migration-guide/v1-v2/#to-section","title":"To section","text":"<p>In the <code>to</code> section these have changed:</p> <ul> <li>The default type is <code>output</code></li> <li>If <code>role</code> is set, type is inferred to be <code>extra</code></li> <li>The type <code>error</code> needs to be defined explicitly</li> </ul> <pre><code>  to:\n    topics:\n      ${pipeline_name}-topic-1:\n-       type: extra\n       role: \"role-1\"\n        ...\n      ${pipeline_name}-topic-2:\n-       type: output \n       ...\n      ${pipeline_name}-topic-3:\n         type: error \n         ...\n</code></pre>"},{"location":"user/migration-guide/v1-v2/#from-section","title":"From section","text":"<p>In the <code>from</code> section these have changed:</p> <ul> <li>The default type is <code>input</code></li> <li><code>input-pattern</code> type is replaced by <code>pattern</code></li> <li>If <code>role</code> is set, type is inferred to be <code>extra</code></li> <li>If <code>role</code> is set, type is explicitly set to <code>pattern</code>, this would be inferred type <code>extra-pattern</code></li> </ul> <pre><code>  from:\n    topics:\n      ${pipeline_name}-input-topic:\n-       type: input\n       ...\n      ${pipeline_name}-extra-topic:\n-       type: extra\n       role: topic-role\n        ...\n      ${pipeline_name}-input-pattern-topic:\n-       type: input-pattern \n+       type: pattern\n       ...\n      ${pipeline_name}-extra-pattern-topic:\n-       type: extra-pattern \n+       type: pattern\n       role: some-role\n        ...\n</code></pre>"},{"location":"user/migration-guide/v1-v2/#remove-camel-case-conversion-of-internal-models","title":"Remove camel case conversion of internal models","text":"<p>All the internal KPOps models are now snake_case, and only Helm/Kubernetes values require camel casing. You can find an example of a <code>pipeline.yaml</code> in the following. Notice that the <code>app</code> section here remains untouched.</p> <pre><code>...\ntype: streams-app\n name: streams-app\n  namespace: namespace \n  app: \n    streams:\n      brokers: ${brokers} \n      schemaRegistryUrl: ${schema_registry_url}\n     autoscaling:\n      consumerGroup: consumer-group\n      lagThreshold: 0\n      enabled: false \n      pollingInterval: 30\n\n to:\n    topics:\n      ${pipeline_name}-output-topic:\n        type: error\n-       keySchema: key-schema\n+       key_schema: key-schema\n-       valueSchema: value-schema\n+       value_schema: value-schema\n       partitions_count: 1\n        replication_factor: 1\n        configs:\n          cleanup.policy: compact\n    models:\n      model: model\n  prefix: ${pipeline_name}-\n- repoConfig:\n+ repo_config:\n-   repositoryName: bakdata-streams-bootstrap\n+   repository_name: bakdata-streams-bootstrap\n   url: https://bakdata.github.io/streams-bootstrap/\n-   repoAuthFlags: \n+   repo_auth_flags: \n     username: user\n      password: pass\n      ca_file: /home/user/path/to/ca-file\n      insecure_skip_tls_verify: false\n  version: \"1.0.4\"\n...\n</code></pre>"},{"location":"user/migration-guide/v1-v2/#refactor-handling-of-helm-flags","title":"Refactor handling of Helm flags","text":"<p>If you are using the <code>KubernetesApp</code> class to define your own Kubernetes resource to deploy, the abstract function <code>get_helm_chart</code> that returns the chart for deploying the app using Helm is now a Python property and renamed to <code>helm_chart</code>.</p> <pre><code>class MyCoolApp(KubernetesApp):\n+   @property\n   @override\n-   def get_helm_chart(self) -&gt; str:\n+   def helm_chart(self) -&gt; str:\n       return \"./charts/charts-folder\"\n</code></pre>"},{"location":"user/migration-guide/v1-v2/#plural-broker-field-in-pipeline-config","title":"Plural broker field in pipeline config","text":"<p>Since you can pass a comma separated string of broker address, the broker field in KPOps is now plural. The pluralization has affected multiple areas:</p>"},{"location":"user/migration-guide/v1-v2/#configyaml","title":"config.yaml","text":"<pre><code>  environment: development\n- broker: \"http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092\"\n+ brokers: \"http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092\"\nkafka_connect_host: \"http://localhost:8083\"\n kafka_rest_host: \"http://localhost:8082\"\n schema_registry_url: \"http://localhost:8081\"\n</code></pre>"},{"location":"user/migration-guide/v1-v2/#pipelineyaml-and-defaultyaml","title":"pipeline.yaml and default.yaml","text":"<p>The variable is now called <code>brokers</code>.</p> <pre><code>...\n app:\n    streams:\n-     brokers: ${broker}\n+     brokers: ${brokers}\n     schemaRegistryUrl: ${schema_registry_url}\n    nameOverride: override-with-this-name\n    imageTag: \"1.0.0\"\n...\n</code></pre>"},{"location":"user/migration-guide/v1-v2/#environment-variable","title":"Environment variable","text":"<p>Previously, if you set the environment variable <code>KPOPS_KAFKA_BROKER</code>, you need to replace that now with <code>KPOPS_KAFKA_BROKERS</code>.</p>"},{"location":"user/references/cli-commands/","title":"CLI Usage","text":"<p>Usage:</p> <pre><code>$ kpops [OPTIONS] COMMAND [ARGS]...\n</code></pre> <p>Options:</p> <ul> <li><code>-V, --version</code>: Print KPOps version</li> <li><code>--install-completion</code>: Install completion for the current shell.</li> <li><code>--show-completion</code>: Show completion for the current shell, to copy it or customize the installation.</li> <li><code>--help</code>: Show this message and exit.</li> </ul> <p>Commands:</p> <ul> <li><code>clean</code>: Clean pipeline steps</li> <li><code>deploy</code>: Deploy pipeline steps</li> <li><code>destroy</code>: Destroy pipeline steps</li> <li><code>generate</code>: Enriches pipelines steps with defaults.</li> <li><code>reset</code>: Reset pipeline steps</li> <li><code>schema</code>: Generate json schema.</li> </ul>"},{"location":"user/references/cli-commands/#kpops-clean","title":"<code>kpops clean</code>","text":"<p>Clean pipeline steps</p> <p>Usage:</p> <pre><code>$ kpops clean [OPTIONS] PIPELINE_PATH [COMPONENTS_MODULE]\n</code></pre> <p>Arguments:</p> <ul> <li><code>PIPELINE_PATH</code>: Path to YAML with pipeline definition  [env var: KPOPS_PIPELINE_PATH;required]</li> <li><code>[COMPONENTS_MODULE]</code>: Custom Python module containing your project-specific components</li> </ul> <p>Options:</p> <ul> <li><code>--pipeline-base-dir DIRECTORY</code>: Base directory to the pipelines (default is current working directory)  [env var: KPOPS_PIPELINE_BASE_DIR; default: .]</li> <li><code>--defaults DIRECTORY</code>: Path to defaults folder  [env var: KPOPS_DEFAULT_PATH]</li> <li><code>--config FILE</code>: Path to the config.yaml file  [env var: KPOPS_CONFIG_PATH; default: config.yaml]</li> <li><code>--steps TEXT</code>: Comma separated list of steps to apply the command on  [env var: KPOPS_PIPELINE_STEPS]</li> <li><code>--filter-type [include|exclude]</code>: Whether the --steps option should include/exclude the steps  [default: include]</li> <li><code>--dry-run / --execute</code>: Whether to dry run the command or execute it  [default: dry-run]</li> <li><code>--verbose / --no-verbose</code>: Enable verbose printing  [default: no-verbose]</li> <li><code>--help</code>: Show this message and exit.</li> </ul>"},{"location":"user/references/cli-commands/#kpops-deploy","title":"<code>kpops deploy</code>","text":"<p>Deploy pipeline steps</p> <p>Usage:</p> <pre><code>$ kpops deploy [OPTIONS] PIPELINE_PATH [COMPONENTS_MODULE]\n</code></pre> <p>Arguments:</p> <ul> <li><code>PIPELINE_PATH</code>: Path to YAML with pipeline definition  [env var: KPOPS_PIPELINE_PATH;required]</li> <li><code>[COMPONENTS_MODULE]</code>: Custom Python module containing your project-specific components</li> </ul> <p>Options:</p> <ul> <li><code>--pipeline-base-dir DIRECTORY</code>: Base directory to the pipelines (default is current working directory)  [env var: KPOPS_PIPELINE_BASE_DIR; default: .]</li> <li><code>--defaults DIRECTORY</code>: Path to defaults folder  [env var: KPOPS_DEFAULT_PATH]</li> <li><code>--config FILE</code>: Path to the config.yaml file  [env var: KPOPS_CONFIG_PATH; default: config.yaml]</li> <li><code>--steps TEXT</code>: Comma separated list of steps to apply the command on  [env var: KPOPS_PIPELINE_STEPS]</li> <li><code>--filter-type [include|exclude]</code>: Whether the --steps option should include/exclude the steps  [default: include]</li> <li><code>--dry-run / --execute</code>: Whether to dry run the command or execute it  [default: dry-run]</li> <li><code>--verbose / --no-verbose</code>: Enable verbose printing  [default: no-verbose]</li> <li><code>--help</code>: Show this message and exit.</li> </ul>"},{"location":"user/references/cli-commands/#kpops-destroy","title":"<code>kpops destroy</code>","text":"<p>Destroy pipeline steps</p> <p>Usage:</p> <pre><code>$ kpops destroy [OPTIONS] PIPELINE_PATH [COMPONENTS_MODULE]\n</code></pre> <p>Arguments:</p> <ul> <li><code>PIPELINE_PATH</code>: Path to YAML with pipeline definition  [env var: KPOPS_PIPELINE_PATH;required]</li> <li><code>[COMPONENTS_MODULE]</code>: Custom Python module containing your project-specific components</li> </ul> <p>Options:</p> <ul> <li><code>--pipeline-base-dir DIRECTORY</code>: Base directory to the pipelines (default is current working directory)  [env var: KPOPS_PIPELINE_BASE_DIR; default: .]</li> <li><code>--defaults DIRECTORY</code>: Path to defaults folder  [env var: KPOPS_DEFAULT_PATH]</li> <li><code>--config FILE</code>: Path to the config.yaml file  [env var: KPOPS_CONFIG_PATH; default: config.yaml]</li> <li><code>--steps TEXT</code>: Comma separated list of steps to apply the command on  [env var: KPOPS_PIPELINE_STEPS]</li> <li><code>--filter-type [include|exclude]</code>: Whether the --steps option should include/exclude the steps  [default: include]</li> <li><code>--dry-run / --execute</code>: Whether to dry run the command or execute it  [default: dry-run]</li> <li><code>--verbose / --no-verbose</code>: Enable verbose printing  [default: no-verbose]</li> <li><code>--help</code>: Show this message and exit.</li> </ul>"},{"location":"user/references/cli-commands/#kpops-generate","title":"<code>kpops generate</code>","text":"<p>Enriches pipelines steps with defaults. The output is used as input for the deploy/destroy/... commands.</p> <p>Usage:</p> <pre><code>$ kpops generate [OPTIONS] PIPELINE_PATH [COMPONENTS_MODULE]\n</code></pre> <p>Arguments:</p> <ul> <li><code>PIPELINE_PATH</code>: Path to YAML with pipeline definition  [env var: KPOPS_PIPELINE_PATH;required]</li> <li><code>[COMPONENTS_MODULE]</code>: Custom Python module containing your project-specific components</li> </ul> <p>Options:</p> <ul> <li><code>--pipeline-base-dir DIRECTORY</code>: Base directory to the pipelines (default is current working directory)  [env var: KPOPS_PIPELINE_BASE_DIR; default: .]</li> <li><code>--defaults DIRECTORY</code>: Path to defaults folder  [env var: KPOPS_DEFAULT_PATH]</li> <li><code>--config FILE</code>: Path to the config.yaml file  [env var: KPOPS_CONFIG_PATH; default: config.yaml]</li> <li><code>--template / --no-template</code>: Run Helm template  [default: no-template]</li> <li><code>--steps TEXT</code>: Comma separated list of steps to apply the command on  [env var: KPOPS_PIPELINE_STEPS]</li> <li><code>--filter-type [include|exclude]</code>: Whether the --steps option should include/exclude the steps  [default: include]</li> <li><code>--verbose / --no-verbose</code>: Enable verbose printing  [default: no-verbose]</li> <li><code>--help</code>: Show this message and exit.</li> </ul>"},{"location":"user/references/cli-commands/#kpops-reset","title":"<code>kpops reset</code>","text":"<p>Reset pipeline steps</p> <p>Usage:</p> <pre><code>$ kpops reset [OPTIONS] PIPELINE_PATH [COMPONENTS_MODULE]\n</code></pre> <p>Arguments:</p> <ul> <li><code>PIPELINE_PATH</code>: Path to YAML with pipeline definition  [env var: KPOPS_PIPELINE_PATH;required]</li> <li><code>[COMPONENTS_MODULE]</code>: Custom Python module containing your project-specific components</li> </ul> <p>Options:</p> <ul> <li><code>--pipeline-base-dir DIRECTORY</code>: Base directory to the pipelines (default is current working directory)  [env var: KPOPS_PIPELINE_BASE_DIR; default: .]</li> <li><code>--defaults DIRECTORY</code>: Path to defaults folder  [env var: KPOPS_DEFAULT_PATH]</li> <li><code>--config FILE</code>: Path to the config.yaml file  [env var: KPOPS_CONFIG_PATH; default: config.yaml]</li> <li><code>--steps TEXT</code>: Comma separated list of steps to apply the command on  [env var: KPOPS_PIPELINE_STEPS]</li> <li><code>--filter-type [include|exclude]</code>: Whether the --steps option should include/exclude the steps  [default: include]</li> <li><code>--dry-run / --execute</code>: Whether to dry run the command or execute it  [default: dry-run]</li> <li><code>--verbose / --no-verbose</code>: Enable verbose printing  [default: no-verbose]</li> <li><code>--help</code>: Show this message and exit.</li> </ul>"},{"location":"user/references/cli-commands/#kpops-schema","title":"<code>kpops schema</code>","text":"<p>Generate json schema.</p> <p>The schemas can be used to enable support for kpops files in a text editor.</p> <p>Usage:</p> <pre><code>$ kpops schema [OPTIONS] SCOPE:{pipeline|config} [COMPONENTS_MODULE]\n</code></pre> <p>Arguments:</p> <ul> <li><code>SCOPE:{pipeline|config}</code>:          Scope of the generated schema<pre><code>pipeline: Schema of PipelineComponents. Includes the built-in kpops components by default. To include custom components, provide [COMPONENTS_MODULES].\n\nconfig: Schema of PipelineConfig.  [required]\n</code></pre> <ul> <li><code>[COMPONENTS_MODULE]</code>: Custom Python module containing your project-specific components</li> </ul> </li> </ul> <p>Options:</p> <ul> <li><code>--include-stock-components / --no-include-stock-components</code>: Include the built-in KPOps components.  [default: include-stock-components]</li> <li><code>--help</code>: Show this message and exit.</li> </ul>"},{"location":"user/references/editor-integration/","title":"Editor integration","text":"<p>KPOps provides JSON schemas that enable autocompletion and validation for some of the files that the user must work with.</p>"},{"location":"user/references/editor-integration/#supported-files","title":"Supported files","text":"<ul> <li><code>pipeline.yaml</code></li> <li><code>config.yaml</code></li> </ul>"},{"location":"user/references/editor-integration/#usage","title":"Usage","text":"<ol> <li>Install the yaml-language-server in your editor of choice. (requires LSP support)</li> <li>Configure the extension with the settings below.</li> </ol> <code>settings.json</code> <pre><code>{\n\"yaml.schemas\": {\n\"https://bakdata.github.io/kpops/2.0/schema/config.json\": \"config.yaml\",\n\"https://bakdata.github.io/kpops/2.0/schema/pipeline.json\": \"pipeline.yaml\"\n}\n}\n</code></pre> <p>Advanced usage</p> <p>It is possible to generate schemas with the <code>kpops schema</code> command. Useful when using custom components or when using a pre-release version of KPOps.</p>"},{"location":"user/references/ci-integration/github-actions/","title":"GitHub Actions integration","text":"<p>We provided a GitHub composite action called kpops-runner that installs all the necessary dependencies and runs KPOps commands with the given parameters.</p>"},{"location":"user/references/ci-integration/github-actions/#input-parameters","title":"Input Parameters","text":"Name Required Default Value Type Description command \u2705 - string KPOps command to run. generate, deploy, destroy, reset, clean are possible values. Flags such as --dry-run and --execute need to be specified pipeline \u2705 - string Pipeline to run by KPOps working-directory \u274c . string root directory used by KPOps to run pipelines pipeline-base-dir \u274c - string directory where relative pipeline variables are initialized from defaults \u274c - string defaults folder path config \u274c - string config.yaml file path components \u274c - string components package path filter-type \u274c - string Whether to include/exclude the steps defined in KPOPS_PIPELINE_STEPS python-version \u274c \"3.11.x\" string Python version to install (Defaults to the latest stable version of Python 3.11) kpops-version \u274c latest string KPOps version to install helm-version \u274c latest string Helm version to install token \u274c latest string secrets.GITHUB_TOKEN, needed for setup-helm action if helm-version is set to latest"},{"location":"user/references/ci-integration/github-actions/#usage","title":"Usage","text":"<pre><code>steps:\n# ...\n# This step is useful for debugging reasons\n- name: Generate Kafka pipeline\nuses: bakdata/kpops/actions/kpops-runner@main\nwith:\ncommand: generate\nworking-directory: home/my-kpops-root-dir\npipeline: pipelines/my-pipeline-file.yaml\nkpops-version: 1.2.3\n- name: Deploy Kafka pipeline\nuses: bakdata/kpops/actions/kpops-runner@main\nwith:\ncommand: deploy --execute\nworking-directory: home/my-kpops-root-dir\npipeline: pipelines/my-pipeline-file.yaml\nkpops-version: 1.2.3\n# ...\n</code></pre> <p>It is possible to execute the KPOps runner on a dev version from the test.pypi.</p> <pre><code>steps:\n- name: Deploy Kafka pipeline\nuses: bakdata/kpops/actions/kpops-runner@main\nwith:\ncommand: deploy --execute\nworking-directory: home/my-kpops-root-dir\npipeline: pipelines/my-pipeline-file.yaml\nkpops-version: 1.2.5.dev20230707132709 -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/\n</code></pre>"}]}
\ No newline at end of file
+{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"developer/auto-generation/","title":"Auto generation","text":"<p>Auto generation happens mostly with <code>pre-commit</code> hooks. You can find the pre-commit configuration here. These pre-commit hooks call different Python scripts to auto generate code for the documentation.</p>"},{"location":"developer/auto-generation/#generation-scripts-and-their-respective-files","title":"Generation scripts and their respective files","text":""},{"location":"developer/auto-generation/#documentation","title":"Documentation","text":""},{"location":"developer/auto-generation/#variables","title":"Variables","text":"<ul> <li><code>cli_env_vars.env</code> -- All CLI environment variables in a <code>dotenv</code> file.</li> <li><code>cli_env_vars.md</code> -- All CLI environment variables in a table.</li> <li><code>config_env_vars.env</code> -- Almost all pipeline config environment variables in a <code>dotenv</code> file. The script checks for each field in <code>PipelineConfig</code> whether it has an <code>env</code> attribute defined. The script is currently unable to visit the classes of fields like <code>topic_name_config</code>, hence any environment variables defined there would remain unknown to it.</li> <li><code>config_env_vars.env</code> -- Almost all pipeline config environment variables in a table.</li> <li><code>variable_substitution.yaml</code> -- A copy of <code>./tests/pipeline/resources/component-type-substitution/pipeline.yaml</code> used as an example of substitution.</li> </ul>"},{"location":"developer/auto-generation/#cli-commands","title":"CLI commands","text":"<p>Generated by <code>typer-cli</code> from the code in <code>main.py</code>. It is called with Python's <code>subprocess</code> module.</p>"},{"location":"developer/auto-generation/#pipeline-and-defaults-example-definitions","title":"Pipeline and defaults example definitions","text":"<p>Generates example <code>pipeline.yaml</code> and <code>defaults.yaml</code> for each individual component, stores them and also concatenates them into 1 big pipeline definition and 1 big pipeline defaults definition.</p> <p>User input</p> <ul> <li><code>headers/*\\.yaml</code> -- The top of each example. Includes a description comment, <code>type</code> and <code>name</code>. The headers for <code>pipeline.yaml</code> reside in the <code>pipeline-components</code> dir and the <code>defaults.yaml</code> headers reside in the <code>pipeline-defaults</code> dir. The names of the files must be equal to the respective component <code>type</code>.</li> <li><code>sections/*\\.yaml</code> -- Each YAML file contains a single section (component attribute) definition. The intention is to keep the minimal set of definitions there from which any component definition can be built. The names of the files must be equal to the respective component <code>type</code> and the attribute name. The sections are used for both <code>defaults.yaml</code> and <code>pipeline.yaml</code> generation and reside in the <code>pipeline-components</code> dir.</li> </ul> <p>Generated</p> <ul> <li><code>pipeline-components/dependencies/*</code>   Cached information about KPOps components</li> <li><code>pipeline_component_dependencies.yaml</code> -- Specifies per component which files in the <code>sections</code> dir should be used for the <code>pipeline.yaml</code> generation.</li> <li><code>defaults_pipeline_component_dependencies.yaml</code> -- Specifies per component which files in the <code>sections</code> dir should be used for the <code>defaults.yaml</code> generation.</li> <li><code>kpops_structure.yaml</code> -- Specifies the inheritance hierarchy of the components and what sections exist in each component.</li> <li><code>pipeline-components/*\\.yaml</code> -- All single-component pipeline definitions and one big (complete) <code>pipeline.yaml</code> that contains all of them.</li> <li><code>pipeline-defaults/*\\.yaml</code> -- All single-component defaults definitions and one big (complete) <code>defaults.yaml</code> that contains all of them.</li> </ul>"},{"location":"developer/auto-generation/#editor-integration","title":"Editor integration","text":""},{"location":"developer/auto-generation/#schemas","title":"Schemas","text":"<ul> <li>config.json</li> <li>pipeline.json</li> </ul>"},{"location":"developer/formatting/","title":"Formatting","text":""},{"location":"developer/formatting/#markdown","title":"Markdown","text":"<p>To ensure a consistent markdown style, we use dprint to check and reformat.</p> <pre><code>dprint fmt\n</code></pre> <p>Use the official documentation to set up dprint. The configuration can be found here.</p>"},{"location":"resources/architecture/components-hierarchy/","title":"Components hierarchy","text":"<pre><code>flowchart BT\n    KubernetesApp --&gt; PipelineComponent\n    KafkaConnector --&gt; PipelineComponent\n    KafkaApp --&gt; KubernetesApp\n    StreamsApp --&gt; KafkaApp\n    ProducerApp --&gt; KafkaApp\n    KafkaSourceConnector --&gt; KafkaConnector\n    KafkaSinkConnector --&gt; KafkaConnector\n    \n    click KubernetesApp \"../kubernetes-app\"\n    click KafkaApp \"../kafka-app\"\n    click StreamsApp \"../streams-app\"\n    click ProducerApp \"../producer-app\"\n    click KafkaConnector \"../kafka-connector\"\n    click KafkaSourceConnector \"../kafka-source-connector\"\n    click KafkaSinkConnector \"../kafka-sink-connector\"</code></pre> <p>KPOps component hierarchy</p>"},{"location":"resources/examples/defaults/","title":"Example <code>defaults.yaml</code> files","text":""},{"location":"resources/examples/defaults/#atm-fraud-pipeline","title":"ATM Fraud Pipeline","text":"defaults.yaml <pre><code>pipeline-component:\nprefix: \"\"\nkubernetes-app:\nnamespace: ${NAMESPACE}\nkafka-connector:\nnamespace: ${NAMESPACE}\nkafka-app:\napp:\nstreams:\nbrokers: ${brokers}\nschemaRegistryUrl: ${schema_registry_url}\noptimizeLeaveGroupBehavior: false\nproducer-app:\nto:\ntopics:\n${output_topic_name}:\npartitions_count: 3\nstreams-app:\napp:\nlabels:\npipeline: ${pipeline_name}\nstreams:\noptimizeLeaveGroupBehavior: false\nto:\ntopics:\n${error_topic_name}:\ntype: error\npartitions_count: 1\n${output_topic_name}:\npartitions_count: 3\n</code></pre>"},{"location":"resources/examples/defaults/#word-count-pipeline","title":"Word-count Pipeline","text":"defaults.yaml <pre><code>pipeline-component:\nprefix: word-count-\nnamespace: kpops\nkafka-app:\napp:\nstreams:\nbrokers: ${brokers}\nprometheus:\njmx:\nenabled: false\ndebug: true\nproducer-app:\nto:\ntopics:\n${output_topic_name}:\ntype: output\npartitions_count: 1\nstreams-app:\napp:\nlabels:\npipeline: ${pipeline_name}\ncommandLine:\nPRODUCTIVE: false\nto:\ntopics:\n${error_topic_name}:\ntype: error\npartitions_count: 1\n${output_topic_name}:\ntype: output\npartitions_count: 1\n</code></pre>"},{"location":"resources/examples/pipeline/","title":"Example <code>pipeline.yaml</code> files","text":""},{"location":"resources/examples/pipeline/#atm-fraud-pipeline","title":"ATM Fraud Pipeline","text":"pipeline.yaml <pre><code> - type: producer-app\nname: account-producer\napp:\nreplicaCount: 1\nimage: ${DOCKER_REGISTRY}/atm-demo-accountproducer\nimageTag: 1.0.0\nschedule: 0 12 * * *\nsuspend: true\nprometheus:\njmx:\nenabled: false\ndebug: true\n- type: producer-app\nname: transaction-avro-producer\napp:\nreplicaCount: 1\nimage: ${DOCKER_REGISTRY}/atm-demo-transactionavroproducer\nimageTag: 1.0.0\ncommandLine:\nREAL_TX: 19\nITERATION: 20\nschedule: 0 12 * * *\nsuspend: true\nprometheus:\njmx:\nenabled: false\ndebug: true\n- type: streams-app\nname: transaction-joiner\napp:\nreplicaCount: 1\nimage: ${DOCKER_REGISTRY}/atm-demo-transactionjoiner\nimageTag: 1.0.0\ncommandLine:\nPRODUCTIVE: false\nannotations:\nconsumerGroup: atm-transactionjoiner-atm-fraud-joinedtransactions-topic\nprometheus:\njmx:\nenabled: false\ndebug: true\n- type: streams-app\nname: fraud-detector\napp:\nreplicaCount: 1\nimage: ${DOCKER_REGISTRY}/atm-demo-frauddetector\nimageTag: 1.0.0\ncommandLine:\nPRODUCTIVE: false\nannotations:\nconsumerGroup: atm-frauddetector-atm-fraud-possiblefraudtransactions-topic\nprometheus:\njmx:\nenabled: false\ndebug: true\n- type: streams-app\nname: account-linker\nfrom:\ncomponents:\nfraud-detector:\ntype: input\naccount-producer:\nrole: accounts\napp:\nreplicaCount: 1\nimage: ${DOCKER_REGISTRY}/atm-demo-accountlinker\nimageTag: 1.0.0\ncommandLine:\nPRODUCTIVE: false\nannotations:\nconsumerGroup: atm-accountlinker-atm-fraud-output-topic\nprometheus:\njmx:\nenabled: false\ndebug: true\n- type: kafka-sink-connector\nname: postgresql-connector\napp:\nconnector.class: io.confluent.connect.jdbc.JdbcSinkConnector\ntasks.max: 1\ntopics: ${pipeline_name}-account-linker-topic\nconnection.url: jdbc:postgresql://postgresql-dev.kpops.svc.cluster.local:5432/app_db\nconnection.user: app1\nconnection.password: AppPassword\nconnection.ds.pool.size: 5\ninsert.mode: insert\ninsert.mode.databaselevel: true\nvalue.converter: io.confluent.connect.avro.AvroConverter\nvalue.converter.schema.registry.url: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081\nkey.converter: org.apache.kafka.connect.storage.StringConverter\ntransforms: flatten\ntransforms.flatten.type: org.apache.kafka.connect.transforms.Flatten$Value\nauto.create: true\ntable.name.format: fraud_transactions\nerrors.deadletterqueue.context.headers.enable: true\nerrors.deadletterqueue.topic.name: postgres-request-sink-dead-letters\nerrors.deadletterqueue.topic.replication.factor: 1\nerrors.tolerance: all\npk.mode: record_value\n</code></pre>"},{"location":"resources/examples/pipeline/#word-count-pipeline","title":"Word-count Pipeline","text":"pipeline.yaml <pre><code>- type: producer-app\nname: data-producer\napp:\nimage: bakdata/kpops-demo-sentence-producer\nimageTag: \"1.0.0\"\n- type: streams-app\nname: word-counter\nto:\ntopics:\n${output_topic_name}:\ntype: output\nconfigs:\ncleanup.policy: compact\napp:\nreplicaCount: 1\nimage: bakdata/kpops-demo-word-count-app\nimageTag: \"1.0.0\"\n- type: kafka-sink-connector\nname: redis-sink-connector\napp:\nconnector.class: com.github.jcustenborder.kafka.connect.redis.RedisSinkConnector\nredis.hosts: redis-headless:6379\nredis.database: 0\ntasks.max: 1\nkey.converter: org.apache.kafka.connect.storage.StringConverter\nvalue.converter: org.apache.kafka.connect.storage.StringConverter\n</code></pre>"},{"location":"resources/pipeline-components/pipeline/","title":"pipeline.yaml","text":"<pre><code># Base component for Kafka-based components.\n# Producer or streaming apps should inherit from this class.\n- type: kafka-app # required\nname: kafka-app # required\n# Pipeline prefix that will prefix every component name. If you wish to not\n# have any prefix you can specify an empty string.\nprefix: ${pipeline_name}-\nfrom: # Must not be null\ntopics: # read from topic\n${pipeline_name}-input-topic:\ntype: input # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra\n${pipeline_name}-input-pattern-topic:\ntype: pattern # Implied to be an input pattern if `role` is undefined\n${pipeline_name}-extra-pattern-topic:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\ncomponents: # read from specific component\naccount-producer:\ntype: output # Implied when role is NOT specified\nother-producer:\nrole: some-role # Implies `type` to be extra\ncomponent-as-input-pattern:\ntype: pattern # Implied to be an input pattern if `role` is undefined\ncomponent-as-extra-pattern:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\n# Topic(s) into which the component will write output\nto:\ntopics:\n${pipeline_name}-output-topic:\ntype: output # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined\n${pipeline_name}-error-topic:\ntype: error\n# Currently KPOps supports Avro and JSON schemas.\nkey_schema: key-schema # must implement SchemaProvider to use\nvalue_schema: value-schema\npartitions_count: 1\nreplication_factor: 1\nconfigs: # https://kafka.apache.org/documentation/#topicconfigs\ncleanup.policy: compact\nmodels: # SchemaProvider is initiated with the values given here\nmodel: model\nnamespace: namespace # required\n# `app` can contain application-specific settings, hence  the user is free to\n# add the key-value pairs they need.\napp: # required\nstreams: # required\nbrokers: ${kafka_brokers} # required\nschemaRegistryUrl: ${schema_registry_url}\nnameOverride: override-with-this-name # kafka-app-specific\nimageTag: \"1.0.0\" # Example values that are shared between streams-app and producer-app\n# Helm repository configuration (optional)\n# If not set the helm repo add will not be called. Useful when using local Helm charts\nrepo_config:\nrepository_name: bakdata-streams-bootstrap # required\nurl: https://bakdata.github.io/streams-bootstrap/ # required\nrepo_auth_flags:\nusername: user\npassword: pass\nca_file: /home/user/path/to/ca-file\ninsecure_skip_tls_verify: false\nversion: \"2.12.0\" # Helm chart version\n# Kafka sink connector\n- type: kafka-sink-connector\nname: kafka-sink-connector # required\n# Pipeline prefix that will prefix every component name. If you wish to not\n# have any prefix you can specify an empty string.\nprefix: ${pipeline_name}-\nfrom: # Must not be null\ntopics: # read from topic\n${pipeline_name}-input-topic:\ntype: input # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra\n${pipeline_name}-input-pattern-topic:\ntype: pattern # Implied to be an input pattern if `role` is undefined\n${pipeline_name}-extra-pattern-topic:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\ncomponents: # read from specific component\naccount-producer:\ntype: output # Implied when role is NOT specified\nother-producer:\nrole: some-role # Implies `type` to be extra\ncomponent-as-input-pattern:\ntype: pattern # Implied to be an input pattern if `role` is undefined\ncomponent-as-extra-pattern:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\n# Topic(s) into which the component will write output\nto:\ntopics:\n${pipeline_name}-output-topic:\ntype: output # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined\n${pipeline_name}-error-topic:\ntype: error\n# Currently KPOps supports Avro and JSON schemas.\nkey_schema: key-schema # must implement SchemaProvider to use\nvalue_schema: value-schema\npartitions_count: 1\nreplication_factor: 1\nconfigs: # https://kafka.apache.org/documentation/#topicconfigs\ncleanup.policy: compact\nmodels: # SchemaProvider is initiated with the values given here\nmodel: model\nnamespace: namespace # required\n# `app` contains application-specific settings, hence it does not have a rigid\n# structure. The fields below are just an example. Extensive documentation on\n# connectors: https://kafka.apache.org/documentation/#connectconfigs\napp: # required\ntasks.max: 1\n# Helm repository configuration for resetter\nrepo_config:\nrepository_name: my-repo # required\nurl: https://bakdata.github.io/kafka-connect-resetter/ # required\nrepo_auth_flags:\nusername: user\npassword: pass\nca_file: /home/user/path/to/ca-file\ninsecure_skip_tls_verify: false\nversion: \"1.0.6\" # Helm chart version\n# Overriding Kafka Connect Resetter Helm values. E.g. to override the\n# Image Tag etc.\nresetter_values:\nimageTag: \"1.2.3\"\n# Kafka source connector\n- type: kafka-source-connector # required\nname: kafka-source-connector # required\n# Pipeline prefix that will prefix every component name. If you wish to not\n# have any prefix you can specify an empty string.\nprefix: ${pipeline_name}-\n# The source connector has no `from` section\n# from:\n# Topic(s) into which the component will write output\nto:\ntopics:\n${pipeline_name}-output-topic:\ntype: output # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined\n${pipeline_name}-error-topic:\ntype: error\n# Currently KPOps supports Avro and JSON schemas.\nkey_schema: key-schema # must implement SchemaProvider to use\nvalue_schema: value-schema\npartitions_count: 1\nreplication_factor: 1\nconfigs: # https://kafka.apache.org/documentation/#topicconfigs\ncleanup.policy: compact\nmodels: # SchemaProvider is initiated with the values given here\nmodel: model\nnamespace: namespace # required\n# `app` contains application-specific settings, hence it does not have a rigid\n# structure. The fields below are just an example. Extensive documentation on\n# connectors: https://kafka.apache.org/documentation/#connectconfigs\napp: # required\ntasks.max: 1\n# Helm repository configuration for resetter\nrepo_config:\nrepository_name: my-repo # required\nurl: https://bakdata.github.io/kafka-connect-resetter/ # required\nrepo_auth_flags:\nusername: user\npassword: pass\nca_file: /home/user/path/to/ca-file\ninsecure_skip_tls_verify: false\nversion: \"1.0.6\" # Helm chart version\n# Overriding Kafka Connect Resetter Helm values. E.g. to override the\n# Image Tag etc.\nresetter_values:\nimageTag: \"1.2.3\"\n# offset.storage.topic\n# https://kafka.apache.org/documentation/#connect_running\noffset_topic: offset_topic\n# Base Kubernetes App\n- type: kubernetes-app\nname: kubernetes-app # required\n# Pipeline prefix that will prefix every component name. If you wish to not\n# have any prefix you can specify an empty string.\nprefix: ${pipeline_name}-\nfrom: # Must not be null\ntopics: # read from topic\n${pipeline_name}-input-topic:\ntype: input # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra\n${pipeline_name}-input-pattern-topic:\ntype: pattern # Implied to be an input pattern if `role` is undefined\n${pipeline_name}-extra-pattern-topic:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\ncomponents: # read from specific component\naccount-producer:\ntype: output # Implied when role is NOT specified\nother-producer:\nrole: some-role # Implies `type` to be extra\ncomponent-as-input-pattern:\ntype: pattern # Implied to be an input pattern if `role` is undefined\ncomponent-as-extra-pattern:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\n# Topic(s) into which the component will write output\nto:\ntopics:\n${pipeline_name}-output-topic:\ntype: output # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined\n${pipeline_name}-error-topic:\ntype: error\n# Currently KPOps supports Avro and JSON schemas.\nkey_schema: key-schema # must implement SchemaProvider to use\nvalue_schema: value-schema\npartitions_count: 1\nreplication_factor: 1\nconfigs: # https://kafka.apache.org/documentation/#topicconfigs\ncleanup.policy: compact\nmodels: # SchemaProvider is initiated with the values given here\nmodel: model\nnamespace: namespace # required\n# `app` contains application-specific settings, hence it does not have a rigid\n# structure. The fields below are just an example.\napp: # required\nimage: exampleImage # Example\ndebug: false # Example\ncommandLine: {} # Example\n# Helm repository configuration (optional)\n# If not set the helm repo add will not be called. Useful when using local Helm charts\nrepo_config:\nrepository_name: bakdata-streams-bootstrap # required\nurl: https://bakdata.github.io/streams-bootstrap/ # required\nrepo_auth_flags:\nusername: user\npassword: pass\nca_file: /home/user/path/to/ca-file\ninsecure_skip_tls_verify: false\nversion: \"1.0.0\" # Helm chart version\n# Holds configuration to use as values for the streams bootstrap producer-app Helm\n# chart.\n# More documentation on ProducerApp:\n# https://github.com/bakdata/streams-bootstrap\n- type: producer-app\nname: producer-app # required\n# Pipeline prefix that will prefix every component name. If you wish to not\n# have any prefix you can specify an empty string.\nprefix: ${pipeline_name}-\n# from: # While the producer-app does inherit from kafka-app, it does not need a\n# `from` section, hence it does not support it.\n# Topic(s) into which the component will write output\nto:\ntopics:\n${pipeline_name}-output-topic:\ntype: output # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined\n${pipeline_name}-error-topic:\ntype: error\n# Currently KPOps supports Avro and JSON schemas.\nkey_schema: key-schema # must implement SchemaProvider to use\nvalue_schema: value-schema\npartitions_count: 1\nreplication_factor: 1\nconfigs: # https://kafka.apache.org/documentation/#topicconfigs\ncleanup.policy: compact\nmodels: # SchemaProvider is initiated with the values given here\nmodel: model\nnamespace: namespace # required\n# Allowed configs:\n# https://github.com/bakdata/streams-bootstrap/tree/master/charts/producer-app\napp: # required\nstreams: # required, producer-app-specific\nbrokers: ${kafka_brokers} # required\nschemaRegistryUrl: ${schema_registry_url}\noutputTopic: output_topic\nextraOutputTopics:\noutput_role1: output_topic1\noutput_role2: output_topic2\nnameOverride: override-with-this-name # kafka-app-specific\n# Helm repository configuration (optional)\n# If not set the helm repo add will not be called. Useful when using local Helm charts\nrepo_config:\nrepository_name: bakdata-streams-bootstrap # required\nurl: https://bakdata.github.io/streams-bootstrap/ # required\nrepo_auth_flags:\nusername: user\npassword: pass\nca_file: /home/user/path/to/ca-file\ninsecure_skip_tls_verify: false\nversion: \"2.12.0\" # Helm chart version\n# StreamsApp component that configures a streams bootstrap app.\n# More documentation on StreamsApp: https://github.com/bakdata/streams-bootstrap\n- type: streams-app # required\nname: streams-app # required\n# Pipeline prefix that will prefix every component name. If you wish to not\n# have any prefix you can specify an empty string.\nprefix: ${pipeline_name}-\nfrom: # Must not be null\ntopics: # read from topic\n${pipeline_name}-input-topic:\ntype: input # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra\n${pipeline_name}-input-pattern-topic:\ntype: pattern # Implied to be an input pattern if `role` is undefined\n${pipeline_name}-extra-pattern-topic:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\ncomponents: # read from specific component\naccount-producer:\ntype: output # Implied when role is NOT specified\nother-producer:\nrole: some-role # Implies `type` to be extra\ncomponent-as-input-pattern:\ntype: pattern # Implied to be an input pattern if `role` is undefined\ncomponent-as-extra-pattern:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\n# Topic(s) into which the component will write output\nto:\ntopics:\n${pipeline_name}-output-topic:\ntype: output # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined\n${pipeline_name}-error-topic:\ntype: error\n# Currently KPOps supports Avro and JSON schemas.\nkey_schema: key-schema # must implement SchemaProvider to use\nvalue_schema: value-schema\npartitions_count: 1\nreplication_factor: 1\nconfigs: # https://kafka.apache.org/documentation/#topicconfigs\ncleanup.policy: compact\nmodels: # SchemaProvider is initiated with the values given here\nmodel: model\nnamespace: namespace # required\n# No arbitrary keys are allowed under `app`here\n# Allowed configs:\n# https://github.com/bakdata/streams-bootstrap/tree/master/charts/streams-app\napp: # required\n# Streams Bootstrap streams section\nstreams: # required, streams-app-specific\nbrokers: ${kafka_brokers} # required\nschemaRegistryUrl: ${schema_registry_url}\ninputTopics:\n- topic1\n- topic2\noutputTopic: output-topic\ninputPattern: input-pattern\nextraInputTopics:\ninput_role1:\n- input_topic1\n- input_topic2\ninput_role2:\n- input_topic3\n- input_topic4\nextraInputPatterns:\npattern_role1: input_pattern1\nextraOutputTopics:\noutput_role1: output_topic1\noutput_role2: output_topic2\nerrorTopic: error-topic\nconfig:\nmy.streams.config: my.value\nnameOverride: override-with-this-name # streams-app-specific\nautoscaling: # streams-app-specific\nconsumerGroup: consumer-group # required\nlagThreshold: 0 # Average target value to trigger scaling actions.\nenabled: false # Whether to enable auto-scaling using KEDA.\n# This is the interval to check each trigger on.\n# https://keda.sh/docs/2.9/concepts/scaling-deployments/#pollinginterval\npollingInterval: 30\n# The period to wait after the last trigger reported active before scaling\n#  the resource back to 0. https://keda.sh/docs/2.9/concepts/scaling-deployments/#cooldownperiod\ncooldownPeriod: 300\n# The offset reset policy for the consumer if the the consumer group is\n# not yet subscribed to a partition.\noffsetResetPolicy: earliest\n# This setting is passed to the HPA definition that KEDA will create for a\n# given resource and holds the maximum number of replicas of the target resouce.\n# https://keda.sh/docs/2.9/concepts/scaling-deployments/#maxreplicacount\nmaxReplicas: 1\n# Minimum number of replicas KEDA will scale the resource down to.\n# https://keda.sh/docs/2.7/concepts/scaling-deployments/#minreplicacount\nminReplicas: 0\n# If this property is set, KEDA will scale the resource down to this\n# number of replicas.\n# https://keda.sh/docs/2.9/concepts/scaling-deployments/#idlereplicacount\nidleReplicas: 0\ntopics: # List of auto-generated Kafka Streams topics used by the streams app.\n- topic1\n- topic2\n# Helm repository configuration (optional)\n# If not set the helm repo add will not be called. Useful when using local Helm charts\nrepo_config:\nrepository_name: bakdata-streams-bootstrap # required\nurl: https://bakdata.github.io/streams-bootstrap/ # required\nrepo_auth_flags:\nusername: user\npassword: pass\nca_file: /home/user/path/to/ca-file\ninsecure_skip_tls_verify: false\nversion: \"2.12.0\" # Helm chart version\n</code></pre>"},{"location":"resources/pipeline-defaults/defaults/","title":"defaults.yaml","text":"<pre><code># Base component for Kafka-based components.\n#\n# Parent of: ProducerApp, StreamsApp\n# Child of: KubernetesApp\nkafka-app:\n# `app` can contain application-specific settings, hence  the user is free to\n# add the key-value pairs they need.\napp: # required\nstreams: # required\nbrokers: ${kafka_brokers} # required\nschemaRegistryUrl: ${schema_registry_url}\nnameOverride: override-with-this-name # kafka-app-specific\nimageTag: \"1.0.0\" # Example values that are shared between streams-app and producer-app\nversion: \"2.12.0\" # Helm chart version\n# Kafka connector\n#\n# Parent of: KafkaSinkConnector, KafkaSourceConnector\n# Child of: PipelineComponent\nkafka-connector:\n# Pipeline prefix that will prefix every component name. If you wish to not\n# have any prefix you can specify an empty string.\nprefix: ${pipeline_name}-\nfrom: # Must not be null\ntopics: # read from topic\n${pipeline_name}-input-topic:\ntype: input # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra\n${pipeline_name}-input-pattern-topic:\ntype: pattern # Implied to be an input pattern if `role` is undefined\n${pipeline_name}-extra-pattern-topic:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\ncomponents: # read from specific component\naccount-producer:\ntype: output # Implied when role is NOT specified\nother-producer:\nrole: some-role # Implies `type` to be extra\ncomponent-as-input-pattern:\ntype: pattern # Implied to be an input pattern if `role` is undefined\ncomponent-as-extra-pattern:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\n# Topic(s) into which the component will write output\nto:\ntopics:\n${pipeline_name}-output-topic:\ntype: output # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined\n${pipeline_name}-error-topic:\ntype: error\n# Currently KPOps supports Avro and JSON schemas.\nkey_schema: key-schema # must implement SchemaProvider to use\nvalue_schema: value-schema\npartitions_count: 1\nreplication_factor: 1\nconfigs: # https://kafka.apache.org/documentation/#topicconfigs\ncleanup.policy: compact\nmodels: # SchemaProvider is initiated with the values given here\nmodel: model\nnamespace: namespace # required\n# `app` contains application-specific settings, hence it does not have a rigid\n# structure. The fields below are just an example. Extensive documentation on\n# connectors: https://kafka.apache.org/documentation/#connectconfigs\napp: # required\ntasks.max: 1\n# Helm repository configuration for resetter\nrepo_config:\nrepository_name: my-repo # required\nurl: https://bakdata.github.io/kafka-connect-resetter/ # required\nrepo_auth_flags:\nusername: user\npassword: pass\nca_file: /home/user/path/to/ca-file\ninsecure_skip_tls_verify: false\nversion: \"1.0.6\" # Helm chart version\n# Overriding Kafka Connect Resetter Helm values. E.g. to override the\n# Image Tag etc.\nresetter_values:\nimageTag: \"1.2.3\"\n# Kafka sink connector\n#\n# Child of: KafkaConnector\nkafka-sink-connector:\n# No settings differ from `kafka-connector`\n# Kafka source connector\n#\n# Child of: KafkaConnector\nkafka-source-connector:\n# The source connector has no `from` section\n# from:\n# offset.storage.topic\n# https://kafka.apache.org/documentation/#connect_running\noffset_topic: offset_topic\n# Base Kubernetes App\n#\n# Parent of: KafkaApp\n# Child of: PipelineComponent\nkubernetes-app:\n# Pipeline prefix that will prefix every component name. If you wish to not\n# have any prefix you can specify an empty string.\nprefix: ${pipeline_name}-\nfrom: # Must not be null\ntopics: # read from topic\n${pipeline_name}-input-topic:\ntype: input # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra\n${pipeline_name}-input-pattern-topic:\ntype: pattern # Implied to be an input pattern if `role` is undefined\n${pipeline_name}-extra-pattern-topic:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\ncomponents: # read from specific component\naccount-producer:\ntype: output # Implied when role is NOT specified\nother-producer:\nrole: some-role # Implies `type` to be extra\ncomponent-as-input-pattern:\ntype: pattern # Implied to be an input pattern if `role` is undefined\ncomponent-as-extra-pattern:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\n# Topic(s) into which the component will write output\nto:\ntopics:\n${pipeline_name}-output-topic:\ntype: output # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined\n${pipeline_name}-error-topic:\ntype: error\n# Currently KPOps supports Avro and JSON schemas.\nkey_schema: key-schema # must implement SchemaProvider to use\nvalue_schema: value-schema\npartitions_count: 1\nreplication_factor: 1\nconfigs: # https://kafka.apache.org/documentation/#topicconfigs\ncleanup.policy: compact\nmodels: # SchemaProvider is initiated with the values given here\nmodel: model\nnamespace: namespace # required\n# `app` contains application-specific settings, hence it does not have a rigid\n# structure. The fields below are just an example.\napp: # required\nimage: exampleImage # Example\ndebug: false # Example\ncommandLine: {} # Example\n# Helm repository configuration (optional)\n# If not set the helm repo add will not be called. Useful when using local Helm charts\nrepo_config:\nrepository_name: bakdata-streams-bootstrap # required\nurl: https://bakdata.github.io/streams-bootstrap/ # required\nrepo_auth_flags:\nusername: user\npassword: pass\nca_file: /home/user/path/to/ca-file\ninsecure_skip_tls_verify: false\nversion: \"1.0.0\" # Helm chart version\n# Holds configuration to use as values for the streams bootstrap producer-app Helm\n# chart.\n#\n# Child of: KafkaApp\n# More documentation on ProducerApp: https://github.com/bakdata/streams-bootstrap\nproducer-app:\n# from: # While the producer-app does inherit from kafka-app, it does not need a\n# `from` section, hence it does not support it.\n# Allowed configs:\n# https://github.com/bakdata/streams-bootstrap/tree/master/charts/producer-app\napp: # required\nstreams: # required, producer-app-specific\nbrokers: ${kafka_brokers} # required\nschemaRegistryUrl: ${schema_registry_url}\noutputTopic: output_topic\nextraOutputTopics:\noutput_role1: output_topic1\noutput_role2: output_topic2\nnameOverride: override-with-this-name # kafka-app-specific\n# StreamsApp component that configures a streams bootstrap app.\n#\n# Child of: KafkaApp\n# More documentation on StreamsApp: https://github.com/bakdata/streams-bootstrap\nstreams-app:\n# No arbitrary keys are allowed under `app`here\n# Allowed configs:\n# https://github.com/bakdata/streams-bootstrap/tree/master/charts/streams-app\napp: # required\n# Streams Bootstrap streams section\nstreams: # required, streams-app-specific\nbrokers: ${kafka_brokers} # required\nschemaRegistryUrl: ${schema_registry_url}\ninputTopics:\n- topic1\n- topic2\noutputTopic: output-topic\ninputPattern: input-pattern\nextraInputTopics:\ninput_role1:\n- input_topic1\n- input_topic2\ninput_role2:\n- input_topic3\n- input_topic4\nextraInputPatterns:\npattern_role1: input_pattern1\nextraOutputTopics:\noutput_role1: output_topic1\noutput_role2: output_topic2\nerrorTopic: error-topic\nconfig:\nmy.streams.config: my.value\nnameOverride: override-with-this-name # streams-app-specific\nautoscaling: # streams-app-specific\nconsumerGroup: consumer-group # required\nlagThreshold: 0 # Average target value to trigger scaling actions.\nenabled: false # Whether to enable auto-scaling using KEDA.\n# This is the interval to check each trigger on.\n# https://keda.sh/docs/2.9/concepts/scaling-deployments/#pollinginterval\npollingInterval: 30\n# The period to wait after the last trigger reported active before scaling\n#  the resource back to 0. https://keda.sh/docs/2.9/concepts/scaling-deployments/#cooldownperiod\ncooldownPeriod: 300\n# The offset reset policy for the consumer if the the consumer group is\n# not yet subscribed to a partition.\noffsetResetPolicy: earliest\n# This setting is passed to the HPA definition that KEDA will create for a\n# given resource and holds the maximum number of replicas of the target resouce.\n# https://keda.sh/docs/2.9/concepts/scaling-deployments/#maxreplicacount\nmaxReplicas: 1\n# Minimum number of replicas KEDA will scale the resource down to.\n# https://keda.sh/docs/2.7/concepts/scaling-deployments/#minreplicacount\nminReplicas: 0\n# If this property is set, KEDA will scale the resource down to this\n# number of replicas.\n# https://keda.sh/docs/2.9/concepts/scaling-deployments/#idlereplicacount\nidleReplicas: 0\ntopics: # List of auto-generated Kafka Streams topics used by the streams app.\n- topic1\n- topic2\n</code></pre>"},{"location":"resources/variables/cli_env_vars/","title":"Cli env vars","text":"<p>These variables are a lower priority alternative to the commands' flags. If a variable is set, the corresponding flag does not have to be specified in commands. Variables marked as required can instead be set as flags.</p> Name Default Value Required Description KPOPS_PIPELINE_BASE_DIR . False Base directory to the pipelines (default is current working directory) KPOPS_CONFIG_PATH config.yaml False Path to the config.yaml file KPOPS_DEFAULT_PATH False Path to defaults folder KPOPS_PIPELINE_PATH True Path to YAML with pipeline definition KPOPS_PIPELINE_STEPS False Comma separated list of steps to apply the command on"},{"location":"resources/variables/config_env_vars/","title":"Config env vars","text":"<p>These variables are a lower priority alternative to the settings in <code>config.yaml</code>. Variables marked as required can instead be set in the pipeline config.</p> Name Default Value Required Description Setting name KPOPS_ENVIRONMENT True The environment you want to generate and deploy the pipeline to. Suffix your environment files with this value (e.g. defaults_development.yaml for environment=development). environment KPOPS_KAFKA_BROKERS True The comma separated Kafka brokers address. kafka_brokers KPOPS_SCHEMA_REGISTRY_URL http://localhost:8081 False Address of the Schema Registry. url KPOPS_KAFKA_REST_URL http://localhost:8082 False Address of the Kafka REST Proxy. url KPOPS_KAFKA_CONNECT_URL http://localhost:8083 False Address of Kafka Connect. url KPOPS_TIMEOUT 300 False The timeout in seconds that specifies when actions like deletion or deploy timeout. timeout KPOPS_RETAIN_CLEAN_JOBS False False Whether to retain clean up jobs in the cluster or uninstall the, after completion. retain_clean_jobs"},{"location":"user/what-is-kpops/","title":"What is KPOps?","text":"<p>With a couple of easy commands in the shell and a <code>pipeline.yaml</code> of under 30 lines, KPOps can not only <code>deploy</code> a Kafka pipeline1 to a Kubernetes cluster, but also <code>reset</code>, <code>clean</code> or <code>destroy</code> it!</p>"},{"location":"user/what-is-kpops/#key-features","title":"Key features","text":"<ul> <li>Deploy Kafka apps to Kubernetes: KPOps allows to deploy consecutive Kafka Streams applications and producers using an easy-to-read and -write pipeline definition.</li> <li>Manage Kafka Connectors: KPOps connects with your Kafka Connect cluster and deploys, validates, and deletes your connectors.</li> <li>Configure multiple pipelines and steps: KPOps has various abstractions that simplify configuring multiple pipelines and steps within pipelines by sharing common configuration between different components, such as producers or streaming applications.</li> <li>Handle your topics and schemas: KPOps not only creates and deletes your topics but also registers and deletes your schemas.</li> <li>Clean termination of Kafka components: KPOps removes your pipeline components (i.e., Kafka Streams applications) from the Kubernetes cluster and cleans up the component-related states (i.e., removing/resetting offset of Kafka consumer groups).</li> <li>Preview your pipeline changes: With the KPOps dry-run, you can ensure your pipeline definition is set up correctly. This helps to minimize downtime and prevent potential errors or issues that could impact your production environment.</li> </ul>"},{"location":"user/what-is-kpops/#example","title":"Example","text":"An overview of Word-count pipeline shown in Streams Explorer Word-count pipeline.yaml<pre><code>- type: producer-app\nname: data-producer\napp:\nimage: bakdata/kpops-demo-sentence-producer\nimageTag: \"1.0.0\"\n- type: streams-app\nname: word-counter\nto:\ntopics:\n${output_topic_name}:\ntype: output\nconfigs:\ncleanup.policy: compact\napp:\nreplicaCount: 1\nimage: bakdata/kpops-demo-word-count-app\nimageTag: \"1.0.0\"\n- type: kafka-sink-connector\nname: redis-sink-connector\napp:\nconnector.class: com.github.jcustenborder.kafka.connect.redis.RedisSinkConnector\nredis.hosts: redis-headless:6379\nredis.database: 0\ntasks.max: 1\nkey.converter: org.apache.kafka.connect.storage.StringConverter\nvalue.converter: org.apache.kafka.connect.storage.StringConverter\n</code></pre> <ol> <li> <p>A Kafka pipeline can consist of consecutive streaming applications, producers, and connectors.\u00a0\u21a9</p> </li> </ol>"},{"location":"user/core-concepts/config/","title":"Configuration","text":"<p>KPOps reads its global configuration that is unrelated to a pipeline's components from <code>config.yaml</code>.</p> <p>Consider enabling KPOps' editor integration feature to enjoy the benefits of autocompletion and validation when configuring your pipeline.</p> <p>To learn about any of the available settings, take a look at the example below.</p> <code>config.yaml</code> <pre><code># CONFIGURATION\n#\n# The path to the folder containing the defaults.yaml file and the environment\n# defaults files.\ndefaults_path: .\n# The environment you want to generate and deploy the pipeline to. Suffix your\n# environment files with this value (e.g. defaults_development.yaml and\n# pipeline_development.yaml for environment=development).\n# REQUIRED\nenvironment: development\n# The Kafka brokers address.\n# REQUIRED\nbrokers: \"http://broker1:9092,http://broker2:9092\"\n# The name of the defaults file and the prefix of the defaults environment file.\ndefaults_filename_prefix: defaults\n# Configures topic names.\ntopic_name_config: # Configures the value for the variable ${output_topic_name}\ndefault_output_topic_name: ${pipeline_name}-${component_name}\n# Configures the value for the variable ${error_topic_name}\ndefault_error_topic_name: ${pipeline_name}-${component_name}-error\n# Address of the Schema Registry\nschema_registry_url: \"http://localhost:8081\"\n# Address of the Kafka REST Proxy.\nkafka_rest_host: \"http://localhost:8082\"\n# Address of Kafka Connect.\nkafka_connect_host: \"http://localhost:8083\"\n# The timeout in seconds that specifies when actions like deletion or deploy\n# timeout.\ntimeout: 300\n# Flag for `helm upgrade --install`.\n# Create the release namespace if not present.\ncreate_namespace: false\n# Global flags for Helm.\nhelm_config:\n# Set the name of the kubeconfig context. (--kube-context)\ncontext: name\n# Run Helm in Debug mode.\ndebug: false\n# Configure Helm Diff.\nhelm_diff_config: # Set of keys that should not be checked.\nignore: - name\n- imageTag\n# Whether to retain clean up jobs in the cluster or uninstall the, after\n# completion.\nretain_clean_jobs: false\n</code></pre> <p>Environment-specific pipeline definitions</p> <p>Similarly to defaults, it is possible to have an unlimited amount of additional environment-specific pipeline definitions. The naming convention is the same: add a suffix of the form <code>_{environment}</code> to the filename.</p>"},{"location":"user/core-concepts/defaults/","title":"Defaults","text":"<p>KPOps has a very efficient way of dealing with repeating settings which manifests as <code>defaults.yaml</code>. This file provides the user with the power to set defaults for any and all components, thus omitting the need to repeat the same settings in <code>pipeline.yaml</code>.</p> <p>See real-world examples for <code>defaults</code>.</p>"},{"location":"user/core-concepts/defaults/#features","title":"Features","text":""},{"location":"user/core-concepts/defaults/#inheritance","title":"Inheritance","text":"<p>An important mechanic of KPOps is that <code>defaults</code> set for a component apply to all components that inherit from it.</p> <p>It is possible, although not recommended, to add settings that are specific to a component's subclass. An example would be configuring <code>offset_topic</code> under <code>kafka-connector</code> instead of <code>kafka-source-connector</code>.</p>"},{"location":"user/core-concepts/defaults/#configuration","title":"Configuration","text":"<p>It is possible to set specific <code>defaults</code> for each <code>environment</code> by adding files called <code>defaults_{environment}.yaml</code> to the defaults folder at <code>defaults_path</code>. The defaults are loaded based on the currently set environment.</p> <p>It is important to note that <code>defaults_{environment}.yaml</code> overrides only the settings that are explicitly set to be different from the ones in the base <code>defaults</code> file.</p> <p>Tip</p> <p><code>defaults</code> is the default value of <code>defaults_filename_prefix</code>. Together with <code>defaults_path</code> and <code>environment</code> it can be changed in <code>config.yaml</code></p>"},{"location":"user/core-concepts/defaults/#components","title":"Components","text":"<p>The <code>defaults</code> codeblocks in this section contain the full set of settings that are specific to the component. If a setting already exists in a parent config, it will not be included in the child's.</p>"},{"location":"user/core-concepts/defaults/#kubernetesapp","title":"KubernetesApp","text":"<code>defaults.yaml</code> <pre><code># Base Kubernetes App\n#\n# Parent of: KafkaApp\n# Child of: PipelineComponent\nkubernetes-app:\n# Pipeline prefix that will prefix every component name. If you wish to not\n# have any prefix you can specify an empty string.\nprefix: ${pipeline_name}-\nfrom: # Must not be null\ntopics: # read from topic\n${pipeline_name}-input-topic:\ntype: input # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra\n${pipeline_name}-input-pattern-topic:\ntype: pattern # Implied to be an input pattern if `role` is undefined\n${pipeline_name}-extra-pattern-topic:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\ncomponents: # read from specific component\naccount-producer:\ntype: output # Implied when role is NOT specified\nother-producer:\nrole: some-role # Implies `type` to be extra\ncomponent-as-input-pattern:\ntype: pattern # Implied to be an input pattern if `role` is undefined\ncomponent-as-extra-pattern:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\n# Topic(s) into which the component will write output\nto:\ntopics:\n${pipeline_name}-output-topic:\ntype: output # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined\n${pipeline_name}-error-topic:\ntype: error\n# Currently KPOps supports Avro and JSON schemas.\nkey_schema: key-schema # must implement SchemaProvider to use\nvalue_schema: value-schema\npartitions_count: 1\nreplication_factor: 1\nconfigs: # https://kafka.apache.org/documentation/#topicconfigs\ncleanup.policy: compact\nmodels: # SchemaProvider is initiated with the values given here\nmodel: model\nnamespace: namespace # required\n# `app` contains application-specific settings, hence it does not have a rigid\n# structure. The fields below are just an example.\napp: # required\nimage: exampleImage # Example\ndebug: false # Example\ncommandLine: {} # Example\n# Helm repository configuration (optional)\n# If not set the helm repo add will not be called. Useful when using local Helm charts\nrepo_config:\nrepository_name: bakdata-streams-bootstrap # required\nurl: https://bakdata.github.io/streams-bootstrap/ # required\nrepo_auth_flags:\nusername: user\npassword: pass\nca_file: /home/user/path/to/ca-file\ninsecure_skip_tls_verify: false\nversion: \"1.0.0\" # Helm chart version\n</code></pre>"},{"location":"user/core-concepts/defaults/#kafkaapp","title":"KafkaApp","text":"<code>defaults.yaml</code> <pre><code># Base component for Kafka-based components.\n#\n# Parent of: ProducerApp, StreamsApp\n# Child of: KubernetesApp\nkafka-app:\n# `app` can contain application-specific settings, hence  the user is free to\n# add the key-value pairs they need.\napp: # required\nstreams: # required\nbrokers: ${kafka_brokers} # required\nschemaRegistryUrl: ${schema_registry_url}\nnameOverride: override-with-this-name # kafka-app-specific\nimageTag: \"1.0.0\" # Example values that are shared between streams-app and producer-app\nversion: \"2.12.0\" # Helm chart version\n</code></pre>"},{"location":"user/core-concepts/defaults/#streamsapp","title":"StreamsApp","text":"<code>defaults.yaml</code> <pre><code># StreamsApp component that configures a streams bootstrap app.\n#\n# Child of: KafkaApp\n# More documentation on StreamsApp: https://github.com/bakdata/streams-bootstrap\nstreams-app:\n# No arbitrary keys are allowed under `app`here\n# Allowed configs:\n# https://github.com/bakdata/streams-bootstrap/tree/master/charts/streams-app\napp: # required\n# Streams Bootstrap streams section\nstreams: # required, streams-app-specific\nbrokers: ${kafka_brokers} # required\nschemaRegistryUrl: ${schema_registry_url}\ninputTopics:\n- topic1\n- topic2\noutputTopic: output-topic\ninputPattern: input-pattern\nextraInputTopics:\ninput_role1:\n- input_topic1\n- input_topic2\ninput_role2:\n- input_topic3\n- input_topic4\nextraInputPatterns:\npattern_role1: input_pattern1\nextraOutputTopics:\noutput_role1: output_topic1\noutput_role2: output_topic2\nerrorTopic: error-topic\nconfig:\nmy.streams.config: my.value\nnameOverride: override-with-this-name # streams-app-specific\nautoscaling: # streams-app-specific\nconsumerGroup: consumer-group # required\nlagThreshold: 0 # Average target value to trigger scaling actions.\nenabled: false # Whether to enable auto-scaling using KEDA.\n# This is the interval to check each trigger on.\n# https://keda.sh/docs/2.9/concepts/scaling-deployments/#pollinginterval\npollingInterval: 30\n# The period to wait after the last trigger reported active before scaling\n#  the resource back to 0. https://keda.sh/docs/2.9/concepts/scaling-deployments/#cooldownperiod\ncooldownPeriod: 300\n# The offset reset policy for the consumer if the the consumer group is\n# not yet subscribed to a partition.\noffsetResetPolicy: earliest\n# This setting is passed to the HPA definition that KEDA will create for a\n# given resource and holds the maximum number of replicas of the target resouce.\n# https://keda.sh/docs/2.9/concepts/scaling-deployments/#maxreplicacount\nmaxReplicas: 1\n# Minimum number of replicas KEDA will scale the resource down to.\n# https://keda.sh/docs/2.7/concepts/scaling-deployments/#minreplicacount\nminReplicas: 0\n# If this property is set, KEDA will scale the resource down to this\n# number of replicas.\n# https://keda.sh/docs/2.9/concepts/scaling-deployments/#idlereplicacount\nidleReplicas: 0\ntopics: # List of auto-generated Kafka Streams topics used by the streams app.\n- topic1\n- topic2\n</code></pre>"},{"location":"user/core-concepts/defaults/#producerapp","title":"ProducerApp","text":"<code>defaults.yaml</code> <pre><code>\n</code></pre>"},{"location":"user/core-concepts/defaults/#kafkaconnector","title":"KafkaConnector","text":"<code>defaults.yaml</code> <pre><code># Kafka connector\n#\n# Parent of: KafkaSinkConnector, KafkaSourceConnector\n# Child of: PipelineComponent\nkafka-connector:\n# Pipeline prefix that will prefix every component name. If you wish to not\n# have any prefix you can specify an empty string.\nprefix: ${pipeline_name}-\nfrom: # Must not be null\ntopics: # read from topic\n${pipeline_name}-input-topic:\ntype: input # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra\n${pipeline_name}-input-pattern-topic:\ntype: pattern # Implied to be an input pattern if `role` is undefined\n${pipeline_name}-extra-pattern-topic:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\ncomponents: # read from specific component\naccount-producer:\ntype: output # Implied when role is NOT specified\nother-producer:\nrole: some-role # Implies `type` to be extra\ncomponent-as-input-pattern:\ntype: pattern # Implied to be an input pattern if `role` is undefined\ncomponent-as-extra-pattern:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\n# Topic(s) into which the component will write output\nto:\ntopics:\n${pipeline_name}-output-topic:\ntype: output # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined\n${pipeline_name}-error-topic:\ntype: error\n# Currently KPOps supports Avro and JSON schemas.\nkey_schema: key-schema # must implement SchemaProvider to use\nvalue_schema: value-schema\npartitions_count: 1\nreplication_factor: 1\nconfigs: # https://kafka.apache.org/documentation/#topicconfigs\ncleanup.policy: compact\nmodels: # SchemaProvider is initiated with the values given here\nmodel: model\nnamespace: namespace # required\n# `app` contains application-specific settings, hence it does not have a rigid\n# structure. The fields below are just an example. Extensive documentation on\n# connectors: https://kafka.apache.org/documentation/#connectconfigs\napp: # required\ntasks.max: 1\n# Helm repository configuration for resetter\nrepo_config:\nrepository_name: my-repo # required\nurl: https://bakdata.github.io/kafka-connect-resetter/ # required\nrepo_auth_flags:\nusername: user\npassword: pass\nca_file: /home/user/path/to/ca-file\ninsecure_skip_tls_verify: false\nversion: \"1.0.6\" # Helm chart version\n# Overriding Kafka Connect Resetter Helm values. E.g. to override the\n# Image Tag etc.\nresetter_values:\nimageTag: \"1.2.3\"\n</code></pre>"},{"location":"user/core-concepts/defaults/#kafkasourceconnector","title":"KafkaSourceConnector","text":"<code>defaults.yaml</code> <pre><code># Kafka source connector\n#\n# Child of: KafkaConnector\nkafka-source-connector:\n# The source connector has no `from` section\n# from:\n# offset.storage.topic\n# https://kafka.apache.org/documentation/#connect_running\noffset_topic: offset_topic\n</code></pre>"},{"location":"user/core-concepts/defaults/#kafkasinkconnector","title":"KafkaSinkConnector","text":"<code>defaults.yaml</code> <pre><code># Kafka sink connector\n#\n# Child of: KafkaConnector\nkafka-sink-connector:\n# No settings differ from `kafka-connector`\n</code></pre>"},{"location":"user/core-concepts/components/kafka-app/","title":"KafkaApp","text":"<p>Subclass of KubernetesApp.</p>"},{"location":"user/core-concepts/components/kafka-app/#usage","title":"Usage","text":"<ul> <li>Defines a streams-bootstrap component</li> <li>Should not be used in <code>pipeline.yaml</code> as the component can be defined as either a StreamsApp or a ProducerApp</li> <li>Often used in <code>defaults.yaml</code></li> </ul>"},{"location":"user/core-concepts/components/kafka-app/#configuration","title":"Configuration","text":"<code>pipeline.yaml</code> <pre><code># Base component for Kafka-based components.\n# Producer or streaming apps should inherit from this class.\n- type: kafka-app # required\nname: kafka-app # required\n# Pipeline prefix that will prefix every component name. If you wish to not\n# have any prefix you can specify an empty string.\nprefix: ${pipeline_name}-\nfrom: # Must not be null\ntopics: # read from topic\n${pipeline_name}-input-topic:\ntype: input # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra\n${pipeline_name}-input-pattern-topic:\ntype: pattern # Implied to be an input pattern if `role` is undefined\n${pipeline_name}-extra-pattern-topic:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\ncomponents: # read from specific component\naccount-producer:\ntype: output # Implied when role is NOT specified\nother-producer:\nrole: some-role # Implies `type` to be extra\ncomponent-as-input-pattern:\ntype: pattern # Implied to be an input pattern if `role` is undefined\ncomponent-as-extra-pattern:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\n# Topic(s) into which the component will write output\nto:\ntopics:\n${pipeline_name}-output-topic:\ntype: output # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined\n${pipeline_name}-error-topic:\ntype: error\n# Currently KPOps supports Avro and JSON schemas.\nkey_schema: key-schema # must implement SchemaProvider to use\nvalue_schema: value-schema\npartitions_count: 1\nreplication_factor: 1\nconfigs: # https://kafka.apache.org/documentation/#topicconfigs\ncleanup.policy: compact\nmodels: # SchemaProvider is initiated with the values given here\nmodel: model\nnamespace: namespace # required\n# `app` can contain application-specific settings, hence  the user is free to\n# add the key-value pairs they need.\napp: # required\nstreams: # required\nbrokers: ${kafka_brokers} # required\nschemaRegistryUrl: ${schema_registry_url}\nnameOverride: override-with-this-name # kafka-app-specific\nimageTag: \"1.0.0\" # Example values that are shared between streams-app and producer-app\n# Helm repository configuration (optional)\n# If not set the helm repo add will not be called. Useful when using local Helm charts\nrepo_config:\nrepository_name: bakdata-streams-bootstrap # required\nurl: https://bakdata.github.io/streams-bootstrap/ # required\nrepo_auth_flags:\nusername: user\npassword: pass\nca_file: /home/user/path/to/ca-file\ninsecure_skip_tls_verify: false\nversion: \"2.12.0\" # Helm chart version\n</code></pre>"},{"location":"user/core-concepts/components/kafka-app/#operations","title":"Operations","text":""},{"location":"user/core-concepts/components/kafka-app/#deploy","title":"deploy","text":"<p>In addition to KubernetesApp's <code>deploy</code>:</p> <ul> <li>Create topics if provided (optional)</li> <li>Submit Avro schemas to the registry if provided (optional)</li> </ul>"},{"location":"user/core-concepts/components/kafka-app/#destroy","title":"destroy","text":"<p>Uninstall Helm release.</p>"},{"location":"user/core-concepts/components/kafka-app/#reset","title":"reset","text":"<p>Do nothing.</p>"},{"location":"user/core-concepts/components/kafka-app/#clean","title":"clean","text":"<p>Do nothing.</p>"},{"location":"user/core-concepts/components/kafka-connector/","title":"KafkaConnector","text":"<p><code>KafkaConnector</code> is a component that deploys Kafka Connectors. Since a connector cannot be different from sink or source it is not recommended to use <code>KafkaConnector</code> for deployment in <code>pipeline.yaml</code>. Instead, <code>KafkaConnector</code> should be used in <code>defaults.yaml</code> to set defaults for all connectors in the pipeline as they can share some common settings.</p>"},{"location":"user/core-concepts/components/kafka-sink-connector/","title":"KafkaSinkConnector","text":"<p>Subclass of KafkaConnector.</p>"},{"location":"user/core-concepts/components/kafka-sink-connector/#usage","title":"Usage","text":"<p>Lets other systems pull data from Apache Kafka.</p>"},{"location":"user/core-concepts/components/kafka-sink-connector/#configuration","title":"Configuration","text":"<code>pipeline.yaml</code> <pre><code># Kafka sink connector\n- type: kafka-sink-connector\nname: kafka-sink-connector # required\n# Pipeline prefix that will prefix every component name. If you wish to not\n# have any prefix you can specify an empty string.\nprefix: ${pipeline_name}-\nfrom: # Must not be null\ntopics: # read from topic\n${pipeline_name}-input-topic:\ntype: input # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra\n${pipeline_name}-input-pattern-topic:\ntype: pattern # Implied to be an input pattern if `role` is undefined\n${pipeline_name}-extra-pattern-topic:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\ncomponents: # read from specific component\naccount-producer:\ntype: output # Implied when role is NOT specified\nother-producer:\nrole: some-role # Implies `type` to be extra\ncomponent-as-input-pattern:\ntype: pattern # Implied to be an input pattern if `role` is undefined\ncomponent-as-extra-pattern:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\n# Topic(s) into which the component will write output\nto:\ntopics:\n${pipeline_name}-output-topic:\ntype: output # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined\n${pipeline_name}-error-topic:\ntype: error\n# Currently KPOps supports Avro and JSON schemas.\nkey_schema: key-schema # must implement SchemaProvider to use\nvalue_schema: value-schema\npartitions_count: 1\nreplication_factor: 1\nconfigs: # https://kafka.apache.org/documentation/#topicconfigs\ncleanup.policy: compact\nmodels: # SchemaProvider is initiated with the values given here\nmodel: model\nnamespace: namespace # required\n# `app` contains application-specific settings, hence it does not have a rigid\n# structure. The fields below are just an example. Extensive documentation on\n# connectors: https://kafka.apache.org/documentation/#connectconfigs\napp: # required\ntasks.max: 1\n# Helm repository configuration for resetter\nrepo_config:\nrepository_name: my-repo # required\nurl: https://bakdata.github.io/kafka-connect-resetter/ # required\nrepo_auth_flags:\nusername: user\npassword: pass\nca_file: /home/user/path/to/ca-file\ninsecure_skip_tls_verify: false\nversion: \"1.0.6\" # Helm chart version\n# Overriding Kafka Connect Resetter Helm values. E.g. to override the\n# Image Tag etc.\nresetter_values:\nimageTag: \"1.2.3\"\n</code></pre>"},{"location":"user/core-concepts/components/kafka-sink-connector/#operations","title":"Operations","text":""},{"location":"user/core-concepts/components/kafka-sink-connector/#deploy","title":"deploy","text":"<ul> <li>Add the sink connector to the Kafka Connect cluster</li> <li>Create the output topics if provided (optional)</li> <li>Register schemas in the Schema Registry if provided (optional)</li> </ul>"},{"location":"user/core-concepts/components/kafka-sink-connector/#destroy","title":"destroy","text":"<p>The associated sink connector is removed from the Kafka Connect cluster.</p>"},{"location":"user/core-concepts/components/kafka-sink-connector/#reset","title":"reset","text":"<p>Reset the consumer group offsets using bakdata's sink resetter.</p>"},{"location":"user/core-concepts/components/kafka-sink-connector/#clean","title":"clean","text":"<ul> <li>Delete associated consumer group</li> <li>Delete configured error topics</li> </ul>"},{"location":"user/core-concepts/components/kafka-source-connector/","title":"KafkaSourceConnector","text":"<p>Subclass of KafkaConnector.</p>"},{"location":"user/core-concepts/components/kafka-source-connector/#usage","title":"Usage","text":"<p>Manages source connectors in your Kafka Connect cluster.</p>"},{"location":"user/core-concepts/components/kafka-source-connector/#configuration","title":"Configuration","text":"<code>pipeline.yaml</code> <pre><code># Kafka source connector\n- type: kafka-source-connector # required\nname: kafka-source-connector # required\n# Pipeline prefix that will prefix every component name. If you wish to not\n# have any prefix you can specify an empty string.\nprefix: ${pipeline_name}-\n# The source connector has no `from` section\n# from:\n# Topic(s) into which the component will write output\nto:\ntopics:\n${pipeline_name}-output-topic:\ntype: output # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined\n${pipeline_name}-error-topic:\ntype: error\n# Currently KPOps supports Avro and JSON schemas.\nkey_schema: key-schema # must implement SchemaProvider to use\nvalue_schema: value-schema\npartitions_count: 1\nreplication_factor: 1\nconfigs: # https://kafka.apache.org/documentation/#topicconfigs\ncleanup.policy: compact\nmodels: # SchemaProvider is initiated with the values given here\nmodel: model\nnamespace: namespace # required\n# `app` contains application-specific settings, hence it does not have a rigid\n# structure. The fields below are just an example. Extensive documentation on\n# connectors: https://kafka.apache.org/documentation/#connectconfigs\napp: # required\ntasks.max: 1\n# Helm repository configuration for resetter\nrepo_config:\nrepository_name: my-repo # required\nurl: https://bakdata.github.io/kafka-connect-resetter/ # required\nrepo_auth_flags:\nusername: user\npassword: pass\nca_file: /home/user/path/to/ca-file\ninsecure_skip_tls_verify: false\nversion: \"1.0.6\" # Helm chart version\n# Overriding Kafka Connect Resetter Helm values. E.g. to override the\n# Image Tag etc.\nresetter_values:\nimageTag: \"1.2.3\"\n# offset.storage.topic\n# https://kafka.apache.org/documentation/#connect_running\noffset_topic: offset_topic\n</code></pre>"},{"location":"user/core-concepts/components/kafka-source-connector/#operations","title":"Operations","text":""},{"location":"user/core-concepts/components/kafka-source-connector/#deploy","title":"deploy","text":"<ul> <li>Add the source connector to the Kafka Connect cluster</li> <li>Create the output topics if provided (optional)</li> <li>Register schemas in the Schema registry if provided (optional)</li> </ul>"},{"location":"user/core-concepts/components/kafka-source-connector/#destroy","title":"destroy","text":"<p>Remove the source connector from the Kafka Connect cluster.</p>"},{"location":"user/core-concepts/components/kafka-source-connector/#reset","title":"reset","text":"<p>Delete state associated with the connector using bakdata's sink resetter.</p>"},{"location":"user/core-concepts/components/kafka-source-connector/#clean","title":"clean","text":"<ul> <li>Delete all associated output topics</li> <li>Delete all associated schemas in the Schema Registry</li> <li>Delete state associated with the connector</li> </ul>"},{"location":"user/core-concepts/components/kubernetes-app/","title":"KubernetesApp","text":""},{"location":"user/core-concepts/components/kubernetes-app/#usage","title":"Usage","text":"<p>Can be used to deploy any app in Kubernetes using Helm, for example, a REST service that serves Kafka data.</p>"},{"location":"user/core-concepts/components/kubernetes-app/#configuration","title":"Configuration","text":"<code>pipeline.yaml</code> <pre><code># Base Kubernetes App\n- type: kubernetes-app\nname: kubernetes-app # required\n# Pipeline prefix that will prefix every component name. If you wish to not\n# have any prefix you can specify an empty string.\nprefix: ${pipeline_name}-\nfrom: # Must not be null\ntopics: # read from topic\n${pipeline_name}-input-topic:\ntype: input # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra\n${pipeline_name}-input-pattern-topic:\ntype: pattern # Implied to be an input pattern if `role` is undefined\n${pipeline_name}-extra-pattern-topic:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\ncomponents: # read from specific component\naccount-producer:\ntype: output # Implied when role is NOT specified\nother-producer:\nrole: some-role # Implies `type` to be extra\ncomponent-as-input-pattern:\ntype: pattern # Implied to be an input pattern if `role` is undefined\ncomponent-as-extra-pattern:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\n# Topic(s) into which the component will write output\nto:\ntopics:\n${pipeline_name}-output-topic:\ntype: output # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined\n${pipeline_name}-error-topic:\ntype: error\n# Currently KPOps supports Avro and JSON schemas.\nkey_schema: key-schema # must implement SchemaProvider to use\nvalue_schema: value-schema\npartitions_count: 1\nreplication_factor: 1\nconfigs: # https://kafka.apache.org/documentation/#topicconfigs\ncleanup.policy: compact\nmodels: # SchemaProvider is initiated with the values given here\nmodel: model\nnamespace: namespace # required\n# `app` contains application-specific settings, hence it does not have a rigid\n# structure. The fields below are just an example.\napp: # required\nimage: exampleImage # Example\ndebug: false # Example\ncommandLine: {} # Example\n# Helm repository configuration (optional)\n# If not set the helm repo add will not be called. Useful when using local Helm charts\nrepo_config:\nrepository_name: bakdata-streams-bootstrap # required\nurl: https://bakdata.github.io/streams-bootstrap/ # required\nrepo_auth_flags:\nusername: user\npassword: pass\nca_file: /home/user/path/to/ca-file\ninsecure_skip_tls_verify: false\nversion: \"1.0.0\" # Helm chart version\n</code></pre>"},{"location":"user/core-concepts/components/kubernetes-app/#operations","title":"Operations","text":""},{"location":"user/core-concepts/components/kubernetes-app/#deploy","title":"deploy","text":"<p>Deploy using Helm.</p>"},{"location":"user/core-concepts/components/kubernetes-app/#destroy","title":"destroy","text":"<p>Uninstall Helm release.</p>"},{"location":"user/core-concepts/components/kubernetes-app/#reset","title":"reset","text":"<p>Do nothing.</p>"},{"location":"user/core-concepts/components/kubernetes-app/#clean","title":"clean","text":"<p>Do nothing.</p>"},{"location":"user/core-concepts/components/overview/","title":"Overview","text":"<p>This section explains the different components of KPOps, their usage and configuration in the pipeline definition <code>pipeline.yaml</code>.</p> <pre><code>flowchart BT\n    KubernetesApp --&gt; PipelineComponent\n    KafkaConnector --&gt; PipelineComponent\n    KafkaApp --&gt; KubernetesApp\n    StreamsApp --&gt; KafkaApp\n    ProducerApp --&gt; KafkaApp\n    KafkaSourceConnector --&gt; KafkaConnector\n    KafkaSinkConnector --&gt; KafkaConnector\n    \n    click KubernetesApp \"../kubernetes-app\"\n    click KafkaApp \"../kafka-app\"\n    click StreamsApp \"../streams-app\"\n    click ProducerApp \"../producer-app\"\n    click KafkaConnector \"../kafka-connector\"\n    click KafkaSourceConnector \"../kafka-source-connector\"\n    click KafkaSinkConnector \"../kafka-sink-connector\"</code></pre> <p>KPOps component hierarchy</p>"},{"location":"user/core-concepts/components/producer-app/","title":"ProducerApp","text":"<p>Subclass of KafkaApp.</p>"},{"location":"user/core-concepts/components/producer-app/#usage","title":"Usage","text":"<p>Configures a streams-bootstrap Kafka producer app</p>"},{"location":"user/core-concepts/components/producer-app/#configuration","title":"Configuration","text":"<code>pipeline.yaml</code> <pre><code># Holds configuration to use as values for the streams bootstrap producer-app Helm\n# chart.\n# More documentation on ProducerApp:\n# https://github.com/bakdata/streams-bootstrap\n- type: producer-app\nname: producer-app # required\n# Pipeline prefix that will prefix every component name. If you wish to not\n# have any prefix you can specify an empty string.\nprefix: ${pipeline_name}-\n# from: # While the producer-app does inherit from kafka-app, it does not need a\n# `from` section, hence it does not support it.\n# Topic(s) into which the component will write output\nto:\ntopics:\n${pipeline_name}-output-topic:\ntype: output # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined\n${pipeline_name}-error-topic:\ntype: error\n# Currently KPOps supports Avro and JSON schemas.\nkey_schema: key-schema # must implement SchemaProvider to use\nvalue_schema: value-schema\npartitions_count: 1\nreplication_factor: 1\nconfigs: # https://kafka.apache.org/documentation/#topicconfigs\ncleanup.policy: compact\nmodels: # SchemaProvider is initiated with the values given here\nmodel: model\nnamespace: namespace # required\n# Allowed configs:\n# https://github.com/bakdata/streams-bootstrap/tree/master/charts/producer-app\napp: # required\nstreams: # required, producer-app-specific\nbrokers: ${kafka_brokers} # required\nschemaRegistryUrl: ${schema_registry_url}\noutputTopic: output_topic\nextraOutputTopics:\noutput_role1: output_topic1\noutput_role2: output_topic2\nnameOverride: override-with-this-name # kafka-app-specific\n# Helm repository configuration (optional)\n# If not set the helm repo add will not be called. Useful when using local Helm charts\nrepo_config:\nrepository_name: bakdata-streams-bootstrap # required\nurl: https://bakdata.github.io/streams-bootstrap/ # required\nrepo_auth_flags:\nusername: user\npassword: pass\nca_file: /home/user/path/to/ca-file\ninsecure_skip_tls_verify: false\nversion: \"2.12.0\" # Helm chart version\n</code></pre>"},{"location":"user/core-concepts/components/producer-app/#operations","title":"Operations","text":""},{"location":"user/core-concepts/components/producer-app/#deploy","title":"deploy","text":"<p>In addition to KubernetesApp's <code>deploy</code>:</p> <ul> <li>Create topics if provided (optional)</li> <li>Submit Avro schemas to the registry if provided (optional)</li> </ul>"},{"location":"user/core-concepts/components/producer-app/#destroy","title":"destroy","text":"<p>Uninstall Helm release.</p>"},{"location":"user/core-concepts/components/producer-app/#reset","title":"reset","text":"<p>Do nothing, producers are stateless.</p>"},{"location":"user/core-concepts/components/producer-app/#clean","title":"clean","text":"<ul> <li>Delete the output topics of the Kafka producer</li> <li>Delete all associated schemas in the Schema Registry</li> </ul>"},{"location":"user/core-concepts/components/streams-app/","title":"StreamsApp","text":"<p>Subclass of KafkaApp.</p>"},{"location":"user/core-concepts/components/streams-app/#usage","title":"Usage","text":"<p>Configures a streams-bootstrap Kafka Streams app</p>"},{"location":"user/core-concepts/components/streams-app/#configuration","title":"Configuration","text":"<code>pipeline.yaml</code> <pre><code># StreamsApp component that configures a streams bootstrap app.\n# More documentation on StreamsApp: https://github.com/bakdata/streams-bootstrap\n- type: streams-app # required\nname: streams-app # required\n# Pipeline prefix that will prefix every component name. If you wish to not\n# have any prefix you can specify an empty string.\nprefix: ${pipeline_name}-\nfrom: # Must not be null\ntopics: # read from topic\n${pipeline_name}-input-topic:\ntype: input # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra\n${pipeline_name}-input-pattern-topic:\ntype: pattern # Implied to be an input pattern if `role` is undefined\n${pipeline_name}-extra-pattern-topic:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\ncomponents: # read from specific component\naccount-producer:\ntype: output # Implied when role is NOT specified\nother-producer:\nrole: some-role # Implies `type` to be extra\ncomponent-as-input-pattern:\ntype: pattern # Implied to be an input pattern if `role` is undefined\ncomponent-as-extra-pattern:\ntype: pattern # Implied to be an extra pattern if `role` is defined\nrole: some-role\n# Topic(s) into which the component will write output\nto:\ntopics:\n${pipeline_name}-output-topic:\ntype: output # Implied when role is NOT specified\n${pipeline_name}-extra-topic:\nrole: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined\n${pipeline_name}-error-topic:\ntype: error\n# Currently KPOps supports Avro and JSON schemas.\nkey_schema: key-schema # must implement SchemaProvider to use\nvalue_schema: value-schema\npartitions_count: 1\nreplication_factor: 1\nconfigs: # https://kafka.apache.org/documentation/#topicconfigs\ncleanup.policy: compact\nmodels: # SchemaProvider is initiated with the values given here\nmodel: model\nnamespace: namespace # required\n# No arbitrary keys are allowed under `app`here\n# Allowed configs:\n# https://github.com/bakdata/streams-bootstrap/tree/master/charts/streams-app\napp: # required\n# Streams Bootstrap streams section\nstreams: # required, streams-app-specific\nbrokers: ${kafka_brokers} # required\nschemaRegistryUrl: ${schema_registry_url}\ninputTopics:\n- topic1\n- topic2\noutputTopic: output-topic\ninputPattern: input-pattern\nextraInputTopics:\ninput_role1:\n- input_topic1\n- input_topic2\ninput_role2:\n- input_topic3\n- input_topic4\nextraInputPatterns:\npattern_role1: input_pattern1\nextraOutputTopics:\noutput_role1: output_topic1\noutput_role2: output_topic2\nerrorTopic: error-topic\nconfig:\nmy.streams.config: my.value\nnameOverride: override-with-this-name # streams-app-specific\nautoscaling: # streams-app-specific\nconsumerGroup: consumer-group # required\nlagThreshold: 0 # Average target value to trigger scaling actions.\nenabled: false # Whether to enable auto-scaling using KEDA.\n# This is the interval to check each trigger on.\n# https://keda.sh/docs/2.9/concepts/scaling-deployments/#pollinginterval\npollingInterval: 30\n# The period to wait after the last trigger reported active before scaling\n#  the resource back to 0. https://keda.sh/docs/2.9/concepts/scaling-deployments/#cooldownperiod\ncooldownPeriod: 300\n# The offset reset policy for the consumer if the the consumer group is\n# not yet subscribed to a partition.\noffsetResetPolicy: earliest\n# This setting is passed to the HPA definition that KEDA will create for a\n# given resource and holds the maximum number of replicas of the target resouce.\n# https://keda.sh/docs/2.9/concepts/scaling-deployments/#maxreplicacount\nmaxReplicas: 1\n# Minimum number of replicas KEDA will scale the resource down to.\n# https://keda.sh/docs/2.7/concepts/scaling-deployments/#minreplicacount\nminReplicas: 0\n# If this property is set, KEDA will scale the resource down to this\n# number of replicas.\n# https://keda.sh/docs/2.9/concepts/scaling-deployments/#idlereplicacount\nidleReplicas: 0\ntopics: # List of auto-generated Kafka Streams topics used by the streams app.\n- topic1\n- topic2\n# Helm repository configuration (optional)\n# If not set the helm repo add will not be called. Useful when using local Helm charts\nrepo_config:\nrepository_name: bakdata-streams-bootstrap # required\nurl: https://bakdata.github.io/streams-bootstrap/ # required\nrepo_auth_flags:\nusername: user\npassword: pass\nca_file: /home/user/path/to/ca-file\ninsecure_skip_tls_verify: false\nversion: \"2.12.0\" # Helm chart version\n</code></pre>"},{"location":"user/core-concepts/components/streams-app/#operations","title":"Operations","text":""},{"location":"user/core-concepts/components/streams-app/#deploy","title":"deploy","text":"<p>In addition to KubernetesApp's <code>deploy</code>:</p> <ul> <li>Create topics if provided (optional)</li> <li>Submit Avro schemas to the registry if provided (optional)</li> </ul>"},{"location":"user/core-concepts/components/streams-app/#destroy","title":"destroy","text":"<p>Uninstall Helm release.</p>"},{"location":"user/core-concepts/components/streams-app/#reset","title":"reset","text":"<ul> <li>Delete the consumer group offsets</li> <li>Delete Kafka Streams state</li> </ul>"},{"location":"user/core-concepts/components/streams-app/#clean","title":"clean","text":"<p>Similar to <code>reset</code> with to additional steps:</p> <ul> <li>Delete the app's output topics</li> <li>Delete all associated schemas in the Schema Registry</li> </ul>"},{"location":"user/core-concepts/variables/environment_variables/","title":"Environment variables","text":"<p>Environment variables can be set by using the export command in Linux or the set command in Windows.</p> <p>dotenv files</p> <p>Support for <code>.env</code> files is on the roadmap,  but not implemented in KPOps yet. One of the possible ways to still  use one and export the contents manually is with the following command: <code>export $(xargs &lt; .env)</code>. This would work in <code>bash</code> suppose there are no spaces inside the values.</p>"},{"location":"user/core-concepts/variables/environment_variables/#config","title":"Config","text":"<p>These variables are a lower priority alternative to the settings in <code>config.yaml</code>. Variables marked as required can instead be set in the pipeline config.</p> Name Default Value Required Description Setting name KPOPS_ENVIRONMENT True The environment you want to generate and deploy the pipeline to. Suffix your environment files with this value (e.g. defaults_development.yaml for environment=development). environment KPOPS_KAFKA_BROKERS True The comma separated Kafka brokers address. kafka_brokers KPOPS_SCHEMA_REGISTRY_URL http://localhost:8081 False Address of the Schema Registry. url KPOPS_KAFKA_REST_URL http://localhost:8082 False Address of the Kafka REST Proxy. url KPOPS_KAFKA_CONNECT_URL http://localhost:8083 False Address of Kafka Connect. url KPOPS_TIMEOUT 300 False The timeout in seconds that specifies when actions like deletion or deploy timeout. timeout KPOPS_RETAIN_CLEAN_JOBS False False Whether to retain clean up jobs in the cluster or uninstall the, after completion. retain_clean_jobs config_env_vars.env Exhaustive list of all config-related environment variables<pre><code># Pipeline config environment variables\n#\n# The default setup is shown. These variables are a lower priority\n# alternative to the settings in `config.yaml`. Variables marked as\n# required can instead be set in the pipeline config.\n#\n# environment\n# The environment you want to generate and deploy the pipeline to.\n# Suffix your environment files with this value (e.g.\n# defaults_development.yaml for environment=development).\nKPOPS_ENVIRONMENT # No default value, required\n# kafka_brokers\n# The comma separated Kafka brokers address.\nKPOPS_KAFKA_BROKERS # No default value, required\n# url\n# Address of the Schema Registry.\nKPOPS_SCHEMA_REGISTRY_URL=http://localhost:8081\n# url\n# Address of the Kafka REST Proxy.\nKPOPS_KAFKA_REST_URL=http://localhost:8082\n# url\n# Address of Kafka Connect.\nKPOPS_KAFKA_CONNECT_URL=http://localhost:8083\n# timeout\n# The timeout in seconds that specifies when actions like deletion or\n# deploy timeout.\nKPOPS_TIMEOUT=300\n# retain_clean_jobs\n# Whether to retain clean up jobs in the cluster or uninstall the,\n# after completion.\nKPOPS_RETAIN_CLEAN_JOBS=False\n</code></pre>"},{"location":"user/core-concepts/variables/environment_variables/#cli","title":"CLI","text":"<p>These variables are a lower priority alternative to the commands' flags. If a variable is set, the corresponding flag does not have to be specified in commands. Variables marked as required can instead be set as flags.</p> Name Default Value Required Description KPOPS_PIPELINE_BASE_DIR . False Base directory to the pipelines (default is current working directory) KPOPS_CONFIG_PATH config.yaml False Path to the config.yaml file KPOPS_DEFAULT_PATH False Path to defaults folder KPOPS_PIPELINE_PATH True Path to YAML with pipeline definition KPOPS_PIPELINE_STEPS False Comma separated list of steps to apply the command on cli_env_vars.env Exhaustive list of all cli-related environment variables<pre><code># CLI Environment variables\n#\n# The default setup is shown. These variables are a lower priority\n# alternative to the commands' flags. If a variable is set, the\n# corresponding flag does not have to be specified in commands.\n# Variables marked as required can instead be set as flags.\n#\n# Base directory to the pipelines (default is current working\n# directory)\nKPOPS_PIPELINE_BASE_DIR=.\n# Path to the config.yaml file\nKPOPS_CONFIG_PATH=config.yaml\n# Path to defaults folder\nKPOPS_DEFAULT_PATH # No default value, not required\n# Path to YAML with pipeline definition\nKPOPS_PIPELINE_PATH # No default value, required\n# Comma separated list of steps to apply the command on\nKPOPS_PIPELINE_STEPS # No default value, not required\n</code></pre>"},{"location":"user/core-concepts/variables/substitution/","title":"Substitution","text":"<p>KPOps supports the usage of placeholders and environment variables in pipeline definition and defaults.</p>"},{"location":"user/core-concepts/variables/substitution/#component-specific-variables","title":"Component-specific variables","text":"<p>These variables can be used in a component's definition to refer to any of its attributes, including ones that the user has defined in the defaults.</p> <p>All of them are prefixed with <code>component_</code> and follow the following form: <code>component_{attribute_name}</code>. If the attribute itself contains attributes, they can be referred to like this: <code>component_{attribute_name}_{subattribute_name}</code>.</p> Example <pre><code>- type: scheduled-producer\napp:\nlabels:\napp_type: \"${component_type}\"\napp_name: \"${component_name}\"\napp_schedule: \"${component_app_schedule}\"\ncommandLine:\nFAKE_ARG: \"fake-arg-value\"\nschedule: \"30 3/8 * * *\"\n- type: converter\napp:\ncommandLine:\nCONVERT_XML: true\nresources:\nlimits:\nmemory: 2G\nrequests:\nmemory: 2G\n- type: filter\nname: \"filter-app\"\napp:\nlabels:\napp_type: \"${component_type}\"\napp_name: \"${component_name}\"\napp_resources_requests_memory: \"${component_app_resources_requests_memory}\"\n${component_type}: \"${component_app_labels_app_name}-${component_app_labels_app_type}\"\ntest_placeholder_in_placeholder: \"${component_app_labels_${component_type}}\"\ncommandLine:\nTYPE: \"nothing\"\nresources:\nrequests:\nmemory: 3G\nreplicaCount: 4\nautoscaling:\nminReplicas: 4\nmaxReplicas: 4\n</code></pre>"},{"location":"user/core-concepts/variables/substitution/#pipeline-config-specific-variables","title":"Pipeline-config-specific variables","text":"<p>These variables include all fields in the config and refer to the pipeline configuration that is independent of the components.</p> <p>Info</p> <p><code>error_topic_name</code> is an alias for <code>topic_name_config_default_error_topic_name</code> <code>output_topic_name</code> is an alias for <code>topic_name_config_default_output_topic_name</code></p>"},{"location":"user/core-concepts/variables/substitution/#environment-variables","title":"Environment variables","text":"<p>Environment variables such as <code>$PATH</code> can be used in the pipeline definition and defaults without any transformation following the form <code>${ENV_VAR_NAME}</code>. This, of course, includes variables like the ones relevant to the KPOps cli that are exported by the user.</p> <p>See all KPOps environment variables</p>"},{"location":"user/core-concepts/variables/substitution/#pipeline-name-variables","title":"Pipeline name variables","text":"<p>These are special variables that refer to the name and path of a pipeline.</p> <ul> <li> <p><code>${pipeline_name}</code>: Concatenated path of the parent directory where pipeline.yaml is defined in.   For instance, <code>./data/pipelines/v1/pipeline.yaml</code>, here the value for the variable would be <code>data-pipelines-v1</code>.</p> </li> <li> <p><code>${pipeline_name_&lt;level&gt;}</code>: Similar to the previous variable, each <code>&lt;level&gt;</code> contains a part of the path to the <code>pipeline.yaml</code> file.   Consider the previous example, <code>${pipeline_name_0}</code> would be <code>data</code>, <code>${pipeline_name_1}</code> would be <code>pipelines</code>, and <code>${pipeline_name_2}</code> equals to <code>v1</code>.</p> </li> </ul>"},{"location":"user/core-concepts/variables/substitution/#advanced-use-cases","title":"Advanced use cases","text":"<ol> <li>Refer to default component field values: As long as a value is assigned to a component attribute, it is possible to refer to it with a placeholder. To see all component fields, take a look at the pipeline schema.</li> <li>Chaining variables: It is possible to chain any number of variables, see the example above.</li> <li>Cross-component substitution: YAML is quite an intricate language and with some of its magic one could write cross-component references.</li> </ol>"},{"location":"user/examples/atm-fraud-pipeline/","title":"ATM fraud detection pipeline","text":"<p>ATM fraud is a demo pipeline for ATM fraud detection. The original by Confluent is written in KSQL and outlined in this blogpost. The one used in this example is re-built from scratch using bakdata's <code>streams-bootstrap</code> library.</p>"},{"location":"user/examples/atm-fraud-pipeline/#what-this-will-demonstrate","title":"What this will demonstrate","text":"<ul> <li>Deploying a PostgreSQL database using Helm</li> <li>Deploying a pipeline using KPOps</li> <li>Destroying a pipeline using KPOps</li> </ul>"},{"location":"user/examples/atm-fraud-pipeline/#prerequisites","title":"Prerequisites","text":"<p>Completed all steps in the setup.</p>"},{"location":"user/examples/atm-fraud-pipeline/#setup-and-deployment","title":"Setup and deployment","text":""},{"location":"user/examples/atm-fraud-pipeline/#postgresql","title":"PostgreSQL","text":"<p>Deploy PostgreSQL using the Bitnami Helm chart: Add the helm repository:</p> <pre><code>helm repo add bitnami https://charts.bitnami.com/bitnami &amp;&amp; \\\nhelm repo update\n</code></pre> <p>Install the PostgreSQL with helm:</p> <pre><code>helm upgrade --install -f ./postgresql.yaml \\\n--namespace kpops \\\npostgresql bitnami/postgresql\n</code></pre> PostgreSQL Example Helm chart values (<code>postgresql.yaml</code>) <pre><code>auth:\ndatabase: app_db  enablePostgresUser: true\npassword: AppPassword\npostgresPassword: StrongPassword\nusername: app1\nprimary:\npersistence:\nenabled: false\nexistingClaim: postgresql-data-claim\nvolumePermissions:\nenabled: true\n</code></pre>"},{"location":"user/examples/atm-fraud-pipeline/#atm-fraud-detection-example-pipeline-setup","title":"ATM fraud detection example pipeline setup","text":""},{"location":"user/examples/atm-fraud-pipeline/#port-forwarding","title":"Port forwarding","text":"<p>Before we deploy the pipeline, we need to forward the ports of <code>kafka-rest-proxy</code> and <code>kafka-connect</code>. Run the following commands in two different terminals.</p> <pre><code>kubectl port-forward --namespace kpops service/k8kafka-cp-rest 8082:8082\n</code></pre> <pre><code>kubectl port-forward --namespace kpops service/k8kafka-cp-kafka-connect 8083:8083\n</code></pre>"},{"location":"user/examples/atm-fraud-pipeline/#deploying-the-atm-fraud-detection-pipeline","title":"Deploying the ATM fraud detection pipeline","text":"<ol> <li>Export environment variables in your terminal:</li> </ol> <pre><code>export DOCKER_REGISTRY=bakdata &amp;&amp; \\\nexport NAMESPACE=kpops\n</code></pre> <ol> <li>Deploy the pipeline</li> </ol> <pre><code>poetry run kpops deploy ./examples/bakdata/atm-fraud-detection/pipeline.yaml \\\n--pipeline-base-dir ./examples \\\n--config ./examples/bakdata/atm-fraud-detection/config.yaml \\\n--execute\n</code></pre> <p>Note</p> <p>You can use the <code>--dry-run</code> flag instead of the <code>--execute</code> flag and check the logs if your pipeline will be deployed correctly.</p>"},{"location":"user/examples/atm-fraud-pipeline/#check-if-the-deployment-is-successful","title":"Check if the deployment is successful","text":"<p>You can use the Streams Explorer to see the deployed pipeline. To do so, port-forward the service in a separate terminal session using the command below:</p> <pre><code>kubectl port-forward -n kpops service/streams-explorer 8080:8080\n</code></pre> <p>After that open http://localhost:8080 in your browser. You should be able to see pipeline shown in the image below:</p> <p> </p> An overview of ATM fraud pipeline shown in Streams Explorer <p>Attention</p> <p>Kafka Connect needs some time to set up the connector.  Moreover, Streams Explorer needs a while to scrape the information from Kafka connect. Therefore, it might take a bit until you see the whole graph.</p>"},{"location":"user/examples/atm-fraud-pipeline/#teardown-resources","title":"Teardown resources","text":""},{"location":"user/examples/atm-fraud-pipeline/#postrgresql","title":"PostrgreSQL","text":"<p>PostgreSQL can be uninstalled by running the following command:</p> <pre><code>helm --namespace kpops uninstall postgresql\n</code></pre>"},{"location":"user/examples/atm-fraud-pipeline/#atm-fraud-pipeline","title":"ATM fraud pipeline","text":"<ol> <li> <p>Export environment variables in your terminal.</p> <pre><code>export DOCKER_REGISTRY=bakdata &amp;&amp; \\\nexport NAMESPACE=kpops\n</code></pre> </li> <li> <p>Remove the pipeline</p> <pre><code>poetry run kpops clean ./examples/bakdata/atm-fraud-detection/pipeline.yaml \\\n--pipeline-base-dir ./examples \\\n--config ./examples/bakdata/atm-fraud-detection/config.yaml \\\n--verbose \\\n--execute\n</code></pre> </li> </ol> <p>Note</p> <p>You can use the <code>--dry-run</code> flag instead of the <code>--execute</code> flag and check the logs if your pipeline will be destroyed correctly.</p> <p>Attention</p> <p>If you face any issues destroying this example see Teardown for manual deletion.</p>"},{"location":"user/examples/atm-fraud-pipeline/#common-errors","title":"Common errors","text":"<ul> <li><code>deploy</code> fails:</li> <li>Read the error message.</li> <li>Try to correct the mistakes if there were any. Likely the configuration is not correct or the port-forwarding is not working as intended.</li> <li>Run <code>clean</code>.</li> <li>Run <code>deploy --dry-run</code> to avoid havig to <code>clean</code> again. If an error is dropped, start over from step 1.</li> <li>If the dry-run is succesful, run <code>deploy</code>.</li> <li><code>clean</code> fails:</li> <li>Read the error message.</li> <li>Try to correct the indicated mistakes if there were any. Likely the configuration is not correct or the port-forwarding is not working as intended.</li> <li>Run <code>clean</code>.</li> <li>If <code>clean</code> fails, follow the steps in teardown.</li> </ul>"},{"location":"user/getting-started/quick-start/","title":"Quick start","text":""},{"location":"user/getting-started/quick-start/#word-count","title":"Word-count","text":"<p>Word-count is a demo pipeline which consists of a producer producing words to Kafka, a Kafka streams app counting the number of times each word occurs and finally a Redis database into which the words are exported.</p>"},{"location":"user/getting-started/quick-start/#what-this-will-demonstrate","title":"What this will demonstrate","text":"<ul> <li>Deploying a Redis database using Helm</li> <li>Deploying a pipeline using KPOps</li> <li>Destroying a pipeline using KPOps</li> </ul>"},{"location":"user/getting-started/quick-start/#prerequisites","title":"Prerequisites","text":"<p>Completed all steps in the setup.</p>"},{"location":"user/getting-started/quick-start/#setup-and-deployment","title":"Setup and deployment","text":""},{"location":"user/getting-started/quick-start/#redis","title":"Redis","text":"<p>Deploy Redis using the Bitnami Helm chart: Add the Helm repository:</p> <pre><code>helm repo add bitnami https://charts.bitnami.com/bitnami &amp;&amp; \\\nhelm repo update\n</code></pre> <p>Install Redis with Helm:</p> <pre><code>helm upgrade --install -f ./values-redis.yaml \\\n--namespace kpops \\\nredis bitnami/redis\n</code></pre> Redis example Helm chart values (<code>values-redis.yaml</code>) <pre><code>architecture: standalone\nauth:\nenabled: false\nmaster:\ncount: 1\nconfiguration: \"databases 1\"\nimage:\ntag: 7.0.8\n</code></pre>"},{"location":"user/getting-started/quick-start/#word-count-example-pipeline-setup","title":"Word-count example pipeline setup","text":""},{"location":"user/getting-started/quick-start/#port-forwarding","title":"Port forwarding","text":"<p>Before we deploy the pipeline, we need to forward the ports of <code>kafka-rest-proxy</code> and <code>kafka-connect</code>. Run the following commands in two different terminals.</p> <pre><code>kubectl port-forward --namespace kpops service/k8kafka-cp-rest 8082:8082\n</code></pre> <pre><code>kubectl port-forward --namespace kpops service/k8kafka-cp-kafka-connect 8083:8083\n</code></pre>"},{"location":"user/getting-started/quick-start/#deploying-the-word-count-pipeline","title":"Deploying the Word-count pipeline","text":"<ol> <li>Copy the configuration from the kpops-examples repository into <code>kpops&gt;examples&gt;bakdata&gt;word-count</code> like so:</li> </ol> <pre><code>kpops\n\u251c\u2500\u2500 examples\n|   \u251c\u2500\u2500 bakdata\n|   |   \u251c\u2500\u2500 word-count\n|   |   |   \u251c\u2500\u2500 config.yaml\n|   |   |   \u251c\u2500\u2500 defaults\n|   |   |   \u2502\u00a0\u00a0 \u2514\u2500\u2500 defaults.yaml\n|   |   |   \u2514\u2500\u2500 pipeline.yaml\n|   |   |\n</code></pre> <ol> <li>Export environment variables in your terminal:</li> </ol> <pre><code>export DOCKER_REGISTRY=bakdata &amp;&amp; \\\nexport NAMESPACE=kpops\n</code></pre> <ol> <li>Deploy the pipeline</li> </ol> <pre><code>kpops deploy ./examples/bakdata/word-count/pipeline.yaml \\\n--pipeline-base-dir ./examples \\\n--config ./examples/bakdata/word-count/config.yaml \\\n--execute\n</code></pre> <p>Note</p> <p>You can use the <code>--dry-run</code> flag instead of the <code>--execute</code> flag and check the logs if your pipeline will be deployed correctly.</p>"},{"location":"user/getting-started/quick-start/#check-if-the-deployment-is-successful","title":"Check if the deployment is successful","text":"<p>You can use the Streams Explorer to inspect the deployed pipeline. To do so, port-forward the service in a separate terminal session using the command below:</p> <pre><code>kubectl port-forward -n kpops service/streams-explorer 8080:8080\n</code></pre> <p>After that open http://localhost:8080 in your browser.</p> <p>You should be able to see pipeline shown in the image below:</p> <p> </p> An overview of Word-count pipeline shown in Streams Explorer <p>Attention</p> <p>Kafka Connect needs some time to set up the connector. Moreover, Streams Explorer needs a while to scrape the information from Kafka Connect. Therefore, it might take a bit until you see the whole graph.</p>"},{"location":"user/getting-started/quick-start/#teardown-resources","title":"Teardown resources","text":""},{"location":"user/getting-started/quick-start/#redis_1","title":"Redis","text":"<p>Redis can be uninstalled by running the following command:</p> <pre><code>helm --namespace kpops uninstall redis\n</code></pre>"},{"location":"user/getting-started/quick-start/#word-count-pipeline","title":"Word-count pipeline","text":"<ol> <li>Export environment variables in your terminal.</li> </ol> <pre><code>export DOCKER_REGISTRY=bakdata &amp;&amp; \\\nexport NAMESPACE=kpops\n</code></pre> <ol> <li>Remove the pipeline</li> </ol> <pre><code>kpops clean ./examples/bakdata/word-count/pipeline.yaml \\\n--pipeline-base-dir ./examples \\\n--config ./examples/bakdata/word-count/config.yaml \\\n--verbose \\\n--execute\n</code></pre> <p>Note</p> <p>You can use the <code>--dry-run</code> flag instead of the <code>--execute</code> flag and check the logs if your pipeline will be destroyed correctly.</p> <p>Attention</p> <p>If you face any issues destroying this example see Teardown for manual deletion.</p>"},{"location":"user/getting-started/quick-start/#common-errors","title":"Common errors","text":"<ul> <li><code>deploy</code> fails:</li> <li>Read the error message.</li> <li>Try to correct the mistakes if there were any. Likely the configuration is not correct or the port-forwarding is not working as intended.</li> <li>Run <code>clean</code>.</li> <li>Run <code>deploy --dry-run</code> to avoid having to <code>clean</code> again. If an error is dropped, start over from step 1.</li> <li>If the dry-run is successful, run <code>deploy</code>.</li> <li><code>clean</code> fails:</li> <li>Read the error message.</li> <li>Try to correct the indicated mistakes if there were any. Likely the configuration is not correct or the port-forwarding is not working as intended.</li> <li>Run <code>clean</code>.</li> <li>If <code>clean</code> fails, follow the steps in teardown.</li> </ul>"},{"location":"user/getting-started/setup/","title":"Setup KPOps","text":"<p>In this part, you will set up KPOps. This includes:</p> <ul> <li>optionally creating a local Kubernetes cluster</li> <li>running Apache Kafka and Confluent's Schema Registry</li> <li>installing KPOps</li> </ul>"},{"location":"user/getting-started/setup/#prerequisites","title":"Prerequisites","text":"<ul> <li>k3d (Version 5.4.6+) and Docker (Version &gt;= v20.10.5) or an existing Kubernetes cluster (&gt;= 1.21.0)</li> <li>kubectl (Compatible with server version 1.21.0)</li> <li>Helm (Version 3.8.0+)</li> </ul>"},{"location":"user/getting-started/setup/#setup-kubernetes-with-k3d","title":"Setup Kubernetes with k3d","text":"<p>If you don't have access to an existing Kubernetes cluster, this section will guide you through creating a local cluster. We recommend the lightweight Kubernetes distribution k3s for this. k3d is a wrapper around k3s in Docker that lets you get started fast.</p> <ol> <li>You can install k3d with its installation script:</li> </ol> <pre><code>wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/v5.4.6/install.sh | bash\n</code></pre> <p>For other ways of installing k3d, you can have a look at their installation guide.</p> <ol> <li>The Kafka deployment needs a modified Docker image. In that case the image is built and pushed to a Docker registry that holds it. If you do not have access to an existing Docker registry, you can use k3d's Docker registry:</li> </ol> <pre><code>k3d registry create kpops-registry.localhost --port 12345\n</code></pre> <ol> <li>Now you can create a new cluster called <code>kpops</code> that uses the previously created Docker registry:</li> </ol> <pre><code>k3d cluster create kpops --k3s-arg \"--no-deploy=traefik@server:*\" --registry-use k3d-kpops-registry.localhost:12345\n</code></pre> <p>Note</p> <p>Creating a new k3d cluster automatically configures <code>kubectl</code> to connect to the local cluster by modifying your <code>~/.kube/config</code>. In case you manually set the <code>KUBECONFIG</code> variable or don't want k3d to modify your config, k3d offers many other options.</p> <p>You can check the cluster status with <code>kubectl get pods -n kube-system</code>. If all returned elements have a <code>STATUS</code> of <code>Running</code> or <code>Completed</code>, then the cluster is up and running.</p>"},{"location":"user/getting-started/setup/#deploy-kafka","title":"Deploy Kafka","text":"<p>Kafka is an open-source data streaming platform. More information about Kafka can be found in the documentation. To deploy Kafka, this guide uses Confluent's Helm chart.</p> <ol> <li> <p>To allow connectivity to other systems Kafka Connect needs to be extended with drivers. You can install a JDBC driver for Kafka Connect by creating a new Docker image:</p> </li> <li> <p>Create a <code>Dockerfile</code> with the following content:</p> <pre><code>FROM confluentinc/cp-kafka-connect:7.1.3\nRUN confluent-hub install --no-prompt confluentinc/kafka-connect-jdbc:10.6.0\n</code></pre> </li> <li> <p>Build and push the modified image to your private Docker registry:</p> <pre><code>docker build . --tag localhost:12345/kafka-connect-jdbc:7.1.3 &amp;&amp; \\\ndocker push localhost:12345/kafka-connect-jdbc:7.1.3\n</code></pre> </li> </ol> <p>Detailed instructions on building, tagging and pushing a docker image can be found in Docker docs.</p> <ol> <li>Add Confluent's Helm chart repository and update the index:</li> </ol> <pre><code>helm repo add confluentinc https://confluentinc.github.io/cp-helm-charts/ &amp;&amp;  helm repo update\n</code></pre> <ol> <li>Install Kafka, Zookeeper, Confluent's Schema Registry, Kafka Rest Proxy, and Kafka Connect. A single Helm chart installs all five components. Below you can find an example for the <code>--values ./kafka.yaml</code> file configuring the deployment accordingly. Deploy the services:</li> </ol> <pre><code>helm upgrade \\\n--install \\\n--version 0.6.1 \\\n--values ./kafka.yaml \\\n--namespace kpops \\\n--create-namespace \\\n--wait \\\nk8kafka confluentinc/cp-helm-charts\n</code></pre> Kafka Helm chart values (<code>kafka.yaml</code>) <p>An example value configuration for Confluent's Helm chart. This configuration deploys a single Kafka Broker, a Schema Registry, Zookeeper, Kafka Rest Proxy, and Kafka Connect with minimal resources.</p> <pre><code>cp-zookeeper:\nenabled: true\nservers: 1\nimageTag: 7.1.3\nheapOptions: \"-Xms124M -Xmx124M\"\noverrideGroupId: k8kafka\nfullnameOverride: \"k8kafka-cp-zookeeper\"\nresources:\nrequests:\ncpu: 50m\nmemory: 0.2G\nlimits:\ncpu: 250m\nmemory: 0.2G\nprometheus:\njmx:\nenabled: false\ncp-kafka:\nenabled: true\nbrokers: 1\nimageTag: 7.1.3\npodManagementPolicy: Parallel\nconfigurationOverrides:\n\"auto.create.topics.enable\": false\n\"offsets.topic.replication.factor\": 1\n\"transaction.state.log.replication.factor\": 1\n\"transaction.state.log.min.isr\": 1\n\"confluent.metrics.reporter.topic.replicas\": 1\nresources:\nrequests:\ncpu: 50m\nmemory: 0.5G\nlimits:\ncpu: 250m\nmemory: 0.5G\nprometheus:\njmx:\nenabled: false\npersistence:\nenabled: false\ncp-schema-registry:\nenabled: true\nimageTag: 7.1.3\nfullnameOverride: \"k8kafka-cp-schema-registry\"\noverrideGroupId: k8kafka\nkafka:\nbootstrapServers: \"PLAINTEXT://k8kafka-cp-kafka-headless:9092\"\nresources:\nrequests:\ncpu: 50m\nmemory: 0.25G\nlimits:\ncpu: 250m\nmemory: 0.25G\nprometheus:\njmx:\nenabled: false\ncp-kafka-connect:\nenabled: true\nreplicaCount: 1\nimage: k3d-kpops-registry.localhost:12345/kafka-connect-jdbc\nimageTag: 7.1.3\nfullnameOverride: \"k8kafka-cp-kafka-connect\"\noverrideGroupId: k8kafka\nkafka:\nbootstrapServers: \"PLAINTEXT://k8kafka-cp-kafka-headless:9092\"\nheapOptions: \"-Xms256M -Xmx256M\"\nresources:\nrequests:\ncpu: 500m\nmemory: 0.25G\nlimits:\ncpu: 500m\nmemory: 0.25G\nconfigurationOverrides:\n\"consumer.max.poll.records\": \"10\"\n\"consumer.max.poll.interval.ms\": \"900000\"\n\"config.storage.replication.factor\": \"1\"\n\"offset.storage.replication.factor\": \"1\"\n\"status.storage.replication.factor\": \"1\"\ncp-schema-registry:\nurl: http://k8kafka-cp-schema-registry:8081\nprometheus:\njmx:\nenabled: false\ncp-kafka-rest:\nenabled: true\nimageTag: 7.1.3\nfullnameOverride: \"k8kafka-cp-rest\"\nheapOptions: \"-Xms256M -Xmx256M\"\nresources:\nrequests:\ncpu: 50m\nmemory: 0.25G\nlimits:\ncpu: 250m\nmemory: 0.5G\nprometheus:\njmx:\nenabled: false\ncp-ksql-server:\nenabled: false\ncp-control-center:\nenabled: false\n</code></pre>"},{"location":"user/getting-started/setup/#deploy-streams-explorer","title":"Deploy Streams Explorer","text":"<p>Streams Explorer allows examining Apache Kafka data pipelines in a Kubernetes cluster including the inspection of schemas and monitoring of metrics. First, add the Helm repository:</p> <pre><code>helm repo add streams-explorer https://bakdata.github.io/streams-explorer &amp;&amp; \\\nhelm repo update\n</code></pre> <p>Below you can find an example for the <code>--values ./streams-explorer.yaml</code> file configuring the deployment accordingly. Now, deploy the service:</p> <pre><code>helm upgrade \\\n--install \\\n--version 0.2.3 \\\n--values ./streams-explorer.yaml \\\n--namespace kpops \\\nstreams-explorer streams-explorer/streams-explorer\n</code></pre> Streams Explorer Helm chart values (<code>streams-explorer.yaml</code>) <p>An example value configuration for Steams Explorer Helm chart.</p> <pre><code>imageTag: \"v2.1.2\"\nconfig:\nK8S__deployment__cluster: true\nSCHEMAREGISTRY__url: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081\nKAFKACONNECT__url: http://k8kafka-cp-kafka-connect.kpops.svc.cluster.local:8083\nresources:\nrequests:\ncpu: 200m\nmemory: 300Mi\nlimits:\ncpu: 200m\nmemory: 300Mi\n</code></pre>"},{"location":"user/getting-started/setup/#check-the-status-of-your-deployments","title":"Check the status of your deployments","text":"<p>Now we will check if all the pods are running in our namespace. You can list all pods in the namespace with this command:</p> <pre><code>kubectl --namespace kpops get pods\n</code></pre> <p>Then you should see the following output in your terminal:</p> <pre><code>NAME                                          READY   STATUS    RESTARTS   AGE\nk8kafka-cp-kafka-connect-8fc7d544f-8pjnt      1/1     Running   0          15m\nk8kafka-cp-zookeeper-0                        1/1     Running   0          15m\nk8kafka-cp-kafka-0                            1/1     Running   0          15m\nk8kafka-cp-schema-registry-588f8c65db-jdwbq   1/1     Running   0          15m\nk8kafka-cp-rest-6bbfd7b645-nwkf8              1/1     Running   0          15m\nstreams-explorer-54db878c67-s8wbz             1/1     Running   0          15m\n</code></pre> <p>Pay attention to the <code>STATUS</code> row. The pods should have a status of <code>Running</code>.</p>"},{"location":"user/getting-started/setup/#install-kpops","title":"Install KPOps","text":"<p>KPOps comes as a PyPI package. You can install it with <code>pip</code>:</p> <pre><code>pip install kpops\n</code></pre>"},{"location":"user/getting-started/teardown/","title":"Teardown resources","text":""},{"location":"user/getting-started/teardown/#kpops-teardown-commands","title":"KPOps teardown commands","text":"<ul> <li><code>destroy</code>: Removes Kubernetes resources.</li> <li><code>reset</code>: Runs <code>destroy</code>, resets the states of Kafka Streams apps and resets offsets to zero.</li> <li><code>clean</code>: Runs <code>reset</code> and removes all Kafka resources.</li> </ul>"},{"location":"user/getting-started/teardown/#kpops-deployed-pipeline","title":"KPOps-deployed pipeline","text":"<p>The <code>kpops</code> CLI can be used to destroy a pipeline that was previously deployed with KPOps. In case that doesn't work, the pipeline can always be taken down manually with <code>helm</code> (see section Infrastructure).</p> <ol> <li>Export environment variables.</li> </ol> <pre><code>export DOCKER_REGISTRY=bakdata &amp;&amp; \\\nexport NAMESPACE=kpops\n</code></pre> <ol> <li> <p>Navigate to the <code>examples</code> folder.    Replace the <code>&lt;name-of-the-example-directory&gt;</code> with the example you want to tear down.    For example the <code>atm-fraud-detection</code>.</p> </li> <li> <p>Remove the pipeline</p> </li> </ol> <pre><code># Uncomment 1 line to either destroy, reset or clean.\n# poetry run kpops destroy &lt;name-of-the-example-directory&gt;/pipeline.yaml \\\n# poetry run kpops reset &lt;name-of-the-example-directory&gt;/pipeline.yaml \\\n# poetry run kpops clean &lt;name-of-the-example-directory&gt;/pipeline.yaml \\\n--config &lt;name-of-the-example-directory&gt;/config.yaml \\\n--execute\n</code></pre>"},{"location":"user/getting-started/teardown/#infrastructure","title":"Infrastructure","text":"<p>Delete namespace:</p> <pre><code>kubectl delete namespace kpops\n</code></pre> <p>Note</p> <p>In case <code>kpops destroy</code> is not working one can uninstall the pipeline services one by one. This is equivalent to running <code>kpops destroy</code>. In case a clean uninstall (like the one <code>kpops clean</code> does)  is needed, one needs to also delete the topics and schemas created by deployment of the pipeline.</p>"},{"location":"user/getting-started/teardown/#local-cluster","title":"Local cluster","text":"<p>Delete local cluster:</p> <pre><code>k3d cluster delete kpops\n</code></pre>"},{"location":"user/getting-started/teardown/#local-image-registry","title":"Local image registry","text":"<p>Delete local registry:</p> <pre><code>k3d registry delete k3d-kpops-registry.localhost\n</code></pre>"},{"location":"user/migration-guide/v1-v2/","title":"Migrate from V1 to V2","text":""},{"location":"user/migration-guide/v1-v2/#derive-component-type-automatically-from-class-name","title":"Derive component type automatically from class name","text":"<p>KPOps automatically infers the component <code>type</code> from the class name. Therefore, the <code>type</code> and <code>schema_type</code> attributes can be removed from your custom components. By convention the <code>type</code> would be the lower, and kebab cased name of the class.</p> <pre><code>class MyCoolStreamApp(StreamsApp):\n-    type = \"my-cool-stream-app\"\n+    ...\n</code></pre> <p>Because of this new convention <code>producer</code> has been renamed to <code>producer-app</code>. This must be addressed in your <code>pipeline.yaml</code> and <code>defaults.yaml</code>.</p> <pre><code>- producer:\n+ producer-app:\n   app:\n        streams:\n        outputTopic: output_topic\n        extraOutputTopics:\n            output_role1: output_topic1\n            output_role2: output_topic2\n</code></pre>"},{"location":"user/migration-guide/v1-v2/#refactor-inputoutput-types","title":"Refactor input/output types","text":""},{"location":"user/migration-guide/v1-v2/#to-section","title":"To section","text":"<p>In the <code>to</code> section these have changed:</p> <ul> <li>The default type is <code>output</code></li> <li>If <code>role</code> is set, type is inferred to be <code>extra</code></li> <li>The type <code>error</code> needs to be defined explicitly</li> </ul> <pre><code>  to:\n    topics:\n      ${pipeline_name}-topic-1:\n-       type: extra\n       role: \"role-1\"\n        ...\n      ${pipeline_name}-topic-2:\n-       type: output\n       ...\n      ${pipeline_name}-topic-3:\n         type: error\n         ...\n</code></pre>"},{"location":"user/migration-guide/v1-v2/#from-section","title":"From section","text":"<p>In the <code>from</code> section these have changed:</p> <ul> <li>The default type is <code>input</code></li> <li><code>input-pattern</code> type is replaced by <code>pattern</code></li> <li>If <code>role</code> is set, type is inferred to be <code>extra</code></li> <li>If <code>role</code> is set, type is explicitly set to <code>pattern</code>, this would be inferred type <code>extra-pattern</code></li> </ul> <pre><code>  from:\n    topics:\n      ${pipeline_name}-input-topic:\n-       type: input\n       ...\n      ${pipeline_name}-extra-topic:\n-       type: extra\n       role: topic-role\n        ...\n      ${pipeline_name}-input-pattern-topic:\n-       type: input-pattern\n+       type: pattern\n       ...\n      ${pipeline_name}-extra-pattern-topic:\n-       type: extra-pattern\n+       type: pattern\n       role: some-role\n        ...\n</code></pre>"},{"location":"user/migration-guide/v1-v2/#remove-camel-case-conversion-of-internal-models","title":"Remove camel case conversion of internal models","text":"<p>All the internal KPOps models are now snake_case, and only Helm/Kubernetes values require camel casing. You can find an example of a <code>pipeline.yaml</code> in the following. Notice that the <code>app</code> section here remains untouched.</p> <pre><code>...\ntype: streams-app\n name: streams-app\n  namespace: namespace\n  app:\n    streams:\n      brokers: ${brokers}\n      schemaRegistryUrl: ${schema_registry_url}\n     autoscaling:\n      consumerGroup: consumer-group\n      lagThreshold: 0\n      enabled: false\n      pollingInterval: 30\n\n to:\n    topics:\n      ${pipeline_name}-output-topic:\n        type: error\n-       keySchema: key-schema\n+       key_schema: key-schema\n-       valueSchema: value-schema\n+       value_schema: value-schema\n       partitions_count: 1\n        replication_factor: 1\n        configs:\n          cleanup.policy: compact\n    models:\n      model: model\n  prefix: ${pipeline_name}-\n- repoConfig:\n+ repo_config:\n-   repositoryName: bakdata-streams-bootstrap\n+   repository_name: bakdata-streams-bootstrap\n   url: https://bakdata.github.io/streams-bootstrap/\n-   repoAuthFlags:\n+   repo_auth_flags:\n     username: user\n      password: pass\n      ca_file: /home/user/path/to/ca-file\n      insecure_skip_tls_verify: false\n  version: \"1.0.4\"\n...\n</code></pre>"},{"location":"user/migration-guide/v1-v2/#refactor-handling-of-helm-flags","title":"Refactor handling of Helm flags","text":"<p>If you are using the <code>KubernetesApp</code> class to define your own Kubernetes resource to deploy, the abstract function <code>get_helm_chart</code> that returns the chart for deploying the app using Helm is now a Python property and renamed to <code>helm_chart</code>.</p> <pre><code>class MyCoolApp(KubernetesApp):\n+   @property\n   @override\n-   def get_helm_chart(self) -&gt; str:\n+   def helm_chart(self) -&gt; str:\n       return \"./charts/charts-folder\"\n</code></pre>"},{"location":"user/migration-guide/v1-v2/#plural-broker-field-in-pipeline-config","title":"Plural broker field in pipeline config","text":"<p>Since you can pass a comma separated string of broker address, the broker field in KPOps is now plural. The pluralization has affected multiple areas:</p>"},{"location":"user/migration-guide/v1-v2/#configyaml","title":"config.yaml","text":"<pre><code>  environment: development\n- broker: \"http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092\"\n+ brokers: \"http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092\"\n kafka_connect_host: \"http://localhost:8083\"\n  kafka_rest_host: \"http://localhost:8082\"\n  schema_registry_url: \"http://localhost:8081\"\n</code></pre>"},{"location":"user/migration-guide/v1-v2/#pipelineyaml-and-defaultyaml","title":"pipeline.yaml and default.yaml","text":"<p>The variable is now called <code>brokers</code>.</p> <pre><code>...\n app:\n    streams:\n-     brokers: ${broker}\n+     brokers: ${brokers}\n     schemaRegistryUrl: ${schema_registry_url}\n    nameOverride: override-with-this-name\n    imageTag: \"1.0.0\"\n...\n</code></pre>"},{"location":"user/migration-guide/v1-v2/#environment-variable","title":"Environment variable","text":"<p>Previously, if you set the environment variable <code>KPOPS_KAFKA_BROKER</code>, you need to replace that now with <code>KPOPS_KAFKA_BROKERS</code>.</p>"},{"location":"user/migration-guide/v2-v3/","title":"Migrate from V2 to V3","text":""},{"location":"user/migration-guide/v2-v3/#make-kafka-rest-proxy-kafka-connect-hosts-default-and-improve-schema-registry-config","title":"Make Kafka REST Proxy &amp; Kafka Connect hosts default and improve Schema Registry config","text":"<p>The breaking changes target the <code>config.yaml</code> file:</p> <ul> <li> <p>The <code>schema_registry_url</code> is replaced with <code>schema_registry.url</code> (default <code>http://localhost:8081</code>) and <code>schema_registry.enabled</code> (default <code>false</code>).</p> </li> <li> <p><code>kafka_rest_host</code> is renamed to <code>kafka_rest.url</code> (default <code>http://localhost:8082</code>).</p> </li> <li> <p><code>kafka_connect_host</code> is replaced with <code>kafka_connect.url</code> (default <code>http://localhost:8083</code>).</p> </li> <li> <p><code>brokers</code> is renamed to <code>kafka_brokers</code>.</p> </li> </ul> <p>The environment variable names of these config fields changed respectively. Please refer to the environment variables documentation page to see the newest changes.</p>"},{"location":"user/migration-guide/v2-v3/#configyaml","title":"config.yaml","text":"<pre><code>  environment: development\n- brokers: \"http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092\"\n- kafka_rest_host: \"http://my-custom-rest.url:8082\"\n- kafka_connect_host: \"http://my-custom-connect.url:8083\"\n- schema_registry_url: \"http://my-custom-sr.url:8081\"\n+ kafka_brokers: \"http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092\"\n+ kafka_rest:\n+   url: \"http://my-custom-rest.url:8082\"\n+ kafka_connect:\n+   url: \"http://my-custom-connect.url:8083\"\n+ schema_registry:\n+   enabled: true\n+   url: \"http://my-custom-sr.url:8081\"\n</code></pre>"},{"location":"user/migration-guide/v2-v3/#pipelineyaml-and-defaultyaml","title":"pipeline.yaml and default.yaml","text":"<p>The variable is now called <code>kafka_brokers</code>.</p> <pre><code>...\n app:\n    streams:\n-     brokers: ${brokers}\n+     brokers: ${kafka_brokers}\n     schemaRegistryUrl: ${schema_registry_url}\n    nameOverride: override-with-this-name\n    imageTag: \"1.0.0\"\n...\n</code></pre>"},{"location":"user/migration-guide/v2-v3/#move-github-action-to-repsitory-root","title":"Move GitHub action to repsitory root","text":"<p>The location of the GitHub action has changed, and it's now available directly as <code>bakdata/kpops</code>.</p> <p>You'll need to change it in your GitHub CI workflows.</p> <pre><code>steps:\n - name: kpops deploy\n-   uses: bakdata/kpops/actions/kpops-runner@main\n+   uses: bakdata/kpops@main\n   with:\n      command: deploy --execute\n      # ...\n</code></pre>"},{"location":"user/references/cli-commands/","title":"CLI Usage","text":"<p>Usage:</p> <pre><code>$ kpops [OPTIONS] COMMAND [ARGS]...\n</code></pre> <p>Options:</p> <ul> <li><code>-V, --version</code>: Print KPOps version</li> <li><code>--install-completion</code>: Install completion for the current shell.</li> <li><code>--show-completion</code>: Show completion for the current shell, to copy it or customize the installation.</li> <li><code>--help</code>: Show this message and exit.</li> </ul> <p>Commands:</p> <ul> <li><code>clean</code>: Clean pipeline steps</li> <li><code>deploy</code>: Deploy pipeline steps</li> <li><code>destroy</code>: Destroy pipeline steps</li> <li><code>generate</code>: Enriches pipelines steps with defaults.</li> <li><code>reset</code>: Reset pipeline steps</li> <li><code>schema</code>: Generate json schema.</li> </ul>"},{"location":"user/references/cli-commands/#kpops-clean","title":"<code>kpops clean</code>","text":"<p>Clean pipeline steps</p> <p>Usage:</p> <pre><code>$ kpops clean [OPTIONS] PIPELINE_PATH [COMPONENTS_MODULE]\n</code></pre> <p>Arguments:</p> <ul> <li><code>PIPELINE_PATH</code>: Path to YAML with pipeline definition  [env var: KPOPS_PIPELINE_PATH;required]</li> <li><code>[COMPONENTS_MODULE]</code>: Custom Python module containing your project-specific components</li> </ul> <p>Options:</p> <ul> <li><code>--pipeline-base-dir DIRECTORY</code>: Base directory to the pipelines (default is current working directory)  [env var: KPOPS_PIPELINE_BASE_DIR; default: .]</li> <li><code>--defaults DIRECTORY</code>: Path to defaults folder  [env var: KPOPS_DEFAULT_PATH]</li> <li><code>--config FILE</code>: Path to the config.yaml file  [env var: KPOPS_CONFIG_PATH; default: config.yaml]</li> <li><code>--steps TEXT</code>: Comma separated list of steps to apply the command on  [env var: KPOPS_PIPELINE_STEPS]</li> <li><code>--filter-type [include|exclude]</code>: Whether the --steps option should include/exclude the steps  [default: include]</li> <li><code>--dry-run / --execute</code>: Whether to dry run the command or execute it  [default: dry-run]</li> <li><code>--verbose / --no-verbose</code>: Enable verbose printing  [default: no-verbose]</li> <li><code>--help</code>: Show this message and exit.</li> </ul>"},{"location":"user/references/cli-commands/#kpops-deploy","title":"<code>kpops deploy</code>","text":"<p>Deploy pipeline steps</p> <p>Usage:</p> <pre><code>$ kpops deploy [OPTIONS] PIPELINE_PATH [COMPONENTS_MODULE]\n</code></pre> <p>Arguments:</p> <ul> <li><code>PIPELINE_PATH</code>: Path to YAML with pipeline definition  [env var: KPOPS_PIPELINE_PATH;required]</li> <li><code>[COMPONENTS_MODULE]</code>: Custom Python module containing your project-specific components</li> </ul> <p>Options:</p> <ul> <li><code>--pipeline-base-dir DIRECTORY</code>: Base directory to the pipelines (default is current working directory)  [env var: KPOPS_PIPELINE_BASE_DIR; default: .]</li> <li><code>--defaults DIRECTORY</code>: Path to defaults folder  [env var: KPOPS_DEFAULT_PATH]</li> <li><code>--config FILE</code>: Path to the config.yaml file  [env var: KPOPS_CONFIG_PATH; default: config.yaml]</li> <li><code>--steps TEXT</code>: Comma separated list of steps to apply the command on  [env var: KPOPS_PIPELINE_STEPS]</li> <li><code>--filter-type [include|exclude]</code>: Whether the --steps option should include/exclude the steps  [default: include]</li> <li><code>--dry-run / --execute</code>: Whether to dry run the command or execute it  [default: dry-run]</li> <li><code>--verbose / --no-verbose</code>: Enable verbose printing  [default: no-verbose]</li> <li><code>--help</code>: Show this message and exit.</li> </ul>"},{"location":"user/references/cli-commands/#kpops-destroy","title":"<code>kpops destroy</code>","text":"<p>Destroy pipeline steps</p> <p>Usage:</p> <pre><code>$ kpops destroy [OPTIONS] PIPELINE_PATH [COMPONENTS_MODULE]\n</code></pre> <p>Arguments:</p> <ul> <li><code>PIPELINE_PATH</code>: Path to YAML with pipeline definition  [env var: KPOPS_PIPELINE_PATH;required]</li> <li><code>[COMPONENTS_MODULE]</code>: Custom Python module containing your project-specific components</li> </ul> <p>Options:</p> <ul> <li><code>--pipeline-base-dir DIRECTORY</code>: Base directory to the pipelines (default is current working directory)  [env var: KPOPS_PIPELINE_BASE_DIR; default: .]</li> <li><code>--defaults DIRECTORY</code>: Path to defaults folder  [env var: KPOPS_DEFAULT_PATH]</li> <li><code>--config FILE</code>: Path to the config.yaml file  [env var: KPOPS_CONFIG_PATH; default: config.yaml]</li> <li><code>--steps TEXT</code>: Comma separated list of steps to apply the command on  [env var: KPOPS_PIPELINE_STEPS]</li> <li><code>--filter-type [include|exclude]</code>: Whether the --steps option should include/exclude the steps  [default: include]</li> <li><code>--dry-run / --execute</code>: Whether to dry run the command or execute it  [default: dry-run]</li> <li><code>--verbose / --no-verbose</code>: Enable verbose printing  [default: no-verbose]</li> <li><code>--help</code>: Show this message and exit.</li> </ul>"},{"location":"user/references/cli-commands/#kpops-generate","title":"<code>kpops generate</code>","text":"<p>Enriches pipelines steps with defaults. The output is used as input for the deploy/destroy/... commands.</p> <p>Usage:</p> <pre><code>$ kpops generate [OPTIONS] PIPELINE_PATH [COMPONENTS_MODULE]\n</code></pre> <p>Arguments:</p> <ul> <li><code>PIPELINE_PATH</code>: Path to YAML with pipeline definition  [env var: KPOPS_PIPELINE_PATH;required]</li> <li><code>[COMPONENTS_MODULE]</code>: Custom Python module containing your project-specific components</li> </ul> <p>Options:</p> <ul> <li><code>--pipeline-base-dir DIRECTORY</code>: Base directory to the pipelines (default is current working directory)  [env var: KPOPS_PIPELINE_BASE_DIR; default: .]</li> <li><code>--defaults DIRECTORY</code>: Path to defaults folder  [env var: KPOPS_DEFAULT_PATH]</li> <li><code>--config FILE</code>: Path to the config.yaml file  [env var: KPOPS_CONFIG_PATH; default: config.yaml]</li> <li><code>--template / --no-template</code>: Run Helm template  [default: no-template]</li> <li><code>--steps TEXT</code>: Comma separated list of steps to apply the command on  [env var: KPOPS_PIPELINE_STEPS]</li> <li><code>--filter-type [include|exclude]</code>: Whether the --steps option should include/exclude the steps  [default: include]</li> <li><code>--verbose / --no-verbose</code>: Enable verbose printing  [default: no-verbose]</li> <li><code>--help</code>: Show this message and exit.</li> </ul>"},{"location":"user/references/cli-commands/#kpops-reset","title":"<code>kpops reset</code>","text":"<p>Reset pipeline steps</p> <p>Usage:</p> <pre><code>$ kpops reset [OPTIONS] PIPELINE_PATH [COMPONENTS_MODULE]\n</code></pre> <p>Arguments:</p> <ul> <li><code>PIPELINE_PATH</code>: Path to YAML with pipeline definition  [env var: KPOPS_PIPELINE_PATH;required]</li> <li><code>[COMPONENTS_MODULE]</code>: Custom Python module containing your project-specific components</li> </ul> <p>Options:</p> <ul> <li><code>--pipeline-base-dir DIRECTORY</code>: Base directory to the pipelines (default is current working directory)  [env var: KPOPS_PIPELINE_BASE_DIR; default: .]</li> <li><code>--defaults DIRECTORY</code>: Path to defaults folder  [env var: KPOPS_DEFAULT_PATH]</li> <li><code>--config FILE</code>: Path to the config.yaml file  [env var: KPOPS_CONFIG_PATH; default: config.yaml]</li> <li><code>--steps TEXT</code>: Comma separated list of steps to apply the command on  [env var: KPOPS_PIPELINE_STEPS]</li> <li><code>--filter-type [include|exclude]</code>: Whether the --steps option should include/exclude the steps  [default: include]</li> <li><code>--dry-run / --execute</code>: Whether to dry run the command or execute it  [default: dry-run]</li> <li><code>--verbose / --no-verbose</code>: Enable verbose printing  [default: no-verbose]</li> <li><code>--help</code>: Show this message and exit.</li> </ul>"},{"location":"user/references/cli-commands/#kpops-schema","title":"<code>kpops schema</code>","text":"<p>Generate json schema.</p> <p>The schemas can be used to enable support for kpops files in a text editor.</p> <p>Usage:</p> <pre><code>$ kpops schema [OPTIONS] SCOPE:{pipeline|config} [COMPONENTS_MODULE]\n</code></pre> <p>Arguments:</p> <ul> <li><code>SCOPE:{pipeline|config}</code>:          Scope of the generated schema<pre><code>pipeline: Schema of PipelineComponents. Includes the built-in kpops components by default. To include custom components, provide [COMPONENTS_MODULES].\n\nconfig: Schema of PipelineConfig.  [required]\n</code></pre> <ul> <li><code>[COMPONENTS_MODULE]</code>: Custom Python module containing your project-specific components</li> </ul> </li> </ul> <p>Options:</p> <ul> <li><code>--include-stock-components / --no-include-stock-components</code>: Include the built-in KPOps components.  [default: include-stock-components]</li> <li><code>--help</code>: Show this message and exit.</li> </ul>"},{"location":"user/references/editor-integration/","title":"Editor integration","text":"<p>KPOps provides JSON schemas that enable autocompletion and validation for some of the files that the user must work with.</p>"},{"location":"user/references/editor-integration/#supported-files","title":"Supported files","text":"<ul> <li><code>pipeline.yaml</code></li> <li><code>config.yaml</code></li> </ul>"},{"location":"user/references/editor-integration/#usage","title":"Usage","text":"<ol> <li>Install the yaml-language-server in your editor of choice. (requires LSP support)</li> <li>Configure the extension with the settings below.</li> </ol> <code>settings.json</code> <pre><code>{\n\"yaml.schemas\": {\n\"https://bakdata.github.io/kpops/2.0/schema/config.json\": \"config.yaml\",\n\"https://bakdata.github.io/kpops/2.0/schema/pipeline.json\": \"pipeline.yaml\"\n}\n}\n</code></pre> <p>Advanced usage</p> <p>It is possible to generate schemas with the <code>kpops schema</code> command. Useful when using custom components or when using a pre-release version of KPOps.</p>"},{"location":"user/references/ci-integration/github-actions/","title":"GitHub Actions integration","text":"<p>We provided a GitHub composite action <code>bakdata/kpops</code> that installs and executes KPOps commands with the given parameters.</p>"},{"location":"user/references/ci-integration/github-actions/#input-parameters","title":"Input Parameters","text":"Name Required Default Value Type Description command \u2705 - string KPOps command to run. generate, deploy, destroy, reset, clean are possible values. Flags such as --dry-run and --execute need to be specified pipeline \u2705 - string Pipeline to run by KPOps working-directory \u274c . string root directory used by KPOps to run pipelines pipeline-base-dir \u274c - string directory where relative pipeline variables are initialized from defaults \u274c - string defaults folder path config \u274c - string config.yaml file path components \u274c - string components package path filter-type \u274c - string Whether to include/exclude the steps defined in KPOPS_PIPELINE_STEPS python-version \u274c \"3.11.x\" string Python version to install (Defaults to the latest stable version of Python 3.11) kpops-version \u274c latest string KPOps version to install helm-version \u274c latest string Helm version to install token \u274c latest string secrets.GITHUB_TOKEN, needed for setup-helm action if helm-version is set to latest"},{"location":"user/references/ci-integration/github-actions/#usage","title":"Usage","text":"<pre><code>steps:\n# ...\n# This step is useful for debugging reasons\n- name: Generate Kafka pipeline\nuses: bakdata/kpops@main\nwith:\ncommand: generate\nworking-directory: home/my-kpops-root-dir\npipeline: pipelines/my-pipeline-file.yaml\nkpops-version: 1.2.3\n- name: Deploy Kafka pipeline\nuses: bakdata/kpops@main\nwith:\ncommand: deploy --execute\nworking-directory: home/my-kpops-root-dir\npipeline: pipelines/my-pipeline-file.yaml\nkpops-version: 1.2.3\n# ...\n</code></pre> <p>It is possible to use a pre-release KPOps version from TestPyPI.</p> <pre><code>steps:\n- name: Deploy Kafka pipeline\nuses: bakdata/kpops@main\nwith:\ncommand: deploy --execute\nworking-directory: home/my-kpops-root-dir\npipeline: pipelines/my-pipeline-file.yaml\nkpops-version: 1.2.5.dev20230707132709 -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/\n</code></pre>"}]}
\ No newline at end of file
diff --git a/dev/sitemap.xml b/dev/sitemap.xml
index 898220689..a51e9543d 100644
--- a/dev/sitemap.xml
+++ b/dev/sitemap.xml
@@ -2,157 +2,162 @@
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
     <url>
          <loc>https://bakdata.github.io/kpops/dev/</loc>
-         <lastmod>2023-09-28</lastmod>
+         <lastmod>2023-09-29</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://bakdata.github.io/kpops/dev/developer/auto-generation/</loc>
-         <lastmod>2023-09-28</lastmod>
+         <lastmod>2023-09-29</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://bakdata.github.io/kpops/dev/developer/formatting/</loc>
-         <lastmod>2023-09-28</lastmod>
+         <lastmod>2023-09-29</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://bakdata.github.io/kpops/dev/resources/architecture/components-hierarchy/</loc>
-         <lastmod>2023-09-28</lastmod>
+         <lastmod>2023-09-29</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://bakdata.github.io/kpops/dev/resources/examples/defaults/</loc>
-         <lastmod>2023-09-28</lastmod>
+         <lastmod>2023-09-29</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://bakdata.github.io/kpops/dev/resources/examples/pipeline/</loc>
-         <lastmod>2023-09-28</lastmod>
+         <lastmod>2023-09-29</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://bakdata.github.io/kpops/dev/resources/pipeline-components/pipeline/</loc>
-         <lastmod>2023-09-28</lastmod>
+         <lastmod>2023-09-29</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://bakdata.github.io/kpops/dev/resources/pipeline-defaults/defaults/</loc>
-         <lastmod>2023-09-28</lastmod>
+         <lastmod>2023-09-29</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://bakdata.github.io/kpops/dev/resources/variables/cli_env_vars/</loc>
-         <lastmod>2023-09-28</lastmod>
+         <lastmod>2023-09-29</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://bakdata.github.io/kpops/dev/resources/variables/config_env_vars/</loc>
-         <lastmod>2023-09-28</lastmod>
+         <lastmod>2023-09-29</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://bakdata.github.io/kpops/dev/user/what-is-kpops/</loc>
-         <lastmod>2023-09-28</lastmod>
+         <lastmod>2023-09-29</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://bakdata.github.io/kpops/dev/user/core-concepts/config/</loc>
-         <lastmod>2023-09-28</lastmod>
+         <lastmod>2023-09-29</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://bakdata.github.io/kpops/dev/user/core-concepts/defaults/</loc>
-         <lastmod>2023-09-28</lastmod>
+         <lastmod>2023-09-29</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://bakdata.github.io/kpops/dev/user/core-concepts/components/kafka-app/</loc>
-         <lastmod>2023-09-28</lastmod>
+         <lastmod>2023-09-29</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://bakdata.github.io/kpops/dev/user/core-concepts/components/kafka-connector/</loc>
-         <lastmod>2023-09-28</lastmod>
+         <lastmod>2023-09-29</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://bakdata.github.io/kpops/dev/user/core-concepts/components/kafka-sink-connector/</loc>
-         <lastmod>2023-09-28</lastmod>
+         <lastmod>2023-09-29</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://bakdata.github.io/kpops/dev/user/core-concepts/components/kafka-source-connector/</loc>
-         <lastmod>2023-09-28</lastmod>
+         <lastmod>2023-09-29</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://bakdata.github.io/kpops/dev/user/core-concepts/components/kubernetes-app/</loc>
-         <lastmod>2023-09-28</lastmod>
+         <lastmod>2023-09-29</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://bakdata.github.io/kpops/dev/user/core-concepts/components/overview/</loc>
-         <lastmod>2023-09-28</lastmod>
+         <lastmod>2023-09-29</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://bakdata.github.io/kpops/dev/user/core-concepts/components/producer-app/</loc>
-         <lastmod>2023-09-28</lastmod>
+         <lastmod>2023-09-29</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://bakdata.github.io/kpops/dev/user/core-concepts/components/streams-app/</loc>
-         <lastmod>2023-09-28</lastmod>
+         <lastmod>2023-09-29</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://bakdata.github.io/kpops/dev/user/core-concepts/variables/environment_variables/</loc>
-         <lastmod>2023-09-28</lastmod>
+         <lastmod>2023-09-29</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://bakdata.github.io/kpops/dev/user/core-concepts/variables/substitution/</loc>
-         <lastmod>2023-09-28</lastmod>
+         <lastmod>2023-09-29</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://bakdata.github.io/kpops/dev/user/examples/atm-fraud-pipeline/</loc>
-         <lastmod>2023-09-28</lastmod>
+         <lastmod>2023-09-29</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://bakdata.github.io/kpops/dev/user/getting-started/quick-start/</loc>
-         <lastmod>2023-09-28</lastmod>
+         <lastmod>2023-09-29</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://bakdata.github.io/kpops/dev/user/getting-started/setup/</loc>
-         <lastmod>2023-09-28</lastmod>
+         <lastmod>2023-09-29</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://bakdata.github.io/kpops/dev/user/getting-started/teardown/</loc>
-         <lastmod>2023-09-28</lastmod>
+         <lastmod>2023-09-29</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://bakdata.github.io/kpops/dev/user/migration-guide/v1-v2/</loc>
-         <lastmod>2023-09-28</lastmod>
+         <lastmod>2023-09-29</lastmod>
+         <changefreq>daily</changefreq>
+    </url>
+    <url>
+         <loc>https://bakdata.github.io/kpops/dev/user/migration-guide/v2-v3/</loc>
+         <lastmod>2023-09-29</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://bakdata.github.io/kpops/dev/user/references/cli-commands/</loc>
-         <lastmod>2023-09-28</lastmod>
+         <lastmod>2023-09-29</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://bakdata.github.io/kpops/dev/user/references/editor-integration/</loc>
-         <lastmod>2023-09-28</lastmod>
+         <lastmod>2023-09-29</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://bakdata.github.io/kpops/dev/user/references/ci-integration/github-actions/</loc>
-         <lastmod>2023-09-28</lastmod>
+         <lastmod>2023-09-29</lastmod>
          <changefreq>daily</changefreq>
     </url>
 </urlset>
\ No newline at end of file
diff --git a/dev/sitemap.xml.gz b/dev/sitemap.xml.gz
index 75ccd3b4a9f4d354df791aabe133690d4bde07ae..2524fe7dc246e4de6bbeb54e043cd0861589040d 100644
GIT binary patch
literal 562
zcmV-20?qv&iwFpqmKJ3K|8r?{Wo=<_E_iKh0Nt3)ZsQ;j$M1cL$ahSdUN)=5z4Zxp
z&(Q?NHnI&A2D^Rx#de!ywR_sbDpO0A&G4b$e;5$M;rVUoHzS5<oIMtg<+j+MRZbh*
z9E+bnUc|TJxq3LH&_|?AZfxVRXo>uHDW_>Fqahr?$I^vHM%Cs*lpjX9E%!zBu*uIn
z!1}j5hu*1b9wp{s&d_NfC>uj<I+w=D&O0Ba#!(*R-14>~L>io~cH7;P*zUz{pPPKx
zZknkUY=d>c*Ghxw-w*P}vJ%M)(!avx7klS1NJ!*FgBAl&b|e?<+Jym#@?76Halq(O
zP#7fy)#l(*lmbfS2Jb9dilQ~y^||hyix7N+!S^_uYe+pscHrK4?2Sb(;GewW8a8}2
zT%v|s_>2$?oagvay*Z&BPuaqe-r3qTOa!EuH~Xmt5+;h(eE+=4g?X)7g`PhB%?_K#
z^8Bm6FPmfsbq4~x=Px4H*nD_!fxnb!Z1<(~EL-dsQIyUYED}cU%f{u@a5Ok^*5HHF
zNnzkV7m)(Oz%!(-2d8{uGQrs)OFCVxeD%b1j$|m^f8M{yxTJ4DgQx=}EqEkB3w?vj
zZH0&gfl$lWWK_477}7_i<QZ}kLeOsF5blF%)+;&DB%@J|k7C?$<lc$#i2--O8UtFj
z=J)ee%K%nKUgW4vK7a~i2^$U<x_@}y{Kmd^5uoPMf0&BpllQjj7wY*H;awO20LS?s
Ay8r+H

literal 557
zcmV+|0@D2-iwFo(B^6}?|8r?{Wo=<_E_iKh0Nt3)Zrd;n$M1cLAa_}5c4>#yz4ZyU
zbHK>7E!dV+lCs;kpKP~DhV8V&5Y>QTi~7LtPf>!X!}HtFZ$>1x-aT#ai`{mE&iKl@
z=4t!$$BTN{K9_fgNIj!$Qe($Y+g2oe*Ls?!g00{HfeTNKX49qu>kp&e6_4BUZj+vQ
zfc0;A4!t+!JPM~_PS8~#P&8KBcq**dUGRZ*g`+;`x#e|7U>ST>?sxldYWJx252?wA
z?W&n+!8KSDzLpi({{5h@EGv;dBmFB}da?HbNkbH`8gxh?$&tBW*PaFtNppSM#Dwf4
z8DtG)+7w(yBB3^Z2;QL+R;^90Pj&CP2*EcPLXWe#hS&>d2Oey|-a5<${F7H*!X{r0
z=cvgod`2J(r#XI1Z;$B4W3uE(?_F)1Oaw$uoBh;+D9dU!-#)MLG_O@>FvzFBv%}6~
zdH&Vk=S{MMx&s9v<S!!CxO8~%l)n_VuKQAYmM!*+D2gW}hk~5@vhgW394$^cYY614
zXpnNB3l>5c@(iiV!6}`XEO|F1Nym#dUp*X8ELP&}=lzR}bNU7{sG1;FDvu;+p>J@x
ztxzmLf|Y)a)^uwrLwZJuAwzCKpvq4<g!^Ed^-4}P(N?I(do}Je;3lk*(3v$)p06wh
va24l8j+IRZIc1$-lf%X6AD*|rv9Dben7Q;HregV|J)-;t{vk2vr56AIfhH7T

diff --git a/dev/user/core-concepts/components/kafka-app/index.html b/dev/user/core-concepts/components/kafka-app/index.html
index b333b7254..981ec1a1d 100644
--- a/dev/user/core-concepts/components/kafka-app/index.html
+++ b/dev/user/core-concepts/components/kafka-app/index.html
@@ -425,6 +425,11 @@
         Migrate from v1 to v2
       </a>
 </li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../../migration-guide/v2-v3/">
+        Migrate from v2 to v3
+      </a>
+</li>
 </ul>
 </nav>
 </li>
@@ -677,7 +682,7 @@ <h3 id="configuration">Configuration<a class="headerlink" href="#configuration"
 </span><span id="__span-0-49"><a id="__codelineno-0-49" name="__codelineno-0-49"></a><span class="w">  </span><span class="c1"># add the key-value pairs they need.</span>
 </span><span id="__span-0-50"><a id="__codelineno-0-50" name="__codelineno-0-50"></a><span class="w">  </span><span class="nt">app</span><span class="p">:</span><span class="w"> </span><span class="c1"># required</span>
 </span><span id="__span-0-51"><a id="__codelineno-0-51" name="__codelineno-0-51"></a><span class="w">    </span><span class="nt">streams</span><span class="p">:</span><span class="w"> </span><span class="c1"># required</span>
-</span><span id="__span-0-52"><a id="__codelineno-0-52" name="__codelineno-0-52"></a><span class="w">      </span><span class="nt">brokers</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">${brokers}</span><span class="w"> </span><span class="c1"># required</span>
+</span><span id="__span-0-52"><a id="__codelineno-0-52" name="__codelineno-0-52"></a><span class="w">      </span><span class="nt">brokers</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">${kafka_brokers}</span><span class="w"> </span><span class="c1"># required</span>
 </span><span id="__span-0-53"><a id="__codelineno-0-53" name="__codelineno-0-53"></a><span class="w">      </span><span class="nt">schemaRegistryUrl</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">${schema_registry_url}</span>
 </span><span id="__span-0-54"><a id="__codelineno-0-54" name="__codelineno-0-54"></a><span class="w">    </span><span class="nt">nameOverride</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">override-with-this-name</span><span class="w"> </span><span class="c1"># kafka-app-specific</span>
 </span><span id="__span-0-55"><a id="__codelineno-0-55" name="__codelineno-0-55"></a><span class="w">    </span><span class="nt">imageTag</span><span class="p">:</span><span class="w"> </span><span class="s">"1.0.0"</span><span class="w"> </span><span class="c1"># Example values that are shared between streams-app and producer-app</span>
diff --git a/dev/user/core-concepts/components/kafka-connector/index.html b/dev/user/core-concepts/components/kafka-connector/index.html
index cc3bfa9c8..72599219a 100644
--- a/dev/user/core-concepts/components/kafka-connector/index.html
+++ b/dev/user/core-concepts/components/kafka-connector/index.html
@@ -374,6 +374,11 @@
         Migrate from v1 to v2
       </a>
 </li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../../migration-guide/v2-v3/">
+        Migrate from v2 to v3
+      </a>
+</li>
 </ul>
 </nav>
 </li>
diff --git a/dev/user/core-concepts/components/kafka-sink-connector/index.html b/dev/user/core-concepts/components/kafka-sink-connector/index.html
index d27b92301..1336e6eb2 100644
--- a/dev/user/core-concepts/components/kafka-sink-connector/index.html
+++ b/dev/user/core-concepts/components/kafka-sink-connector/index.html
@@ -425,6 +425,11 @@
         Migrate from v1 to v2
       </a>
 </li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../../migration-guide/v2-v3/">
+        Migrate from v2 to v3
+      </a>
+</li>
 </ul>
 </nav>
 </li>
diff --git a/dev/user/core-concepts/components/kafka-source-connector/index.html b/dev/user/core-concepts/components/kafka-source-connector/index.html
index a18e80e21..020c5cc00 100644
--- a/dev/user/core-concepts/components/kafka-source-connector/index.html
+++ b/dev/user/core-concepts/components/kafka-source-connector/index.html
@@ -425,6 +425,11 @@
         Migrate from v1 to v2
       </a>
 </li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../../migration-guide/v2-v3/">
+        Migrate from v2 to v3
+      </a>
+</li>
 </ul>
 </nav>
 </li>
diff --git a/dev/user/core-concepts/components/kubernetes-app/index.html b/dev/user/core-concepts/components/kubernetes-app/index.html
index 234691095..d0afa4c19 100644
--- a/dev/user/core-concepts/components/kubernetes-app/index.html
+++ b/dev/user/core-concepts/components/kubernetes-app/index.html
@@ -425,6 +425,11 @@
         Migrate from v1 to v2
       </a>
 </li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../../migration-guide/v2-v3/">
+        Migrate from v2 to v3
+      </a>
+</li>
 </ul>
 </nav>
 </li>
diff --git a/dev/user/core-concepts/components/overview/index.html b/dev/user/core-concepts/components/overview/index.html
index c6dc5273b..86472f9c5 100644
--- a/dev/user/core-concepts/components/overview/index.html
+++ b/dev/user/core-concepts/components/overview/index.html
@@ -374,6 +374,11 @@
         Migrate from v1 to v2
       </a>
 </li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../../migration-guide/v2-v3/">
+        Migrate from v2 to v3
+      </a>
+</li>
 </ul>
 </nav>
 </li>
diff --git a/dev/user/core-concepts/components/producer-app/index.html b/dev/user/core-concepts/components/producer-app/index.html
index c635adc8d..9f3155026 100644
--- a/dev/user/core-concepts/components/producer-app/index.html
+++ b/dev/user/core-concepts/components/producer-app/index.html
@@ -425,6 +425,11 @@
         Migrate from v1 to v2
       </a>
 </li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../../migration-guide/v2-v3/">
+        Migrate from v2 to v3
+      </a>
+</li>
 </ul>
 </nav>
 </li>
@@ -642,7 +647,7 @@ <h3 id="configuration">Configuration<a class="headerlink" href="#configuration"
 </span><span id="__span-0-32"><a id="__codelineno-0-32" name="__codelineno-0-32"></a><span class="w">  </span><span class="c1"># https://github.com/bakdata/streams-bootstrap/tree/master/charts/producer-app</span>
 </span><span id="__span-0-33"><a id="__codelineno-0-33" name="__codelineno-0-33"></a><span class="w">  </span><span class="nt">app</span><span class="p">:</span><span class="w"> </span><span class="c1"># required</span>
 </span><span id="__span-0-34"><a id="__codelineno-0-34" name="__codelineno-0-34"></a><span class="w">    </span><span class="nt">streams</span><span class="p">:</span><span class="w"> </span><span class="c1"># required, producer-app-specific</span>
-</span><span id="__span-0-35"><a id="__codelineno-0-35" name="__codelineno-0-35"></a><span class="w">      </span><span class="nt">brokers</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">${brokers}</span><span class="w"> </span><span class="c1"># required</span>
+</span><span id="__span-0-35"><a id="__codelineno-0-35" name="__codelineno-0-35"></a><span class="w">      </span><span class="nt">brokers</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">${kafka_brokers}</span><span class="w"> </span><span class="c1"># required</span>
 </span><span id="__span-0-36"><a id="__codelineno-0-36" name="__codelineno-0-36"></a><span class="w">      </span><span class="nt">schemaRegistryUrl</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">${schema_registry_url}</span>
 </span><span id="__span-0-37"><a id="__codelineno-0-37" name="__codelineno-0-37"></a><span class="w">      </span><span class="nt">outputTopic</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">output_topic</span>
 </span><span id="__span-0-38"><a id="__codelineno-0-38" name="__codelineno-0-38"></a><span class="w">      </span><span class="nt">extraOutputTopics</span><span class="p">:</span>
diff --git a/dev/user/core-concepts/components/streams-app/index.html b/dev/user/core-concepts/components/streams-app/index.html
index 002e46396..21b36458e 100644
--- a/dev/user/core-concepts/components/streams-app/index.html
+++ b/dev/user/core-concepts/components/streams-app/index.html
@@ -425,6 +425,11 @@
         Migrate from v1 to v2
       </a>
 </li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../../migration-guide/v2-v3/">
+        Migrate from v2 to v3
+      </a>
+</li>
 </ul>
 </nav>
 </li>
@@ -725,7 +730,7 @@ <h3 id="configuration">Configuration<a class="headerlink" href="#configuration"
 </span><span id="__span-0-51"><a id="__codelineno-0-51" name="__codelineno-0-51"></a><span class="w">  </span><span class="nt">app</span><span class="p">:</span><span class="w"> </span><span class="c1"># required</span>
 </span><span id="__span-0-52"><a id="__codelineno-0-52" name="__codelineno-0-52"></a><span class="w">    </span><span class="c1"># Streams Bootstrap streams section</span>
 </span><span id="__span-0-53"><a id="__codelineno-0-53" name="__codelineno-0-53"></a><span class="w">    </span><span class="nt">streams</span><span class="p">:</span><span class="w"> </span><span class="c1"># required, streams-app-specific</span>
-</span><span id="__span-0-54"><a id="__codelineno-0-54" name="__codelineno-0-54"></a><span class="w">      </span><span class="nt">brokers</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">${brokers}</span><span class="w"> </span><span class="c1"># required</span>
+</span><span id="__span-0-54"><a id="__codelineno-0-54" name="__codelineno-0-54"></a><span class="w">      </span><span class="nt">brokers</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">${kafka_brokers}</span><span class="w"> </span><span class="c1"># required</span>
 </span><span id="__span-0-55"><a id="__codelineno-0-55" name="__codelineno-0-55"></a><span class="w">      </span><span class="nt">schemaRegistryUrl</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">${schema_registry_url}</span>
 </span><span id="__span-0-56"><a id="__codelineno-0-56" name="__codelineno-0-56"></a><span class="w">      </span><span class="nt">inputTopics</span><span class="p">:</span>
 </span><span id="__span-0-57"><a id="__codelineno-0-57" name="__codelineno-0-57"></a><span class="w">        </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">topic1</span>
diff --git a/dev/user/core-concepts/config/index.html b/dev/user/core-concepts/config/index.html
index 4f5037950..e2697ea61 100644
--- a/dev/user/core-concepts/config/index.html
+++ b/dev/user/core-concepts/config/index.html
@@ -374,6 +374,11 @@
         Migrate from v1 to v2
       </a>
 </li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../migration-guide/v2-v3/">
+        Migrate from v2 to v3
+      </a>
+</li>
 </ul>
 </nav>
 </li>
diff --git a/dev/user/core-concepts/defaults/index.html b/dev/user/core-concepts/defaults/index.html
index 42daa8fc4..5d2fb4b1d 100644
--- a/dev/user/core-concepts/defaults/index.html
+++ b/dev/user/core-concepts/defaults/index.html
@@ -449,6 +449,11 @@
         Migrate from v1 to v2
       </a>
 </li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../migration-guide/v2-v3/">
+        Migrate from v2 to v3
+      </a>
+</li>
 </ul>
 </nav>
 </li>
@@ -779,7 +784,7 @@ <h3 id="kafkaapp"><a href="../components/kafka-app/">KafkaApp</a><a class="heade
 </span><span id="__span-1-7"><a id="__codelineno-1-7" name="__codelineno-1-7"></a><span class="w">  </span><span class="c1"># add the key-value pairs they need.</span>
 </span><span id="__span-1-8"><a id="__codelineno-1-8" name="__codelineno-1-8"></a><span class="w">  </span><span class="nt">app</span><span class="p">:</span><span class="w"> </span><span class="c1"># required</span>
 </span><span id="__span-1-9"><a id="__codelineno-1-9" name="__codelineno-1-9"></a><span class="w">    </span><span class="nt">streams</span><span class="p">:</span><span class="w"> </span><span class="c1"># required</span>
-</span><span id="__span-1-10"><a id="__codelineno-1-10" name="__codelineno-1-10"></a><span class="w">      </span><span class="nt">brokers</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">${brokers}</span><span class="w"> </span><span class="c1"># required</span>
+</span><span id="__span-1-10"><a id="__codelineno-1-10" name="__codelineno-1-10"></a><span class="w">      </span><span class="nt">brokers</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">${kafka_brokers}</span><span class="w"> </span><span class="c1"># required</span>
 </span><span id="__span-1-11"><a id="__codelineno-1-11" name="__codelineno-1-11"></a><span class="w">      </span><span class="nt">schemaRegistryUrl</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">${schema_registry_url}</span>
 </span><span id="__span-1-12"><a id="__codelineno-1-12" name="__codelineno-1-12"></a><span class="w">    </span><span class="nt">nameOverride</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">override-with-this-name</span><span class="w"> </span><span class="c1"># kafka-app-specific</span>
 </span><span id="__span-1-13"><a id="__codelineno-1-13" name="__codelineno-1-13"></a><span class="w">    </span><span class="nt">imageTag</span><span class="p">:</span><span class="w"> </span><span class="s">"1.0.0"</span><span class="w"> </span><span class="c1"># Example values that are shared between streams-app and producer-app</span>
@@ -862,7 +867,7 @@ <h3 id="streamsapp"><a href="../components/streams-app/">StreamsApp</a><a class=
 </span><span id="__span-2-9"><a id="__codelineno-2-9" name="__codelineno-2-9"></a><span class="w">  </span><span class="nt">app</span><span class="p">:</span><span class="w"> </span><span class="c1"># required</span>
 </span><span id="__span-2-10"><a id="__codelineno-2-10" name="__codelineno-2-10"></a><span class="w">    </span><span class="c1"># Streams Bootstrap streams section</span>
 </span><span id="__span-2-11"><a id="__codelineno-2-11" name="__codelineno-2-11"></a><span class="w">    </span><span class="nt">streams</span><span class="p">:</span><span class="w"> </span><span class="c1"># required, streams-app-specific</span>
-</span><span id="__span-2-12"><a id="__codelineno-2-12" name="__codelineno-2-12"></a><span class="w">      </span><span class="nt">brokers</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">${brokers}</span><span class="w"> </span><span class="c1"># required</span>
+</span><span id="__span-2-12"><a id="__codelineno-2-12" name="__codelineno-2-12"></a><span class="w">      </span><span class="nt">brokers</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">${kafka_brokers}</span><span class="w"> </span><span class="c1"># required</span>
 </span><span id="__span-2-13"><a id="__codelineno-2-13" name="__codelineno-2-13"></a><span class="w">      </span><span class="nt">schemaRegistryUrl</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">${schema_registry_url}</span>
 </span><span id="__span-2-14"><a id="__codelineno-2-14" name="__codelineno-2-14"></a><span class="w">      </span><span class="nt">inputTopics</span><span class="p">:</span>
 </span><span id="__span-2-15"><a id="__codelineno-2-15" name="__codelineno-2-15"></a><span class="w">        </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">topic1</span>
diff --git a/dev/user/core-concepts/variables/environment_variables/index.html b/dev/user/core-concepts/variables/environment_variables/index.html
index 8bd60bd98..600574d9e 100644
--- a/dev/user/core-concepts/variables/environment_variables/index.html
+++ b/dev/user/core-concepts/variables/environment_variables/index.html
@@ -396,6 +396,11 @@
         Migrate from v1 to v2
       </a>
 </li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../../migration-guide/v2-v3/">
+        Migrate from v2 to v3
+      </a>
+</li>
 </ul>
 </nav>
 </li>
@@ -527,28 +532,28 @@ <h2 id="config">Config<a class="headerlink" href="#config" title="Permanent link
 <td></td>
 <td>True</td>
 <td>The comma separated Kafka brokers address.</td>
-<td>brokers</td>
+<td>kafka_brokers</td>
 </tr>
 <tr>
 <td>KPOPS_SCHEMA_REGISTRY_URL</td>
-<td></td>
+<td>http://localhost:8081</td>
 <td>False</td>
 <td>Address of the Schema Registry.</td>
-<td>schema_registry_url</td>
+<td>url</td>
 </tr>
 <tr>
-<td>KPOPS_REST_PROXY_HOST</td>
-<td></td>
+<td>KPOPS_KAFKA_REST_URL</td>
+<td>http://localhost:8082</td>
 <td>False</td>
 <td>Address of the Kafka REST Proxy.</td>
-<td>kafka_rest_host</td>
+<td>url</td>
 </tr>
 <tr>
-<td>KPOPS_CONNECT_HOST</td>
-<td></td>
+<td>KPOPS_KAFKA_CONNECT_URL</td>
+<td>http://localhost:8083</td>
 <td>False</td>
 <td>Address of Kafka Connect.</td>
-<td>kafka_connect_host</td>
+<td>url</td>
 </tr>
 <tr>
 <td>KPOPS_TIMEOUT</td>
@@ -610,18 +615,18 @@ <h2 id="config">Config<a class="headerlink" href="#config" title="Permanent link
 </span><span id="__span-0-9"><a id="__codelineno-0-9" name="__codelineno-0-9"></a><span class="c1"># Suffix your environment files with this value (e.g.</span>
 </span><span id="__span-0-10"><a id="__codelineno-0-10" name="__codelineno-0-10"></a><span class="c1"># defaults_development.yaml for environment=development).</span>
 </span><span id="__span-0-11"><a id="__codelineno-0-11" name="__codelineno-0-11"></a><span class="n">KPOPS_ENVIRONMENT</span> <span class="c1"># No default value, required</span>
-</span><span id="__span-0-12"><a id="__codelineno-0-12" name="__codelineno-0-12"></a><span class="c1"># brokers</span>
+</span><span id="__span-0-12"><a id="__codelineno-0-12" name="__codelineno-0-12"></a><span class="c1"># kafka_brokers</span>
 </span><span id="__span-0-13"><a id="__codelineno-0-13" name="__codelineno-0-13"></a><span class="c1"># The comma separated Kafka brokers address.</span>
 </span><span id="__span-0-14"><a id="__codelineno-0-14" name="__codelineno-0-14"></a><span class="n">KPOPS_KAFKA_BROKERS</span> <span class="c1"># No default value, required</span>
-</span><span id="__span-0-15"><a id="__codelineno-0-15" name="__codelineno-0-15"></a><span class="c1"># schema_registry_url</span>
+</span><span id="__span-0-15"><a id="__codelineno-0-15" name="__codelineno-0-15"></a><span class="c1"># url</span>
 </span><span id="__span-0-16"><a id="__codelineno-0-16" name="__codelineno-0-16"></a><span class="c1"># Address of the Schema Registry.</span>
-</span><span id="__span-0-17"><a id="__codelineno-0-17" name="__codelineno-0-17"></a><span class="n">KPOPS_SCHEMA_REGISTRY_URL</span> <span class="c1"># No default value, not required</span>
-</span><span id="__span-0-18"><a id="__codelineno-0-18" name="__codelineno-0-18"></a><span class="c1"># kafka_rest_host</span>
+</span><span id="__span-0-17"><a id="__codelineno-0-17" name="__codelineno-0-17"></a><span class="n">KPOPS_SCHEMA_REGISTRY_URL</span><span class="o">=</span><span class="n">http</span><span class="p">:</span><span class="o">//</span><span class="n">localhost</span><span class="p">:</span><span class="mi">8081</span>
+</span><span id="__span-0-18"><a id="__codelineno-0-18" name="__codelineno-0-18"></a><span class="c1"># url</span>
 </span><span id="__span-0-19"><a id="__codelineno-0-19" name="__codelineno-0-19"></a><span class="c1"># Address of the Kafka REST Proxy.</span>
-</span><span id="__span-0-20"><a id="__codelineno-0-20" name="__codelineno-0-20"></a><span class="n">KPOPS_REST_PROXY_HOST</span> <span class="c1"># No default value, not required</span>
-</span><span id="__span-0-21"><a id="__codelineno-0-21" name="__codelineno-0-21"></a><span class="c1"># kafka_connect_host</span>
+</span><span id="__span-0-20"><a id="__codelineno-0-20" name="__codelineno-0-20"></a><span class="n">KPOPS_KAFKA_REST_URL</span><span class="o">=</span><span class="n">http</span><span class="p">:</span><span class="o">//</span><span class="n">localhost</span><span class="p">:</span><span class="mi">8082</span>
+</span><span id="__span-0-21"><a id="__codelineno-0-21" name="__codelineno-0-21"></a><span class="c1"># url</span>
 </span><span id="__span-0-22"><a id="__codelineno-0-22" name="__codelineno-0-22"></a><span class="c1"># Address of Kafka Connect.</span>
-</span><span id="__span-0-23"><a id="__codelineno-0-23" name="__codelineno-0-23"></a><span class="n">KPOPS_CONNECT_HOST</span> <span class="c1"># No default value, not required</span>
+</span><span id="__span-0-23"><a id="__codelineno-0-23" name="__codelineno-0-23"></a><span class="n">KPOPS_KAFKA_CONNECT_URL</span><span class="o">=</span><span class="n">http</span><span class="p">:</span><span class="o">//</span><span class="n">localhost</span><span class="p">:</span><span class="mi">8083</span>
 </span><span id="__span-0-24"><a id="__codelineno-0-24" name="__codelineno-0-24"></a><span class="c1"># timeout</span>
 </span><span id="__span-0-25"><a id="__codelineno-0-25" name="__codelineno-0-25"></a><span class="c1"># The timeout in seconds that specifies when actions like deletion or</span>
 </span><span id="__span-0-26"><a id="__codelineno-0-26" name="__codelineno-0-26"></a><span class="c1"># deploy timeout.</span>
diff --git a/dev/user/core-concepts/variables/substitution/index.html b/dev/user/core-concepts/variables/substitution/index.html
index 64d1d2629..69463c45f 100644
--- a/dev/user/core-concepts/variables/substitution/index.html
+++ b/dev/user/core-concepts/variables/substitution/index.html
@@ -411,6 +411,11 @@
         Migrate from v1 to v2
       </a>
 </li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../../migration-guide/v2-v3/">
+        Migrate from v2 to v3
+      </a>
+</li>
 </ul>
 </nav>
 </li>
diff --git a/dev/user/examples/atm-fraud-pipeline/index.html b/dev/user/examples/atm-fraud-pipeline/index.html
index 1225dbb57..1e454cb91 100644
--- a/dev/user/examples/atm-fraud-pipeline/index.html
+++ b/dev/user/examples/atm-fraud-pipeline/index.html
@@ -458,6 +458,11 @@
         Migrate from v1 to v2
       </a>
 </li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../migration-guide/v2-v3/">
+        Migrate from v2 to v3
+      </a>
+</li>
 </ul>
 </nav>
 </li>
diff --git a/dev/user/getting-started/quick-start/index.html b/dev/user/getting-started/quick-start/index.html
index b00648d54..603c39f8b 100644
--- a/dev/user/getting-started/quick-start/index.html
+++ b/dev/user/getting-started/quick-start/index.html
@@ -463,6 +463,11 @@
         Migrate from v1 to v2
       </a>
 </li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../migration-guide/v2-v3/">
+        Migrate from v2 to v3
+      </a>
+</li>
 </ul>
 </nav>
 </li>
diff --git a/dev/user/getting-started/setup/index.html b/dev/user/getting-started/setup/index.html
index 558670e2a..b8be090f2 100644
--- a/dev/user/getting-started/setup/index.html
+++ b/dev/user/getting-started/setup/index.html
@@ -416,6 +416,11 @@
         Migrate from v1 to v2
       </a>
 </li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../migration-guide/v2-v3/">
+        Migrate from v2 to v3
+      </a>
+</li>
 </ul>
 </nav>
 </li>
diff --git a/dev/user/getting-started/teardown/index.html b/dev/user/getting-started/teardown/index.html
index 1d1687242..2a2cbe82a 100644
--- a/dev/user/getting-started/teardown/index.html
+++ b/dev/user/getting-started/teardown/index.html
@@ -411,6 +411,11 @@
         Migrate from v1 to v2
       </a>
 </li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../migration-guide/v2-v3/">
+        Migrate from v2 to v3
+      </a>
+</li>
 </ul>
 </nav>
 </li>
diff --git a/dev/user/migration-guide/v1-v2/index.html b/dev/user/migration-guide/v1-v2/index.html
index 051ff3ad7..8925feac0 100644
--- a/dev/user/migration-guide/v1-v2/index.html
+++ b/dev/user/migration-guide/v1-v2/index.html
@@ -7,7 +7,7 @@
 <meta content="width=device-width,initial-scale=1" name="viewport"/>
 <link href="https://bakdata.github.io/kpops/dev/user/migration-guide/v1-v2/" rel="canonical"/>
 <link href="../../core-concepts/variables/substitution/" rel="prev"/>
-<link href="../../references/cli-commands/" rel="next"/>
+<link href="../v2-v3/" rel="next"/>
 <link href="../../../assets/images/favicon.png" rel="icon"/>
 <meta content="mkdocs-1.4.2, mkdocs-material-9.1.1" name="generator"/>
 <title>Migrate from v1 to v2 - KPOps</title>
@@ -444,6 +444,11 @@
 </ul>
 </nav>
 </li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../v2-v3/">
+        Migrate from v2 to v3
+      </a>
+</li>
 </ul>
 </nav>
 </li>
@@ -609,7 +614,7 @@ <h2 id="derive-component-type-automatically-from-class-name"><a href="https://gi
 </span><span id="__span-1-3"><a id="__codelineno-1-3" name="__codelineno-1-3"></a><span class="w"> </span>   app:
 </span><span id="__span-1-4"><a id="__codelineno-1-4" name="__codelineno-1-4"></a><span class="w"> </span>       streams:
 </span><span id="__span-1-5"><a id="__codelineno-1-5" name="__codelineno-1-5"></a><span class="w"> </span>       outputTopic: output_topic
-</span><span id="__span-1-6"><a id="__codelineno-1-6" name="__codelineno-1-6"></a><span class="w"> </span>       extraOutputTopics: 
+</span><span id="__span-1-6"><a id="__codelineno-1-6" name="__codelineno-1-6"></a><span class="w"> </span>       extraOutputTopics:
 </span><span id="__span-1-7"><a id="__codelineno-1-7" name="__codelineno-1-7"></a><span class="w"> </span>           output_role1: output_topic1
 </span><span id="__span-1-8"><a id="__codelineno-1-8" name="__codelineno-1-8"></a><span class="w"> </span>           output_role2: output_topic2
 </span></code></pre></div></td></tr></table></div>
@@ -639,10 +644,10 @@ <h3 id="to-section">To section<a class="headerlink" href="#to-section" title="Pe
 </span><span id="__span-2-5"><a id="__codelineno-2-5" name="__codelineno-2-5"></a><span class="w"> </span>       role: "role-1"
 </span><span id="__span-2-6"><a id="__codelineno-2-6" name="__codelineno-2-6"></a><span class="w"> </span>       ...
 </span><span id="__span-2-7"><a id="__codelineno-2-7" name="__codelineno-2-7"></a><span class="w"> </span>     ${pipeline_name}-topic-2:
-</span><span id="__span-2-8"><a id="__codelineno-2-8" name="__codelineno-2-8"></a><span class="gd">-       type: output </span>
+</span><span id="__span-2-8"><a id="__codelineno-2-8" name="__codelineno-2-8"></a><span class="gd">-       type: output</span>
 </span><span id="__span-2-9"><a id="__codelineno-2-9" name="__codelineno-2-9"></a><span class="w"> </span>       ...
 </span><span id="__span-2-10"><a id="__codelineno-2-10" name="__codelineno-2-10"></a><span class="w"> </span>     ${pipeline_name}-topic-3:
-</span><span id="__span-2-11"><a id="__codelineno-2-11" name="__codelineno-2-11"></a><span class="w"> </span>        type: error 
+</span><span id="__span-2-11"><a id="__codelineno-2-11" name="__codelineno-2-11"></a><span class="w"> </span>        type: error
 </span><span id="__span-2-12"><a id="__codelineno-2-12" name="__codelineno-2-12"></a><span class="w"> </span>        ...
 </span></code></pre></div></td></tr></table></div>
 <h3 id="from-section">From section<a class="headerlink" href="#from-section" title="Permanent link">¶</a></h3>
@@ -680,11 +685,11 @@ <h3 id="from-section">From section<a class="headerlink" href="#from-section" tit
 </span><span id="__span-3-8"><a id="__codelineno-3-8" name="__codelineno-3-8"></a><span class="w"> </span>       role: topic-role
 </span><span id="__span-3-9"><a id="__codelineno-3-9" name="__codelineno-3-9"></a><span class="w"> </span>       ...
 </span><span id="__span-3-10"><a id="__codelineno-3-10" name="__codelineno-3-10"></a><span class="w"> </span>     ${pipeline_name}-input-pattern-topic:
-</span><span id="__span-3-11"><a id="__codelineno-3-11" name="__codelineno-3-11"></a><span class="gd">-       type: input-pattern </span>
+</span><span id="__span-3-11"><a id="__codelineno-3-11" name="__codelineno-3-11"></a><span class="gd">-       type: input-pattern</span>
 </span><span id="__span-3-12"><a id="__codelineno-3-12" name="__codelineno-3-12"></a><span class="gi">+       type: pattern</span>
 </span><span id="__span-3-13"><a id="__codelineno-3-13" name="__codelineno-3-13"></a><span class="w"> </span>       ...
 </span><span id="__span-3-14"><a id="__codelineno-3-14" name="__codelineno-3-14"></a><span class="w"> </span>     ${pipeline_name}-extra-pattern-topic:
-</span><span id="__span-3-15"><a id="__codelineno-3-15" name="__codelineno-3-15"></a><span class="gd">-       type: extra-pattern </span>
+</span><span id="__span-3-15"><a id="__codelineno-3-15" name="__codelineno-3-15"></a><span class="gd">-       type: extra-pattern</span>
 </span><span id="__span-3-16"><a id="__codelineno-3-16" name="__codelineno-3-16"></a><span class="gi">+       type: pattern</span>
 </span><span id="__span-3-17"><a id="__codelineno-3-17" name="__codelineno-3-17"></a><span class="w"> </span>       role: some-role
 </span><span id="__span-3-18"><a id="__codelineno-3-18" name="__codelineno-3-18"></a><span class="w"> </span>       ...
@@ -735,15 +740,15 @@ <h2 id="remove-camel-case-conversion-of-internal-models"><a href="https://github
 <span class="normal"><a href="#__codelineno-4-42">42</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-4-1"><a id="__codelineno-4-1" name="__codelineno-4-1"></a><span class="w">...</span>
 </span><span id="__span-4-2"><a id="__codelineno-4-2" name="__codelineno-4-2"></a><span class="w">type: streams-app</span>
 </span><span id="__span-4-3"><a id="__codelineno-4-3" name="__codelineno-4-3"></a><span class="w"> </span> name: streams-app
-</span><span id="__span-4-4"><a id="__codelineno-4-4" name="__codelineno-4-4"></a><span class="w"> </span> namespace: namespace 
-</span><span id="__span-4-5"><a id="__codelineno-4-5" name="__codelineno-4-5"></a><span class="w"> </span> app: 
+</span><span id="__span-4-4"><a id="__codelineno-4-4" name="__codelineno-4-4"></a><span class="w"> </span> namespace: namespace
+</span><span id="__span-4-5"><a id="__codelineno-4-5" name="__codelineno-4-5"></a><span class="w"> </span> app:
 </span><span id="__span-4-6"><a id="__codelineno-4-6" name="__codelineno-4-6"></a><span class="w"> </span>   streams:
-</span><span id="__span-4-7"><a id="__codelineno-4-7" name="__codelineno-4-7"></a><span class="w"> </span>     brokers: ${brokers} 
+</span><span id="__span-4-7"><a id="__codelineno-4-7" name="__codelineno-4-7"></a><span class="w"> </span>     brokers: ${brokers}
 </span><span id="__span-4-8"><a id="__codelineno-4-8" name="__codelineno-4-8"></a><span class="w"> </span>     schemaRegistryUrl: ${schema_registry_url}
 </span><span id="__span-4-9"><a id="__codelineno-4-9" name="__codelineno-4-9"></a><span class="w"> </span>    autoscaling:
 </span><span id="__span-4-10"><a id="__codelineno-4-10" name="__codelineno-4-10"></a><span class="w"> </span>     consumerGroup: consumer-group
 </span><span id="__span-4-11"><a id="__codelineno-4-11" name="__codelineno-4-11"></a><span class="w"> </span>     lagThreshold: 0
-</span><span id="__span-4-12"><a id="__codelineno-4-12" name="__codelineno-4-12"></a><span class="w"> </span>     enabled: false 
+</span><span id="__span-4-12"><a id="__codelineno-4-12" name="__codelineno-4-12"></a><span class="w"> </span>     enabled: false
 </span><span id="__span-4-13"><a id="__codelineno-4-13" name="__codelineno-4-13"></a><span class="w"> </span>     pollingInterval: 30
 </span><span id="__span-4-14"><a id="__codelineno-4-14" name="__codelineno-4-14"></a>
 </span><span id="__span-4-15"><a id="__codelineno-4-15" name="__codelineno-4-15"></a><span class="w"> </span> to:
@@ -766,8 +771,8 @@ <h2 id="remove-camel-case-conversion-of-internal-models"><a href="https://github
 </span><span id="__span-4-32"><a id="__codelineno-4-32" name="__codelineno-4-32"></a><span class="gd">-   repositoryName: bakdata-streams-bootstrap</span>
 </span><span id="__span-4-33"><a id="__codelineno-4-33" name="__codelineno-4-33"></a><span class="gi">+   repository_name: bakdata-streams-bootstrap</span>
 </span><span id="__span-4-34"><a id="__codelineno-4-34" name="__codelineno-4-34"></a><span class="w"> </span>   url: https://bakdata.github.io/streams-bootstrap/
-</span><span id="__span-4-35"><a id="__codelineno-4-35" name="__codelineno-4-35"></a><span class="gd">-   repoAuthFlags: </span>
-</span><span id="__span-4-36"><a id="__codelineno-4-36" name="__codelineno-4-36"></a><span class="gi">+   repo_auth_flags: </span>
+</span><span id="__span-4-35"><a id="__codelineno-4-35" name="__codelineno-4-35"></a><span class="gd">-   repoAuthFlags:</span>
+</span><span id="__span-4-36"><a id="__codelineno-4-36" name="__codelineno-4-36"></a><span class="gi">+   repo_auth_flags:</span>
 </span><span id="__span-4-37"><a id="__codelineno-4-37" name="__codelineno-4-37"></a><span class="w"> </span>     username: user
 </span><span id="__span-4-38"><a id="__codelineno-4-38" name="__codelineno-4-38"></a><span class="w"> </span>     password: pass
 </span><span id="__span-4-39"><a id="__codelineno-4-39" name="__codelineno-4-39"></a><span class="w"> </span>     ca_file: /home/user/path/to/ca-file
@@ -802,9 +807,9 @@ <h4 id="configyaml">config.yaml<a class="headerlink" href="#configyaml" title="P
 <span class="normal"><a href="#__codelineno-6-6">6</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-6-1"><a id="__codelineno-6-1" name="__codelineno-6-1"></a><span class="w"> </span> environment: development
 </span><span id="__span-6-2"><a id="__codelineno-6-2" name="__codelineno-6-2"></a><span class="gd">- broker: "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092"</span>
 </span><span id="__span-6-3"><a id="__codelineno-6-3" name="__codelineno-6-3"></a><span class="gi">+ brokers: "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092"</span>
-</span><span id="__span-6-4"><a id="__codelineno-6-4" name="__codelineno-6-4"></a><span class="w"> </span>kafka_connect_host: "http://localhost:8083"
-</span><span id="__span-6-5"><a id="__codelineno-6-5" name="__codelineno-6-5"></a><span class="w"> </span>kafka_rest_host: "http://localhost:8082"
-</span><span id="__span-6-6"><a id="__codelineno-6-6" name="__codelineno-6-6"></a><span class="w"> </span>schema_registry_url: "http://localhost:8081"
+</span><span id="__span-6-4"><a id="__codelineno-6-4" name="__codelineno-6-4"></a><span class="w"> </span> kafka_connect_host: "http://localhost:8083"
+</span><span id="__span-6-5"><a id="__codelineno-6-5" name="__codelineno-6-5"></a><span class="w"> </span> kafka_rest_host: "http://localhost:8082"
+</span><span id="__span-6-6"><a id="__codelineno-6-6" name="__codelineno-6-6"></a><span class="w"> </span> schema_registry_url: "http://localhost:8081"
 </span></code></pre></div></td></tr></table></div>
 <h4 id="pipelineyaml-and-defaultyaml">pipeline.yaml and default.yaml<a class="headerlink" href="#pipelineyaml-and-defaultyaml" title="Permanent link">¶</a></h4>
 <p>The variable is now called <code>brokers</code>.</p>
diff --git a/dev/user/migration-guide/v2-v3/index.html b/dev/user/migration-guide/v2-v3/index.html
new file mode 100644
index 000000000..fcb4b8af1
--- /dev/null
+++ b/dev/user/migration-guide/v2-v3/index.html
@@ -0,0 +1,641 @@
+
+<!DOCTYPE html>
+
+<html class="no-js" lang="en">
+<head>
+<meta charset="utf-8"/>
+<meta content="width=device-width,initial-scale=1" name="viewport"/>
+<link href="https://bakdata.github.io/kpops/dev/user/migration-guide/v2-v3/" rel="canonical"/>
+<link href="../v1-v2/" rel="prev"/>
+<link href="../../references/cli-commands/" rel="next"/>
+<link href="../../../assets/images/favicon.png" rel="icon"/>
+<meta content="mkdocs-1.4.2, mkdocs-material-9.1.1" name="generator"/>
+<title>Migrate from v2 to v3 - KPOps</title>
+<link href="../../../assets/stylesheets/main.402914a4.min.css" rel="stylesheet"/>
+<link href="../../../assets/stylesheets/palette.a0c5b2b5.min.css" rel="stylesheet"/>
+<link crossorigin="" href="https://fonts.gstatic.com" rel="preconnect"/>
+<link href="https://fonts.googleapis.com/css?family=SF+Pro:300,300i,400,400i,700,700i%7CDejaVu+Sans+Mono:400,400i,700,700i&amp;display=fallback" rel="stylesheet"/>
+<style>:root{--md-text-font:"SF Pro";--md-code-font:"DejaVu Sans Mono"}</style>
+<link href="../../../stylesheets/extra.css" rel="stylesheet"/>
+<script>__md_scope=new URL("../../..",location),__md_hash=e=>[...e].reduce((e,_)=>(e<<5)-e+_.charCodeAt(0),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
+<link href="../../../assets/stylesheets/glightbox.min.css" rel="stylesheet"/><style>
+            html.glightbox-open { overflow: initial; height: 100%; }
+            .gslide-title { margin-top: 0px; user-select: text; }
+            .gslide-desc { color: #666; user-select: text; }
+            .gslide-image img { background: white; }
+            
+                .gscrollbar-fixer { padding-right: 15px; }
+                .gdesc-inner { font-size: 0.75rem; }
+                body[data-md-color-scheme="slate"] .gdesc-inner { background: var(--md-default-bg-color);}
+                body[data-md-color-scheme="slate"] .gslide-title { color: var(--md-default-fg-color);}
+                body[data-md-color-scheme="slate"] .gslide-desc { color: var(--md-default-fg-color);}
+                </style><script src="../../../assets/javascripts/glightbox.min.js"></script></head>
+<body data-md-color-accent="" data-md-color-primary="" data-md-color-scheme="default" dir="ltr">
+<input autocomplete="off" class="md-toggle" data-md-toggle="drawer" id="__drawer" type="checkbox"/>
+<input autocomplete="off" class="md-toggle" data-md-toggle="search" id="__search" type="checkbox"/>
+<label class="md-overlay" for="__drawer"></label>
+<div data-md-component="skip">
+<a class="md-skip" href="#migrate-from-v2-to-v3">
+          Skip to content
+        </a>
+</div>
+<div data-md-component="announce">
+</div>
+<div data-md-color-scheme="default" data-md-component="outdated" hidden="">
+</div>
+<header class="md-header" data-md-component="header">
+<nav aria-label="Header" class="md-header__inner md-grid">
+<a aria-label="KPOps" class="md-header__button md-logo" data-md-component="logo" href="../../.." title="KPOps">
+<svg viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54Z"></path></svg>
+</a>
+<label class="md-header__button md-icon" for="__drawer">
+<svg viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path d="M3 6h18v2H3V6m0 5h18v2H3v-2m0 5h18v2H3v-2Z"></path></svg>
+</label>
+<div class="md-header__title" data-md-component="header-title">
+<div class="md-header__ellipsis">
+<div class="md-header__topic">
+<span class="md-ellipsis">
+            KPOps
+          </span>
+</div>
+<div class="md-header__topic" data-md-component="header-topic">
+<span class="md-ellipsis">
+            
+              Migrate from v2 to v3
+            
+          </span>
+</div>
+</div>
+</div>
+<label class="md-header__button md-icon" for="__search">
+<svg viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"></path></svg>
+</label>
+<div class="md-search" data-md-component="search" role="dialog">
+<label class="md-search__overlay" for="__search"></label>
+<div class="md-search__inner" role="search">
+<form class="md-search__form" name="search">
+<input aria-label="Search" autocapitalize="off" autocomplete="off" autocorrect="off" class="md-search__input" data-md-component="search-query" name="query" placeholder="Search" required="" spellcheck="false" type="text"/>
+<label class="md-search__icon md-icon" for="__search">
+<svg viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"></path></svg>
+<svg viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12Z"></path></svg>
+</label>
+<nav aria-label="Search" class="md-search__options">
+<a aria-label="Share" class="md-search__icon md-icon" data-clipboard="" data-clipboard-text="" data-md-component="search-share" href="javascript:void(0)" tabindex="-1" title="Share">
+<svg viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path d="M18 16.08c-.76 0-1.44.3-1.96.77L8.91 12.7c.05-.23.09-.46.09-.7 0-.24-.04-.47-.09-.7l7.05-4.11c.54.5 1.25.81 2.04.81a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3c0 .24.04.47.09.7L8.04 9.81C7.5 9.31 6.79 9 6 9a3 3 0 0 0-3 3 3 3 0 0 0 3 3c.79 0 1.5-.31 2.04-.81l7.12 4.15c-.05.21-.08.43-.08.66 0 1.61 1.31 2.91 2.92 2.91 1.61 0 2.92-1.3 2.92-2.91A2.92 2.92 0 0 0 18 16.08Z"></path></svg>
+</a>
+<button aria-label="Clear" class="md-search__icon md-icon" tabindex="-1" title="Clear" type="reset">
+<svg viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12 19 6.41Z"></path></svg>
+</button>
+</nav>
+<div class="md-search__suggest" data-md-component="search-suggest"></div>
+</form>
+<div class="md-search__output">
+<div class="md-search__scrollwrap" data-md-scrollfix="">
+<div class="md-search-result" data-md-component="search-result">
+<div class="md-search-result__meta">
+            Initializing search
+          </div>
+<ol class="md-search-result__list" role="presentation"></ol>
+</div>
+</div>
+</div>
+</div>
+</div>
+<div class="md-header__source">
+<a class="md-source" data-md-component="source" href="https://github.com/bakdata/kpops/" title="Go to repository">
+<div class="md-source__icon md-icon">
+<svg viewbox="0 0 448 512" xmlns="http://www.w3.org/2000/svg"><!--! Font Awesome Free 6.3.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"></path></svg>
+</div>
+<div class="md-source__repository">
+    GitHub
+  </div>
+</a>
+</div>
+</nav>
+</header>
+<div class="md-container" data-md-component="container">
+<nav aria-label="Tabs" class="md-tabs" data-md-component="tabs">
+<div class="md-grid">
+<ul class="md-tabs__list">
+<li class="md-tabs__item">
+<a class="md-tabs__link" href="../../..">
+        Home
+      </a>
+</li>
+<li class="md-tabs__item">
+<a class="md-tabs__link md-tabs__link--active" href="../../what-is-kpops/">
+        User Guide
+      </a>
+</li>
+<li class="md-tabs__item">
+<a class="md-tabs__link" href="../../../developer/auto-generation/">
+        Developer Guide
+      </a>
+</li>
+</ul>
+</div>
+</nav>
+<main class="md-main" data-md-component="main">
+<div class="md-main__inner md-grid">
+<div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation">
+<div class="md-sidebar__scrollwrap">
+<div class="md-sidebar__inner">
+<nav aria-label="Navigation" class="md-nav md-nav--primary md-nav--lifted" data-md-level="0">
+<label class="md-nav__title" for="__drawer">
+<a aria-label="KPOps" class="md-nav__button md-logo" data-md-component="logo" href="../../.." title="KPOps">
+<svg viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54Z"></path></svg>
+</a>
+    KPOps
+  </label>
+<div class="md-nav__source">
+<a class="md-source" data-md-component="source" href="https://github.com/bakdata/kpops/" title="Go to repository">
+<div class="md-source__icon md-icon">
+<svg viewbox="0 0 448 512" xmlns="http://www.w3.org/2000/svg"><!--! Font Awesome Free 6.3.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"></path></svg>
+</div>
+<div class="md-source__repository">
+    GitHub
+  </div>
+</a>
+</div>
+<ul class="md-nav__list" data-md-scrollfix="">
+<li class="md-nav__item md-nav__item--nested">
+<input class="md-nav__toggle md-toggle" id="__nav_1" type="checkbox"/>
+<div class="md-nav__link md-nav__link--index">
+<a href="../../..">Home</a>
+</div>
+<nav aria-expanded="false" aria-labelledby="__nav_1_label" class="md-nav" data-md-level="1">
+<label class="md-nav__title" for="__nav_1">
+<span class="md-nav__icon md-icon"></span>
+          Home
+        </label>
+<ul class="md-nav__list" data-md-scrollfix="">
+</ul>
+</nav>
+</li>
+<li class="md-nav__item md-nav__item--active md-nav__item--nested">
+<input checked="" class="md-nav__toggle md-toggle" id="__nav_2" type="checkbox"/>
+<label class="md-nav__link" for="__nav_2" id="__nav_2_label" tabindex="0">
+          User Guide
+          <span class="md-nav__icon md-icon"></span>
+</label>
+<nav aria-expanded="true" aria-labelledby="__nav_2_label" class="md-nav" data-md-level="1">
+<label class="md-nav__title" for="__nav_2">
+<span class="md-nav__icon md-icon"></span>
+          User Guide
+        </label>
+<ul class="md-nav__list" data-md-scrollfix="">
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../what-is-kpops/">
+        What is KPOps
+      </a>
+</li>
+<li class="md-nav__item md-nav__item--section md-nav__item--nested">
+<input class="md-nav__toggle md-toggle" id="__nav_2_2" type="checkbox"/>
+<label class="md-nav__link" for="__nav_2_2" id="__nav_2_2_label" tabindex="0">
+          Getting Started
+          <span class="md-nav__icon md-icon"></span>
+</label>
+<nav aria-expanded="false" aria-labelledby="__nav_2_2_label" class="md-nav" data-md-level="2">
+<label class="md-nav__title" for="__nav_2_2">
+<span class="md-nav__icon md-icon"></span>
+          Getting Started
+        </label>
+<ul class="md-nav__list" data-md-scrollfix="">
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../getting-started/setup/">
+        Setup
+      </a>
+</li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../getting-started/quick-start/">
+        Quick start
+      </a>
+</li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../getting-started/teardown/">
+        Teardown
+      </a>
+</li>
+</ul>
+</nav>
+</li>
+<li class="md-nav__item md-nav__item--section md-nav__item--nested">
+<input class="md-nav__toggle md-toggle" id="__nav_2_3" type="checkbox"/>
+<label class="md-nav__link" for="__nav_2_3" id="__nav_2_3_label" tabindex="0">
+          Examples
+          <span class="md-nav__icon md-icon"></span>
+</label>
+<nav aria-expanded="false" aria-labelledby="__nav_2_3_label" class="md-nav" data-md-level="2">
+<label class="md-nav__title" for="__nav_2_3">
+<span class="md-nav__icon md-icon"></span>
+          Examples
+        </label>
+<ul class="md-nav__list" data-md-scrollfix="">
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../examples/atm-fraud-pipeline/">
+        ATM Fraud detection pipeline
+      </a>
+</li>
+</ul>
+</nav>
+</li>
+<li class="md-nav__item md-nav__item--section md-nav__item--nested">
+<input class="md-nav__toggle md-toggle" id="__nav_2_4" type="checkbox"/>
+<label class="md-nav__link" for="__nav_2_4" id="__nav_2_4_label" tabindex="0">
+          Core Concepts
+          <span class="md-nav__icon md-icon"></span>
+</label>
+<nav aria-expanded="false" aria-labelledby="__nav_2_4_label" class="md-nav" data-md-level="2">
+<label class="md-nav__title" for="__nav_2_4">
+<span class="md-nav__icon md-icon"></span>
+          Core Concepts
+        </label>
+<ul class="md-nav__list" data-md-scrollfix="">
+<li class="md-nav__item md-nav__item--nested">
+<input class="md-nav__toggle md-toggle" id="__nav_2_4_1" type="checkbox"/>
+<label class="md-nav__link" for="__nav_2_4_1" id="__nav_2_4_1_label" tabindex="0">
+          Components
+          <span class="md-nav__icon md-icon"></span>
+</label>
+<nav aria-expanded="false" aria-labelledby="__nav_2_4_1_label" class="md-nav" data-md-level="3">
+<label class="md-nav__title" for="__nav_2_4_1">
+<span class="md-nav__icon md-icon"></span>
+          Components
+        </label>
+<ul class="md-nav__list" data-md-scrollfix="">
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../core-concepts/components/overview/">
+        Overview
+      </a>
+</li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../core-concepts/components/kubernetes-app/">
+        KubernetesApp
+      </a>
+</li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../core-concepts/components/kafka-app/">
+        KafkaApp
+      </a>
+</li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../core-concepts/components/streams-app/">
+        StreamsApp
+      </a>
+</li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../core-concepts/components/producer-app/">
+        ProducerApp
+      </a>
+</li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../core-concepts/components/kafka-connector/">
+        KafkaConnector
+      </a>
+</li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../core-concepts/components/kafka-sink-connector/">
+        KafkaSinkConnector
+      </a>
+</li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../core-concepts/components/kafka-source-connector/">
+        KafkaSourceConnector
+      </a>
+</li>
+</ul>
+</nav>
+</li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../core-concepts/config/">
+        Config
+      </a>
+</li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../core-concepts/defaults/">
+        Defaults
+      </a>
+</li>
+<li class="md-nav__item md-nav__item--nested">
+<input class="md-nav__toggle md-toggle" id="__nav_2_4_4" type="checkbox"/>
+<label class="md-nav__link" for="__nav_2_4_4" id="__nav_2_4_4_label" tabindex="0">
+          Variables
+          <span class="md-nav__icon md-icon"></span>
+</label>
+<nav aria-expanded="false" aria-labelledby="__nav_2_4_4_label" class="md-nav" data-md-level="3">
+<label class="md-nav__title" for="__nav_2_4_4">
+<span class="md-nav__icon md-icon"></span>
+          Variables
+        </label>
+<ul class="md-nav__list" data-md-scrollfix="">
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../core-concepts/variables/environment_variables/">
+        Environment variables
+      </a>
+</li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../core-concepts/variables/substitution/">
+        Substitution
+      </a>
+</li>
+</ul>
+</nav>
+</li>
+</ul>
+</nav>
+</li>
+<li class="md-nav__item md-nav__item--active md-nav__item--section md-nav__item--nested">
+<input checked="" class="md-nav__toggle md-toggle" id="__nav_2_5" type="checkbox"/>
+<label class="md-nav__link" for="__nav_2_5" id="__nav_2_5_label" tabindex="0">
+          References
+          <span class="md-nav__icon md-icon"></span>
+</label>
+<nav aria-expanded="true" aria-labelledby="__nav_2_5_label" class="md-nav" data-md-level="2">
+<label class="md-nav__title" for="__nav_2_5">
+<span class="md-nav__icon md-icon"></span>
+          References
+        </label>
+<ul class="md-nav__list" data-md-scrollfix="">
+<li class="md-nav__item md-nav__item--active md-nav__item--nested">
+<input checked="" class="md-nav__toggle md-toggle" id="__nav_2_5_1" type="checkbox"/>
+<label class="md-nav__link" for="__nav_2_5_1" id="__nav_2_5_1_label" tabindex="0">
+          Migration guide
+          <span class="md-nav__icon md-icon"></span>
+</label>
+<nav aria-expanded="true" aria-labelledby="__nav_2_5_1_label" class="md-nav" data-md-level="3">
+<label class="md-nav__title" for="__nav_2_5_1">
+<span class="md-nav__icon md-icon"></span>
+          Migration guide
+        </label>
+<ul class="md-nav__list" data-md-scrollfix="">
+<li class="md-nav__item">
+<a class="md-nav__link" href="../v1-v2/">
+        Migrate from v1 to v2
+      </a>
+</li>
+<li class="md-nav__item md-nav__item--active">
+<input class="md-nav__toggle md-toggle" id="__toc" type="checkbox"/>
+<label class="md-nav__link md-nav__link--active" for="__toc">
+          Migrate from v2 to v3
+          <span class="md-nav__icon md-icon"></span>
+</label>
+<a class="md-nav__link md-nav__link--active" href="./">
+        Migrate from v2 to v3
+      </a>
+<nav aria-label="Table of contents" class="md-nav md-nav--secondary">
+<label class="md-nav__title" for="__toc">
+<span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix="">
+<li class="md-nav__item">
+<a class="md-nav__link" href="#make-kafka-rest-proxy-kafka-connect-hosts-default-and-improve-schema-registry-config">
+    Make Kafka REST Proxy &amp; Kafka Connect hosts default and improve Schema Registry config
+  </a>
+<nav aria-label="Make Kafka REST Proxy &amp; Kafka Connect hosts default and improve Schema Registry config" class="md-nav">
+<ul class="md-nav__list">
+<li class="md-nav__item">
+<a class="md-nav__link" href="#configyaml">
+    config.yaml
+  </a>
+</li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="#pipelineyaml-and-defaultyaml">
+    pipeline.yaml and default.yaml
+  </a>
+</li>
+</ul>
+</nav>
+</li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="#move-github-action-to-repsitory-root">
+    Move GitHub action to repsitory root
+  </a>
+</li>
+</ul>
+</nav>
+</li>
+</ul>
+</nav>
+</li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../references/cli-commands/">
+        CLI usage
+      </a>
+</li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../references/editor-integration/">
+        Editor integration
+      </a>
+</li>
+<li class="md-nav__item md-nav__item--nested">
+<input class="md-nav__toggle md-toggle" id="__nav_2_5_4" type="checkbox"/>
+<label class="md-nav__link" for="__nav_2_5_4" id="__nav_2_5_4_label" tabindex="0">
+          CI integration
+          <span class="md-nav__icon md-icon"></span>
+</label>
+<nav aria-expanded="false" aria-labelledby="__nav_2_5_4_label" class="md-nav" data-md-level="3">
+<label class="md-nav__title" for="__nav_2_5_4">
+<span class="md-nav__icon md-icon"></span>
+          CI integration
+        </label>
+<ul class="md-nav__list" data-md-scrollfix="">
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../references/ci-integration/github-actions/">
+        GitHub Actions
+      </a>
+</li>
+</ul>
+</nav>
+</li>
+</ul>
+</nav>
+</li>
+</ul>
+</nav>
+</li>
+<li class="md-nav__item md-nav__item--nested">
+<input class="md-nav__toggle md-toggle" id="__nav_3" type="checkbox"/>
+<label class="md-nav__link" for="__nav_3" id="__nav_3_label" tabindex="0">
+          Developer Guide
+          <span class="md-nav__icon md-icon"></span>
+</label>
+<nav aria-expanded="false" aria-labelledby="__nav_3_label" class="md-nav" data-md-level="1">
+<label class="md-nav__title" for="__nav_3">
+<span class="md-nav__icon md-icon"></span>
+          Developer Guide
+        </label>
+<ul class="md-nav__list" data-md-scrollfix="">
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../../developer/auto-generation/">
+        Auto generation
+      </a>
+</li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../../developer/formatting/">
+        Formatting
+      </a>
+</li>
+</ul>
+</nav>
+</li>
+</ul>
+</nav>
+</div>
+</div>
+</div>
+<div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc">
+<div class="md-sidebar__scrollwrap">
+<div class="md-sidebar__inner">
+<nav aria-label="Table of contents" class="md-nav md-nav--secondary">
+<label class="md-nav__title" for="__toc">
+<span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix="">
+<li class="md-nav__item">
+<a class="md-nav__link" href="#make-kafka-rest-proxy-kafka-connect-hosts-default-and-improve-schema-registry-config">
+    Make Kafka REST Proxy &amp; Kafka Connect hosts default and improve Schema Registry config
+  </a>
+<nav aria-label="Make Kafka REST Proxy &amp; Kafka Connect hosts default and improve Schema Registry config" class="md-nav">
+<ul class="md-nav__list">
+<li class="md-nav__item">
+<a class="md-nav__link" href="#configyaml">
+    config.yaml
+  </a>
+</li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="#pipelineyaml-and-defaultyaml">
+    pipeline.yaml and default.yaml
+  </a>
+</li>
+</ul>
+</nav>
+</li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="#move-github-action-to-repsitory-root">
+    Move GitHub action to repsitory root
+  </a>
+</li>
+</ul>
+</nav>
+</div>
+</div>
+</div>
+<div class="md-content" data-md-component="content">
+<article class="md-content__inner md-typeset">
+<h1 id="migrate-from-v2-to-v3">Migrate from V2 to V3<a class="headerlink" href="#migrate-from-v2-to-v3" title="Permanent link">¶</a></h1>
+<h2 id="make-kafka-rest-proxy-kafka-connect-hosts-default-and-improve-schema-registry-config"><a href="https://github.com/bakdata/kpops/pull/354">Make Kafka REST Proxy &amp; Kafka Connect hosts default and improve Schema Registry config</a><a class="headerlink" href="#make-kafka-rest-proxy-kafka-connect-hosts-default-and-improve-schema-registry-config" title="Permanent link">¶</a></h2>
+<p>The breaking changes target the <code>config.yaml</code> file:</p>
+<ul>
+<li>
+<p>The <code>schema_registry_url</code> is replaced with <code>schema_registry.url</code> (default <code>http://localhost:8081</code>) and <code>schema_registry.enabled</code> (default <code>false</code>).</p>
+</li>
+<li>
+<p><code>kafka_rest_host</code> is renamed to <code>kafka_rest.url</code> (default <code>http://localhost:8082</code>).</p>
+</li>
+<li>
+<p><code>kafka_connect_host</code> is replaced with <code>kafka_connect.url</code> (default <code>http://localhost:8083</code>).</p>
+</li>
+<li>
+<p><code>brokers</code> is renamed to <code>kafka_brokers</code>.</p>
+</li>
+</ul>
+<p>The environment variable names of these config fields changed respectively. Please refer to the <a href="../../core-concepts/variables/environment_variables/">environment variables documentation page</a> to see the newest changes.</p>
+<h4 id="configyaml">config.yaml<a class="headerlink" href="#configyaml" title="Permanent link">¶</a></h4>
+<div class="language-diff highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1"> 1</a></span>
+<span class="normal"><a href="#__codelineno-0-2"> 2</a></span>
+<span class="normal"><a href="#__codelineno-0-3"> 3</a></span>
+<span class="normal"><a href="#__codelineno-0-4"> 4</a></span>
+<span class="normal"><a href="#__codelineno-0-5"> 5</a></span>
+<span class="normal"><a href="#__codelineno-0-6"> 6</a></span>
+<span class="normal"><a href="#__codelineno-0-7"> 7</a></span>
+<span class="normal"><a href="#__codelineno-0-8"> 8</a></span>
+<span class="normal"><a href="#__codelineno-0-9"> 9</a></span>
+<span class="normal"><a href="#__codelineno-0-10">10</a></span>
+<span class="normal"><a href="#__codelineno-0-11">11</a></span>
+<span class="normal"><a href="#__codelineno-0-12">12</a></span>
+<span class="normal"><a href="#__codelineno-0-13">13</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-1"><a id="__codelineno-0-1" name="__codelineno-0-1"></a><span class="w"> </span> environment: development
+</span><span id="__span-0-2"><a id="__codelineno-0-2" name="__codelineno-0-2"></a><span class="gd">- brokers: "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092"</span>
+</span><span id="__span-0-3"><a id="__codelineno-0-3" name="__codelineno-0-3"></a><span class="gd">- kafka_rest_host: "http://my-custom-rest.url:8082"</span>
+</span><span id="__span-0-4"><a id="__codelineno-0-4" name="__codelineno-0-4"></a><span class="gd">- kafka_connect_host: "http://my-custom-connect.url:8083"</span>
+</span><span id="__span-0-5"><a id="__codelineno-0-5" name="__codelineno-0-5"></a><span class="gd">- schema_registry_url: "http://my-custom-sr.url:8081"</span>
+</span><span id="__span-0-6"><a id="__codelineno-0-6" name="__codelineno-0-6"></a><span class="gi">+ kafka_brokers: "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092"</span>
+</span><span id="__span-0-7"><a id="__codelineno-0-7" name="__codelineno-0-7"></a><span class="gi">+ kafka_rest:</span>
+</span><span id="__span-0-8"><a id="__codelineno-0-8" name="__codelineno-0-8"></a><span class="gi">+   url: "http://my-custom-rest.url:8082"</span>
+</span><span id="__span-0-9"><a id="__codelineno-0-9" name="__codelineno-0-9"></a><span class="gi">+ kafka_connect:</span>
+</span><span id="__span-0-10"><a id="__codelineno-0-10" name="__codelineno-0-10"></a><span class="gi">+   url: "http://my-custom-connect.url:8083"</span>
+</span><span id="__span-0-11"><a id="__codelineno-0-11" name="__codelineno-0-11"></a><span class="gi">+ schema_registry:</span>
+</span><span id="__span-0-12"><a id="__codelineno-0-12" name="__codelineno-0-12"></a><span class="gi">+   enabled: true</span>
+</span><span id="__span-0-13"><a id="__codelineno-0-13" name="__codelineno-0-13"></a><span class="gi">+   url: "http://my-custom-sr.url:8081"</span>
+</span></code></pre></div></td></tr></table></div>
+<h4 id="pipelineyaml-and-defaultyaml">pipeline.yaml and default.yaml<a class="headerlink" href="#pipelineyaml-and-defaultyaml" title="Permanent link">¶</a></h4>
+<p>The variable is now called <code>kafka_brokers</code>.</p>
+<div class="language-diff highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-1-1">1</a></span>
+<span class="normal"><a href="#__codelineno-1-2">2</a></span>
+<span class="normal"><a href="#__codelineno-1-3">3</a></span>
+<span class="normal"><a href="#__codelineno-1-4">4</a></span>
+<span class="normal"><a href="#__codelineno-1-5">5</a></span>
+<span class="normal"><a href="#__codelineno-1-6">6</a></span>
+<span class="normal"><a href="#__codelineno-1-7">7</a></span>
+<span class="normal"><a href="#__codelineno-1-8">8</a></span>
+<span class="normal"><a href="#__codelineno-1-9">9</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-1-1"><a id="__codelineno-1-1" name="__codelineno-1-1"></a><span class="w">...</span>
+</span><span id="__span-1-2"><a id="__codelineno-1-2" name="__codelineno-1-2"></a><span class="w"> </span> app:
+</span><span id="__span-1-3"><a id="__codelineno-1-3" name="__codelineno-1-3"></a><span class="w"> </span>   streams:
+</span><span id="__span-1-4"><a id="__codelineno-1-4" name="__codelineno-1-4"></a><span class="gd">-     brokers: ${brokers}</span>
+</span><span id="__span-1-5"><a id="__codelineno-1-5" name="__codelineno-1-5"></a><span class="gi">+     brokers: ${kafka_brokers}</span>
+</span><span id="__span-1-6"><a id="__codelineno-1-6" name="__codelineno-1-6"></a><span class="w"> </span>     schemaRegistryUrl: ${schema_registry_url}
+</span><span id="__span-1-7"><a id="__codelineno-1-7" name="__codelineno-1-7"></a><span class="w"> </span>   nameOverride: override-with-this-name
+</span><span id="__span-1-8"><a id="__codelineno-1-8" name="__codelineno-1-8"></a><span class="w"> </span>   imageTag: "1.0.0"
+</span><span id="__span-1-9"><a id="__codelineno-1-9" name="__codelineno-1-9"></a><span class="w">...</span>
+</span></code></pre></div></td></tr></table></div>
+<h2 id="move-github-action-to-repsitory-root"><a href="https://github.com/bakdata/kpops/pull/356">Move GitHub action to repsitory root</a><a class="headerlink" href="#move-github-action-to-repsitory-root" title="Permanent link">¶</a></h2>
+<p>The location of the GitHub action has changed, and it's now available directly as <code>bakdata/kpops</code>.</p>
+<p>You'll need to change it in your GitHub CI workflows.</p>
+<div class="language-diff highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-2-1">1</a></span>
+<span class="normal"><a href="#__codelineno-2-2">2</a></span>
+<span class="normal"><a href="#__codelineno-2-3">3</a></span>
+<span class="normal"><a href="#__codelineno-2-4">4</a></span>
+<span class="normal"><a href="#__codelineno-2-5">5</a></span>
+<span class="normal"><a href="#__codelineno-2-6">6</a></span>
+<span class="normal"><a href="#__codelineno-2-7">7</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-2-1"><a id="__codelineno-2-1" name="__codelineno-2-1"></a><span class="w">steps:</span>
+</span><span id="__span-2-2"><a id="__codelineno-2-2" name="__codelineno-2-2"></a><span class="w"> </span> - name: kpops deploy
+</span><span id="__span-2-3"><a id="__codelineno-2-3" name="__codelineno-2-3"></a><span class="gd">-   uses: bakdata/kpops/actions/kpops-runner@main</span>
+</span><span id="__span-2-4"><a id="__codelineno-2-4" name="__codelineno-2-4"></a><span class="gi">+   uses: bakdata/kpops@main</span>
+</span><span id="__span-2-5"><a id="__codelineno-2-5" name="__codelineno-2-5"></a><span class="w"> </span>   with:
+</span><span id="__span-2-6"><a id="__codelineno-2-6" name="__codelineno-2-6"></a><span class="w"> </span>     command: deploy --execute
+</span><span id="__span-2-7"><a id="__codelineno-2-7" name="__codelineno-2-7"></a><span class="w"> </span>     # ...
+</span></code></pre></div></td></tr></table></div>
+</article>
+</div>
+</div>
+<a class="md-top md-icon" data-md-component="top" hidden="" href="#">
+<svg viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8v12Z"></path></svg>
+            Back to top
+          </a>
+</main>
+<footer class="md-footer">
+<div class="md-footer-meta md-typeset">
+<div class="md-footer-meta__inner md-grid">
+<div class="md-copyright">
+<div class="md-copyright__highlight">
+      Copyright © 2023 bakdata
+    </div>
+  
+  
+    Made with
+    <a href="https://squidfunk.github.io/mkdocs-material/" rel="noopener" target="_blank">
+      Material for MkDocs
+    </a>
+</div>
+</div>
+</div>
+</footer>
+</div>
+<div class="md-dialog" data-md-component="dialog">
+<div class="md-dialog__inner md-typeset"></div>
+</div>
+<script id="__config" type="application/json">{"base": "../../..", "features": ["navigation.sections", "navigation.tabs", "navigation.indexes", "navigation.top", "navigation.tracking", "toc.follow", "search.highlight", "search.share", "search.suggest", "content.tooltips", "content.code.copy", "content.code.select", "content.code.annotate"], "search": "../../../assets/javascripts/workers/search.208ed371.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": {"provider": "mike"}}</script>
+<script src="../../../assets/javascripts/bundle.b78d2936.min.js"></script>
+<script>document$.subscribe(() => {const lightbox = GLightbox({"touchNavigation": true, "loop": false, "zoomable": true, "draggable": true, "openEffect": "zoom", "closeEffect": "zoom", "slideEffect": "slide"});})</script></body>
+</html>
\ No newline at end of file
diff --git a/dev/user/references/ci-integration/github-actions/index.html b/dev/user/references/ci-integration/github-actions/index.html
index 170c23764..78e063239 100644
--- a/dev/user/references/ci-integration/github-actions/index.html
+++ b/dev/user/references/ci-integration/github-actions/index.html
@@ -373,6 +373,11 @@
         Migrate from v1 to v2
       </a>
 </li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../../migration-guide/v2-v3/">
+        Migrate from v2 to v3
+      </a>
+</li>
 </ul>
 </nav>
 </li>
@@ -492,9 +497,7 @@
 <div class="md-content" data-md-component="content">
 <article class="md-content__inner md-typeset">
 <h1 id="github-actions-integration">GitHub Actions integration<a class="headerlink" href="#github-actions-integration" title="Permanent link">¶</a></h1>
-<p>We provided a GitHub composite action called
-<a href="https://github.com/bakdata/kpops/tree/main/actions/kpops-runner/action.yaml">kpops-runner</a>
-that installs all the necessary dependencies and runs KPOps commands with the given parameters.</p>
+<p>We provided a GitHub composite action <code>bakdata/kpops</code> that installs and executes KPOps commands with the given parameters.</p>
 <h2 id="input-parameters">Input Parameters<a class="headerlink" href="#input-parameters" title="Permanent link">¶</a></h2>
 <table>
 <thead>
@@ -616,7 +619,7 @@ <h2 id="usage">Usage<a class="headerlink" href="#usage" title="Permanent link">
 </span><span id="__span-0-2"><a id="__codelineno-0-2" name="__codelineno-0-2"></a><span class="w">  </span><span class="c1"># ...</span>
 </span><span id="__span-0-3"><a id="__codelineno-0-3" name="__codelineno-0-3"></a><span class="w">  </span><span class="c1"># This step is useful for debugging reasons</span>
 </span><span id="__span-0-4"><a id="__codelineno-0-4" name="__codelineno-0-4"></a><span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Generate Kafka pipeline</span>
-</span><span id="__span-0-5"><a id="__codelineno-0-5" name="__codelineno-0-5"></a><span class="w">    </span><span class="nt">uses</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">bakdata/kpops/actions/kpops-runner@main</span>
+</span><span id="__span-0-5"><a id="__codelineno-0-5" name="__codelineno-0-5"></a><span class="w">    </span><span class="nt">uses</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">bakdata/kpops@main</span>
 </span><span id="__span-0-6"><a id="__codelineno-0-6" name="__codelineno-0-6"></a><span class="w">    </span><span class="nt">with</span><span class="p">:</span>
 </span><span id="__span-0-7"><a id="__codelineno-0-7" name="__codelineno-0-7"></a><span class="w">      </span><span class="nt">command</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">generate</span>
 </span><span id="__span-0-8"><a id="__codelineno-0-8" name="__codelineno-0-8"></a><span class="w">      </span><span class="nt">working-directory</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">home/my-kpops-root-dir</span>
@@ -624,7 +627,7 @@ <h2 id="usage">Usage<a class="headerlink" href="#usage" title="Permanent link">
 </span><span id="__span-0-10"><a id="__codelineno-0-10" name="__codelineno-0-10"></a><span class="w">      </span><span class="nt">kpops-version</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">1.2.3</span>
 </span><span id="__span-0-11"><a id="__codelineno-0-11" name="__codelineno-0-11"></a>
 </span><span id="__span-0-12"><a id="__codelineno-0-12" name="__codelineno-0-12"></a><span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Deploy Kafka pipeline</span>
-</span><span id="__span-0-13"><a id="__codelineno-0-13" name="__codelineno-0-13"></a><span class="w">    </span><span class="nt">uses</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">bakdata/kpops/actions/kpops-runner@main</span>
+</span><span id="__span-0-13"><a id="__codelineno-0-13" name="__codelineno-0-13"></a><span class="w">    </span><span class="nt">uses</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">bakdata/kpops@main</span>
 </span><span id="__span-0-14"><a id="__codelineno-0-14" name="__codelineno-0-14"></a><span class="w">    </span><span class="nt">with</span><span class="p">:</span>
 </span><span id="__span-0-15"><a id="__codelineno-0-15" name="__codelineno-0-15"></a><span class="w">      </span><span class="nt">command</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">deploy --execute</span>
 </span><span id="__span-0-16"><a id="__codelineno-0-16" name="__codelineno-0-16"></a><span class="w">      </span><span class="nt">working-directory</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">home/my-kpops-root-dir</span>
@@ -632,8 +635,7 @@ <h2 id="usage">Usage<a class="headerlink" href="#usage" title="Permanent link">
 </span><span id="__span-0-18"><a id="__codelineno-0-18" name="__codelineno-0-18"></a><span class="w">      </span><span class="nt">kpops-version</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">1.2.3</span>
 </span><span id="__span-0-19"><a id="__codelineno-0-19" name="__codelineno-0-19"></a><span class="w">  </span><span class="c1"># ...</span>
 </span></code></pre></div></td></tr></table></div>
-<p>It is possible to execute the KPOps runner on
-a dev version from the <a href="https://test.pypi.org/project/kpops/#history">test.pypi</a>.</p>
+<p>It is possible to use a pre-release KPOps version from <a href="https://test.pypi.org/project/kpops/#history">TestPyPI</a>.</p>
 <div class="language-yaml highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-1-1">1</a></span>
 <span class="normal"><a href="#__codelineno-1-2">2</a></span>
 <span class="normal"><a href="#__codelineno-1-3">3</a></span>
@@ -643,7 +645,7 @@ <h2 id="usage">Usage<a class="headerlink" href="#usage" title="Permanent link">
 <span class="normal"><a href="#__codelineno-1-7">7</a></span>
 <span class="normal"><a href="#__codelineno-1-8">8</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-1-1"><a id="__codelineno-1-1" name="__codelineno-1-1"></a><span class="nt">steps</span><span class="p">:</span>
 </span><span id="__span-1-2"><a id="__codelineno-1-2" name="__codelineno-1-2"></a><span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Deploy Kafka pipeline</span>
-</span><span id="__span-1-3"><a id="__codelineno-1-3" name="__codelineno-1-3"></a><span class="w">    </span><span class="nt">uses</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">bakdata/kpops/actions/kpops-runner@main</span>
+</span><span id="__span-1-3"><a id="__codelineno-1-3" name="__codelineno-1-3"></a><span class="w">    </span><span class="nt">uses</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">bakdata/kpops@main</span>
 </span><span id="__span-1-4"><a id="__codelineno-1-4" name="__codelineno-1-4"></a><span class="w">    </span><span class="nt">with</span><span class="p">:</span>
 </span><span id="__span-1-5"><a id="__codelineno-1-5" name="__codelineno-1-5"></a><span class="w">      </span><span class="nt">command</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">deploy --execute</span>
 </span><span id="__span-1-6"><a id="__codelineno-1-6" name="__codelineno-1-6"></a><span class="w">      </span><span class="nt">working-directory</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">home/my-kpops-root-dir</span>
diff --git a/dev/user/references/cli-commands/index.html b/dev/user/references/cli-commands/index.html
index 6d07e70f7..78c5fd024 100644
--- a/dev/user/references/cli-commands/index.html
+++ b/dev/user/references/cli-commands/index.html
@@ -6,7 +6,7 @@
 <meta charset="utf-8"/>
 <meta content="width=device-width,initial-scale=1" name="viewport"/>
 <link href="https://bakdata.github.io/kpops/dev/user/references/cli-commands/" rel="canonical"/>
-<link href="../../migration-guide/v1-v2/" rel="prev"/>
+<link href="../../migration-guide/v2-v3/" rel="prev"/>
 <link href="../editor-integration/" rel="next"/>
 <link href="../../../assets/images/favicon.png" rel="icon"/>
 <meta content="mkdocs-1.4.2, mkdocs-material-9.1.1" name="generator"/>
@@ -373,6 +373,11 @@
         Migrate from v1 to v2
       </a>
 </li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../migration-guide/v2-v3/">
+        Migrate from v2 to v3
+      </a>
+</li>
 </ul>
 </nav>
 </li>
diff --git a/dev/user/references/editor-integration/index.html b/dev/user/references/editor-integration/index.html
index 4af9b8fd0..fc2f224f3 100644
--- a/dev/user/references/editor-integration/index.html
+++ b/dev/user/references/editor-integration/index.html
@@ -373,6 +373,11 @@
         Migrate from v1 to v2
       </a>
 </li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../../migration-guide/v2-v3/">
+        Migrate from v2 to v3
+      </a>
+</li>
 </ul>
 </nav>
 </li>
diff --git a/dev/user/what-is-kpops/index.html b/dev/user/what-is-kpops/index.html
index ca1780268..d291cbdc2 100644
--- a/dev/user/what-is-kpops/index.html
+++ b/dev/user/what-is-kpops/index.html
@@ -396,6 +396,11 @@
         Migrate from v1 to v2
       </a>
 </li>
+<li class="md-nav__item">
+<a class="md-nav__link" href="../migration-guide/v2-v3/">
+        Migrate from v2 to v3
+      </a>
+</li>
 </ul>
 </nav>
 </li>