KPOps 3.0

Co-authored-by: Ramin Gharib <[email protected]> Co-authored-by: Ivan Yordanov <[email protected]>
bakdata · Jan 17, 2024 · dbf82c4 · dbf82c4
2 parents 4620131 + a90984b
commit dbf82c4
Show file tree

Hide file tree

Showing 178 changed files with 12,094 additions and 6,215 deletions.
diff --git a/.github/actions/update-docs/action.yml b/.github/actions/update-docs/action.yml
@@ -15,10 +15,10 @@ inputs:
     required: true
   version:
     description: "Version name to be deployed by mike"
-    required: true 
+    required: true
   release:
     description: "Determines if the set version is a stable and latest version, otherwise it is a dev version. (Default false)"
-    default: 'false'
+    default: "false"
     required: false
 
 runs:
@@ -35,14 +35,13 @@ runs:
       run: |
         poetry install --with docs
 
-    - name: Update ${{ github.head_ref }} branch
+    - name: Update gh-pages branch
       shell: bash
       run: |
         git config --local user.name  ${{ inputs.username }}
         git config --local user.email ${{ inputs.email }}
         git config --local user.password ${{ inputs.token }}
-
-        git pull 
+        git fetch origin gh-pages
 
     - name: Deploy ${{ inputs.version }} version of the documentation with mike
       shell: bash

diff --git a/README.md b/README.md
@@ -29,6 +29,10 @@ You can install it with [pip](https://github.com/pypa/pip):
 pip install kpops
 ```
 
+# GitHub action
+
+Please refer to the [GitHub Actions section](https://bakdata.github.io/kpops/latest/user/references/ci-integration/github-actions) for the documentation.
+
 ## Contributing
 
 We are happy if you want to contribute to this project.

diff --git a/actions/kpops-runner/action.yaml → action.yaml b/actions/kpops-runner/action.yaml → action.yaml
@@ -11,17 +11,14 @@ inputs:
   working-directory:
     description: "The root directory containing the config.yaml, pipelines folder and defaults"
     default: "."
-  pipeline-base-dir:
-    description: "Directory where relative pipeline variables are initialized from"
-    required: false
   defaults:
     description: "Defaults folder path"
     required: false
   config:
-    description: "config.yaml file path"
+    description: "Directory containing the config*.yaml file(s)"
     required: false
-  components:
-    description: "Components package path"
+  environment:
+    description: "Environment to run KPOps in"
     required: false
   filter-type:
     description: "Whether to include/exclude the steps defined in KPOPS_PIPELINE_STEPS (default is include)"
@@ -73,10 +70,15 @@ runs:
       shell: bash
       run: |
         echo "::group::install kpops package"
-        pip install -r "${{ steps.requirements.outputs.path }}"
+        # Check if kpops-version contains ".dev"
+        if [[ "${{ inputs.kpops-version }}" == *".dev"* ]]; then
+          pip install -r "${{ steps.requirements.outputs.path }}" -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/
+        else
+          pip install -r "${{ steps.requirements.outputs.path }}"
+        fi
         echo "::endgroup::"
 
     - name: ${{ inputs.command }} ${{ inputs.pipeline }} pipeline
       shell: bash
       working-directory: ${{inputs.working-directory}}
-      run: kpops ${{ inputs.command }} ${{ inputs.pipeline }} ${{ inputs.components }} ${{ (inputs.defaults != '' && format('--defaults {0}', inputs.defaults)) || '' }} ${{ (inputs.config != '' && format('--config {0}', inputs.config)) || '' }} ${{ (inputs.pipeline-base-dir != '' && format('--pipeline-base-dir {0}', inputs.pipeline-base-dir)) || '' }} ${{ (inputs.filter-type != '' && format('--filter-type {0}', inputs.filter-type)) || '' }}
+      run: kpops ${{ inputs.command }} ${{ inputs.pipeline }} ${{ (inputs.defaults != '' && format('--defaults {0}', inputs.defaults)) || '' }} ${{ (inputs.config != '' && format('--config {0}', inputs.config)) || '' }} ${{ (inputs.environment != '' && format('--environment {0}', inputs.environment)) || '' }} ${{ (inputs.filter-type != '' && format('--filter-type {0}', inputs.filter-type)) || '' }}
diff --git a/actions/kpops-runner/README.md b/actions/kpops-runner/README.md
diff --git a/config.yaml b/config.yaml
@@ -1,5 +1,3 @@
-environment: development
-brokers: "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092"
-kafka_connect_host: "http://localhost:8083"
-kafka_rest_host: "http://localhost:8082"
-schema_registry_url: "http://localhost:8081"
+kafka_brokers: "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092"
+components_module: tests.pipeline.test_components
+pipeline_base_dir: tests/pipeline
diff --git a/docs/docs/developer/auto-generation.md b/docs/docs/developer/auto-generation.md
@@ -10,7 +10,7 @@ Auto generation happens mostly with [`pre-commit`](https://pre-commit.com/) hook
 
 - `cli_env_vars.env` -- All CLI environment variables in a `dotenv` file.
 - `cli_env_vars.md` -- All CLI environment variables in a table.
-- `config_env_vars.env` -- Almost all pipeline config environment variables in a `dotenv` file. The script checks for each field in [`PipelineConfig`](https://github.com/bakdata/kpops/blob/main/kpops/cli/pipeline_config.py) whether it has an `env` attribute defined. The script is currently unable to visit the classes of fields like `topic_name_config`, hence any environment variables defined there would remain unknown to it.
+- `config_env_vars.env` -- Almost all pipeline config environment variables in a `dotenv` file. The script checks for each field in [`KpopsConfig`](https://github.com/bakdata/kpops/blob/main/kpops/cli/kpops_config.py) whether it has an `env` attribute defined. The script is currently unable to visit the classes of fields like `topic_name_config`, hence any environment variables defined there would remain unknown to it.
 - `config_env_vars.env` -- Almost all pipeline config environment variables in a table.
 - `variable_substitution.yaml` -- A copy of `./tests/pipeline/resources/component-type-substitution/pipeline.yaml` used as an example of substitution.
 

diff --git a/docs/docs/resources/architecture/components-hierarchy.md b/docs/docs/resources/architecture/components-hierarchy.md
@@ -1,20 +1,26 @@
 ```mermaid
 flowchart BT
     KubernetesApp --> PipelineComponent
-    KafkaConnector --> PipelineComponent
-    KafkaApp --> KubernetesApp
+    KafkaApp --> PipelineComponent
+    HelmApp --> KubernetesApp
+    StreamsBootstrap --> HelmApp
     StreamsApp --> KafkaApp
+    StreamsApp --> StreamsBootstrap
     ProducerApp --> KafkaApp
+    ProducerApp --> StreamsBootstrap
+    KafkaConnector --> PipelineComponent
     KafkaSourceConnector --> KafkaConnector
     KafkaSinkConnector --> KafkaConnector
-    
-    click KubernetesApp "../kubernetes-app"
-    click KafkaApp "../kafka-app"
-    click StreamsApp "../streams-app"
-    click ProducerApp "../producer-app"
-    click KafkaConnector "../kafka-connector"
-    click KafkaSourceConnector "../kafka-source-connector"
-    click KafkaSinkConnector "../kafka-sink-connector"
+
+    click KubernetesApp "/kpops/user/core-concepts/components/kubernetes-app"
+    click HelmApp "/kpops/user/core-concepts/components/helm-app"
+    click KafkaApp "/kpops/user/core-concepts/components/kafka-app"
+    click StreamsBootstrap "/kpops/user/core-concepts/components/streams-bootstrap"
+    click StreamsApp "/kpops/user/core-concepts/components/streams-app"
+    click ProducerApp "/kpops/user/core-concepts/components/producer-app"
+    click KafkaConnector "/kpops/user/core-concepts/components/kafka-connector"
+    click KafkaSourceConnector "/kpops/user/core-concepts/components/kafka-source-connector"
+    click KafkaSinkConnector "/kpops/user/core-concepts/components/kafka-sink-connector"
 ```
 
 <p style="text-align: center;"><i>KPOps component hierarchy</i></p>
diff --git a/docs/docs/resources/editor_integration/settings.json b/docs/docs/resources/editor_integration/settings.json
@@ -1,6 +1,16 @@
 {
     "yaml.schemas": {
-        "https://bakdata.github.io/kpops/2.0/schema/config.json": "config.yaml",
-        "https://bakdata.github.io/kpops/2.0/schema/pipeline.json": "pipeline.yaml"
+        "https://bakdata.github.io/kpops/3.0/schema/pipeline.json": [
+            "pipeline.yaml",
+            "pipeline_*.yaml"
+        ],
+        "https://bakdata.github.io/kpops/3.0/schema/defaults.json": [
+            "defaults.yaml",
+            "defaults_*.yaml"
+        ],
+        "https://bakdata.github.io/kpops/3.0/schema/config.json": [
+            "config.yaml",
+            "config_*.yaml"
+        ]
     }
 }
diff --git a/.../resources/pipeline-components/dependencies/defaults_pipeline_component_dependencies.yaml b/.../resources/pipeline-components/dependencies/defaults_pipeline_component_dependencies.yaml
@@ -1,14 +1,16 @@
+helm-app.yaml:
+- app-helm-app.yaml
+- repo_config-helm-app.yaml
 kafka-app.yaml:
+- prefix.yaml
+- from_.yaml
+- to.yaml
 - app-kafka-app.yaml
-- version-kafka-app.yaml
 kafka-connector.yaml:
 - prefix.yaml
 - from_.yaml
 - to.yaml
-- namespace.yaml
 - app-kafka-connector.yaml
-- repo_config-kafka-connector.yaml
-- version-kafka-connector.yaml
 - resetter_values.yaml
 kafka-sink-connector.yaml: []
 kafka-source-connector.yaml:
@@ -20,8 +22,6 @@ kubernetes-app.yaml:
 - to.yaml
 - namespace.yaml
 - app-kubernetes-app.yaml
-- repo_config-kubernetes-app.yaml
-- version.yaml
 producer-app.yaml:
 - from_-producer-app.yaml
 - app-producer-app.yaml

diff --git a/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml b/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml
@@ -1,5 +1,5 @@
 kpops_components_fields:
-  kafka-app:
+  helm-app:
   - name
   - prefix
   - from_
@@ -8,35 +8,35 @@ kpops_components_fields:
   - app
   - repo_config
   - version
+  kafka-app:
+  - name
+  - prefix
+  - from_
+  - to
+  - app
   kafka-connector:
   - name
   - prefix
   - from_
   - to
-  - namespace
   - app
-  - repo_config
-  - version
+  - resetter_namespace
   - resetter_values
   kafka-sink-connector:
   - name
   - prefix
   - from_
   - to
-  - namespace
   - app
-  - repo_config
-  - version
+  - resetter_namespace
   - resetter_values
   kafka-source-connector:
   - name
   - prefix
   - from_
   - to
-  - namespace
   - app
-  - repo_config
-  - version
+  - resetter_namespace
   - resetter_values
   - offset_topic
   kubernetes-app:
@@ -46,8 +46,6 @@ kpops_components_fields:
   - to
   - namespace
   - app
-  - repo_config
-  - version
   pipeline-component:
   - name
   - prefix
@@ -71,12 +69,76 @@ kpops_components_fields:
   - app
   - repo_config
   - version
+  streams-bootstrap:
+  - name
+  - prefix
+  - from_
+  - to
+  - namespace
+  - app
+  - repo_config
+  - version
 kpops_components_inheritance_ref:
-  kafka-app: kubernetes-app
-  kafka-connector: pipeline-component
-  kafka-sink-connector: kafka-connector
-  kafka-source-connector: kafka-connector
-  kubernetes-app: pipeline-component
-  pipeline-component: base-defaults-component
-  producer-app: kafka-app
-  streams-app: kafka-app
+  helm-app:
+    bases:
+    - kubernetes-app
+    parents:
+    - kubernetes-app
+    - pipeline-component
+  kafka-app:
+    bases:
+    - pipeline-component
+    parents:
+    - pipeline-component
+  kafka-connector:
+    bases:
+    - pipeline-component
+    parents:
+    - pipeline-component
+  kafka-sink-connector:
+    bases:
+    - kafka-connector
+    parents:
+    - kafka-connector
+    - pipeline-component
+  kafka-source-connector:
+    bases:
+    - kafka-connector
+    parents:
+    - kafka-connector
+    - pipeline-component
+  kubernetes-app:
+    bases:
+    - pipeline-component
+    parents:
+    - pipeline-component
+  pipeline-component:
+    bases: []
+    parents: []
+  producer-app:
+    bases:
+    - kafka-app
+    - streams-bootstrap
+    parents:
+    - kafka-app
+    - streams-bootstrap
+    - helm-app
+    - kubernetes-app
+    - pipeline-component
+  streams-app:
+    bases:
+    - kafka-app
+    - streams-bootstrap
+    parents:
+    - kafka-app
+    - streams-bootstrap
+    - helm-app
+    - kubernetes-app
+    - pipeline-component
+  streams-bootstrap:
+    bases:
+    - helm-app
+    parents:
+    - helm-app
+    - kubernetes-app
+    - pipeline-component