Merge remote-tracking branch 'origin/v3' into fix/enrich-basemodel

docs/docs/resources/pipeline-components/dependencies/defaults_pipeline_component_dependencies.yaml

@@ -10,10 +10,7 @@ kafka-connector.yaml:
 - prefix.yaml
 - from_.yaml
 - to.yaml
-- namespace.yaml
 - app-kafka-connector.yaml
-- repo_config-kafka-connector.yaml
-- version-kafka-connector.yaml
 - resetter_values.yaml
 kafka-sink-connector.yaml: []
 kafka-source-connector.yaml:

docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml

@@ -19,30 +19,24 @@ kpops_components_fields:
   - prefix
   - from_
   - to
-  - namespace
   - app
-  - repo_config
-  - version
+  - resetter_namespace
   - resetter_values
   kafka-sink-connector:
   - name
   - prefix
   - from_
   - to
-  - namespace
   - app
-  - repo_config
-  - version
+  - resetter_namespace
   - resetter_values
   kafka-source-connector:
   - name
   - prefix
   - from_
   - to
-  - namespace
   - app
-  - repo_config
-  - version
+  - resetter_namespace
   - resetter_values
   - offset_topic
   kubernetes-app:

docs/docs/resources/pipeline-components/dependencies/pipeline_component_dependencies.yaml

@@ -15,28 +15,19 @@ kafka-connector.yaml:
 - prefix.yaml
 - from_.yaml
 - to.yaml
-- namespace.yaml
 - app-kafka-connector.yaml
-- repo_config-kafka-connector.yaml
-- version-kafka-connector.yaml
 - resetter_values.yaml
 kafka-sink-connector.yaml:
 - prefix.yaml
 - from_.yaml
 - to.yaml
-- namespace.yaml
 - app-kafka-connector.yaml
-- repo_config-kafka-connector.yaml
-- version-kafka-connector.yaml
 - resetter_values.yaml
 kafka-source-connector.yaml:
 - prefix.yaml
 - from_-kafka-source-connector.yaml
 - to.yaml
-- namespace.yaml
 - app-kafka-connector.yaml
-- repo_config-kafka-connector.yaml
-- version-kafka-connector.yaml
 - resetter_values.yaml
 - offset_topic-kafka-source-connector.yaml
 kubernetes-app.yaml:

docs/docs/resources/pipeline-components/kafka-connector.yaml

@@ -42,22 +42,11 @@
           cleanup.policy: compact
     models: # SchemaProvider is initiated with the values given here
       model: model
-  namespace: namespace # required
   # `app` contains application-specific settings, hence it does not have a rigid
   # structure. The fields below are just an example. Extensive documentation on
   # connectors: https://kafka.apache.org/documentation/#connectconfigs
   app: # required
     tasks.max: 1
-  # Helm repository configuration for resetter
-  repo_config:
-    repository_name: my-repo # required
-    url: https://bakdata.github.io/kafka-connect-resetter/ # required
-    repo_auth_flags:
-      username: user
-      password: pass
-      ca_file: /home/user/path/to/ca-file
-      insecure_skip_tls_verify: false
-  version: "1.0.6" # Helm chart version
   # Overriding Kafka Connect Resetter Helm values. E.g. to override the
   # Image Tag etc.
   resetter_values:

docs/docs/resources/pipeline-components/kafka-sink-connector.yaml

@@ -43,22 +43,11 @@
           cleanup.policy: compact
     models: # SchemaProvider is initiated with the values given here
       model: model
-  namespace: namespace # required
   # `app` contains application-specific settings, hence it does not have a rigid
   # structure. The fields below are just an example. Extensive documentation on
   # connectors: https://kafka.apache.org/documentation/#connectconfigs
   app: # required
     tasks.max: 1
-  # Helm repository configuration for resetter
-  repo_config:
-    repository_name: my-repo # required
-    url: https://bakdata.github.io/kafka-connect-resetter/ # required
-    repo_auth_flags:
-      username: user
-      password: pass
-      ca_file: /home/user/path/to/ca-file
-      insecure_skip_tls_verify: false
-  version: "1.0.6" # Helm chart version
   # Overriding Kafka Connect Resetter Helm values. E.g. to override the
   # Image Tag etc.
   resetter_values:

docs/docs/resources/pipeline-components/kafka-source-connector.yaml

@@ -24,22 +24,11 @@
           cleanup.policy: compact
     models: # SchemaProvider is initiated with the values given here
       model: model
-  namespace: namespace # required
   # `app` contains application-specific settings, hence it does not have a rigid
   # structure. The fields below are just an example. Extensive documentation on
   # connectors: https://kafka.apache.org/documentation/#connectconfigs
   app: # required
     tasks.max: 1
-  # Helm repository configuration for resetter
-  repo_config:
-    repository_name: my-repo # required
-    url: https://bakdata.github.io/kafka-connect-resetter/ # required
-    repo_auth_flags:
-      username: user
-      password: pass
-      ca_file: /home/user/path/to/ca-file
-      insecure_skip_tls_verify: false
-  version: "1.0.6" # Helm chart version
   # Overriding Kafka Connect Resetter Helm values. E.g. to override the
   # Image Tag etc.
   resetter_values:

docs/docs/resources/pipeline-components/pipeline.yaml

@@ -160,22 +160,11 @@
           cleanup.policy: compact
     models: # SchemaProvider is initiated with the values given here
       model: model
-  namespace: namespace # required
   # `app` contains application-specific settings, hence it does not have a rigid
   # structure. The fields below are just an example. Extensive documentation on
   # connectors: https://kafka.apache.org/documentation/#connectconfigs
   app: # required
     tasks.max: 1
-  # Helm repository configuration for resetter
-  repo_config:
-    repository_name: my-repo # required
-    url: https://bakdata.github.io/kafka-connect-resetter/ # required
-    repo_auth_flags:
-      username: user
-      password: pass
-      ca_file: /home/user/path/to/ca-file
-      insecure_skip_tls_verify: false
-  version: "1.0.6" # Helm chart version
   # Overriding Kafka Connect Resetter Helm values. E.g. to override the
   # Image Tag etc.
   resetter_values:
@@ -206,22 +195,11 @@
           cleanup.policy: compact
     models: # SchemaProvider is initiated with the values given here
       model: model
-  namespace: namespace # required
   # `app` contains application-specific settings, hence it does not have a rigid
   # structure. The fields below are just an example. Extensive documentation on
   # connectors: https://kafka.apache.org/documentation/#connectconfigs
   app: # required
     tasks.max: 1
-  # Helm repository configuration for resetter
-  repo_config:
-    repository_name: my-repo # required
-    url: https://bakdata.github.io/kafka-connect-resetter/ # required
-    repo_auth_flags:
-      username: user
-      password: pass
-      ca_file: /home/user/path/to/ca-file
-      insecure_skip_tls_verify: false
-  version: "1.0.6" # Helm chart version
   # Overriding Kafka Connect Resetter Helm values. E.g. to override the
   # Image Tag etc.
   resetter_values:

docs/docs/resources/pipeline-config/config.yaml

@@ -3,28 +3,35 @@
 # The path to the folder containing the defaults.yaml file and the environment
 # defaults files.
 defaults_path: .
-# The environment you want to generate and deploy the pipeline to. Suffix your
-# environment files with this value (e.g. defaults_development.yaml and
-# pipeline_development.yaml for environment=development).
-# REQUIRED
-environment: development
+# Custom Python module defining project-specific KPOps components
+components_module: null
+# Base directory to the pipelines (default is current working directory)
+pipeline_base_dir: .
 # The Kafka brokers address.
 # REQUIRED
-brokers: "http://broker1:9092,http://broker2:9092"
+kafka_brokers: "http://broker1:9092,http://broker2:9092"
 # The name of the defaults file and the prefix of the defaults environment file.
 defaults_filename_prefix: defaults
-# Configures topic names.
+# Configure the topic name variables you can use in the pipeline definition.
 topic_name_config: 
   # Configures the value for the variable ${output_topic_name}
-  default_output_topic_name: ${pipeline.name}-${component_name}
+  default_output_topic_name: ${pipeline.name}-${component.name}
   # Configures the value for the variable ${error_topic_name}
-  default_error_topic_name: ${pipeline.name}-${component_name}-error
-# Address of the Schema Registry
-schema_registry_url: "http://localhost:8081"
-# Address of the Kafka REST Proxy.
-kafka_rest_host: "http://localhost:8082"
-# Address of Kafka Connect.
-kafka_connect_host: "http://localhost:8083"
+  default_error_topic_name: ${pipeline.name}-${component.name}-error
+# Configuration for Schema Registry.
+schema_registry:
+  # Whether the Schema Registry handler should be initialized.
+  enabled: false
+  # Address of the Schema Registry.
+  url: "http://localhost:8081"
+# Configuration for the Kafka REST Proxy.
+kafka_rest:
+  # Address of the Kafka REST Proxy.
+  url: "http://localhost:8082"
+# Configuration for Kafka Connect.
+kafka_connect:
+  # Address of Kafka Connect.
+  url: "http://localhost:8083"
 # The timeout in seconds that specifies when actions like deletion or deploy
 # timeout.
 timeout: 300
@@ -33,14 +40,16 @@ timeout: 300
 create_namespace: false
 # Global flags for Helm.
 helm_config:
-  # Set the name of the kubeconfig context. (--kube-context)
+  # Name of kubeconfig context (`--kube-context`)
   context: name
   # Run Helm in Debug mode.
   debug: false
+  # Kubernetes API version used for Capabilities.APIVersions
+  api_version: null
 # Configure Helm Diff.
 helm_diff_config: 
   # Set of keys that should not be checked.
-  ignore: 
+  ignore:
     - name
     - imageTag
 # Whether to retain clean up jobs in the cluster or uninstall the, after

docs/docs/resources/pipeline-defaults/defaults-kafka-connector.yaml

@@ -45,22 +45,11 @@ kafka-connector:
           cleanup.policy: compact
     models: # SchemaProvider is initiated with the values given here
       model: model
-  namespace: namespace # required
   # `app` contains application-specific settings, hence it does not have a rigid
   # structure. The fields below are just an example. Extensive documentation on
   # connectors: https://kafka.apache.org/documentation/#connectconfigs
   app: # required
     tasks.max: 1
-  # Helm repository configuration for resetter
-  repo_config:
-    repository_name: my-repo # required
-    url: https://bakdata.github.io/kafka-connect-resetter/ # required
-    repo_auth_flags:
-      username: user
-      password: pass
-      ca_file: /home/user/path/to/ca-file
-      insecure_skip_tls_verify: false
-  version: "1.0.6" # Helm chart version
   # Overriding Kafka Connect Resetter Helm values. E.g. to override the
   # Image Tag etc.
   resetter_values:

docs/docs/resources/pipeline-defaults/defaults.yaml

@@ -121,22 +121,11 @@ kafka-connector:
           cleanup.policy: compact
     models: # SchemaProvider is initiated with the values given here
       model: model
-  namespace: namespace # required
   # `app` contains application-specific settings, hence it does not have a rigid
   # structure. The fields below are just an example. Extensive documentation on
   # connectors: https://kafka.apache.org/documentation/#connectconfigs
   app: # required
     tasks.max: 1
-  # Helm repository configuration for resetter
-  repo_config:
-    repository_name: my-repo # required
-    url: https://bakdata.github.io/kafka-connect-resetter/ # required
-    repo_auth_flags:
-      username: user
-      password: pass
-      ca_file: /home/user/path/to/ca-file
-      insecure_skip_tls_verify: false
-  version: "1.0.6" # Helm chart version
   # Overriding Kafka Connect Resetter Helm values. E.g. to override the
   # Image Tag etc.
   resetter_values: