KPOps 6.0.0

docs/docs/resources/pipeline-config/config.yaml

@@ -7,8 +7,6 @@ pipeline_base_dir: .
 # The Kafka brokers address.
 # REQUIRED
 kafka_brokers: "http://broker1:9092,http://broker2:9092"
-# The name of the defaults file and the prefix of the defaults environment file.
-defaults_filename_prefix: defaults
 # Configure the topic name variables you can use in the pipeline definition.
 topic_name_config: 
   # Configures the value for the variable ${output_topic_name}

docs/docs/resources/variables/cli_env_vars.env

@@ -14,7 +14,8 @@ KPOPS_DOTENV_PATH # No default value, not required
 # Suffix your environment files with this value (e.g.
 # defaults_development.yaml for environment=development).
 KPOPS_ENVIRONMENT # No default value, not required
-# Path to YAML with pipeline definition
-KPOPS_PIPELINE_PATH # No default value, required
+# Paths to dir containing 'pipeline.yaml' or files named
+# 'pipeline.yaml'.
+KPOPS_PIPELINE_PATHS # No default value, required
 # Comma separated list of steps to apply the command on
 KPOPS_PIPELINE_STEPS # No default value, not required

docs/docs/resources/variables/cli_env_vars.md

@@ -5,5 +5,5 @@ These variables take precedence over the commands' flags. If a variable is set,
 |KPOPS_CONFIG_PATH   |.            |False   |Path to the dir containing config.yaml files                                                                                                                                |
 |KPOPS_DOTENV_PATH   |             |False   |Path to dotenv file. Multiple files can be provided. The files will be loaded in order, with each file overriding the previous one.                                         |
 |KPOPS_ENVIRONMENT   |             |False   |The environment you want to generate and deploy the pipeline to. Suffix your environment files with this value (e.g. defaults_development.yaml for environment=development).|
-|KPOPS_PIPELINE_PATH |             |True    |Path to YAML with pipeline definition                                                                                                                                       |
+|KPOPS_PIPELINE_PATHS|             |True    |Paths to dir containing 'pipeline.yaml' or files named 'pipeline.yaml'.                                                                                                     |
 |KPOPS_PIPELINE_STEPS|             |False   |Comma separated list of steps to apply the command on                                                                                                                       |

docs/docs/resources/variables/config_env_vars.env

@@ -14,10 +14,6 @@ KPOPS_PIPELINE_BASE_DIR=.
 # kafka_brokers
 # The comma separated Kafka brokers address.
 KPOPS_KAFKA_BROKERS # No default value, required
-# defaults_filename_prefix
-# The name of the defaults file and the prefix of the defaults
-# environment file.
-KPOPS_DEFAULTS_FILENAME_PREFIX=defaults
 # topic_name_config.default_output_topic_name
 # Configures the value for the variable ${output_topic_name}
 KPOPS_TOPIC_NAME_CONFIG__DEFAULT_OUTPUT_TOPIC_NAME=${pipeline.name}-${component.name}

docs/docs/resources/variables/config_env_vars.md

@@ -5,7 +5,6 @@ These variables take precedence over the settings in `config.yaml`. Variables ma
 |KPOPS_COMPONENTS_MODULE                           |                                        |False   |Custom Python module defining project-specific KPOps components                   |components_module                          |
 |KPOPS_PIPELINE_BASE_DIR                           |.                                       |False   |Base directory to the pipelines (default is current working directory)            |pipeline_base_dir                          |
 |KPOPS_KAFKA_BROKERS                               |                                        |True    |The comma separated Kafka brokers address.                                        |kafka_brokers                              |
-|KPOPS_DEFAULTS_FILENAME_PREFIX                    |defaults                                |False   |The name of the defaults file and the prefix of the defaults environment file.    |defaults_filename_prefix                   |
 |KPOPS_TOPIC_NAME_CONFIG__DEFAULT_OUTPUT_TOPIC_NAME|${pipeline.name}-${component.name}      |False   |Configures the value for the variable ${output_topic_name}                        |topic_name_config.default_output_topic_name|
 |KPOPS_TOPIC_NAME_CONFIG__DEFAULT_ERROR_TOPIC_NAME |${pipeline.name}-${component.name}-error|False   |Configures the value for the variable ${error_topic_name}                         |topic_name_config.default_error_topic_name |
 |KPOPS_SCHEMA_REGISTRY__ENABLED                    |False                                   |False   |Whether the Schema Registry handler should be initialized.                        |schema_registry.enabled                    |

docs/docs/resources/variables/variable_substitution.yaml

@@ -4,8 +4,6 @@
       app_type: "${component.type}"
       app_name: "${component.name}"
       app_schedule: "${component.app.schedule}"
-      helm_release_name: ${component.helm_release_name}
-      helm_name_override: ${component.helm_name_override}
     commandLine:
       FAKE_ARG: "fake-arg-value"
     schedule: "30 3/8 * * *"

docs/docs/schema/config.json

@@ -196,12 +196,6 @@
             "title": "Create Namespace",
             "type": "boolean"
         },
-        "defaults_filename_prefix": {
-            "default": "defaults",
-            "description": "The name of the defaults file and the prefix of the defaults environment file.",
-            "title": "Defaults Filename Prefix",
-            "type": "string"
-        },
         "helm_config": {
             "allOf": [
                 {

docs/docs/user/core-concepts/defaults.md

@@ -44,13 +44,6 @@ It is important to note that `defaults_{environment}.yaml` overrides only the se
 
 <!-- dprint-ignore-end -->
 
-<!-- dprint-ignore-start -->
-
-!!! tip
-    `defaults` is the default value of `defaults_filename_prefix`.
-
-<!-- dprint-ignore-end -->
-
 ## Components
 
 <!-- When possible, automatically generate a list of all component-specific settings under each component. -->

docs/docs/user/references/cli-commands.md

@@ -31,12 +31,12 @@ Clean pipeline steps
 **Usage**:
 
 ```console
-$ kpops clean [OPTIONS] PIPELINE_PATH
+$ kpops clean [OPTIONS] PIPELINE_PATHS...
 ```
 
 **Arguments**:
 
-* `PIPELINE_PATH`: Path to YAML with pipeline definition  [env var: KPOPS_PIPELINE_PATH;required]
+* `PIPELINE_PATHS...`: Paths to dir containing 'pipeline.yaml' or files named 'pipeline.yaml'.  [env var: KPOPS_PIPELINE_PATHS;required]
 
 **Options**:
 
@@ -57,12 +57,12 @@ Deploy pipeline steps
 **Usage**:
 
 ```console
-$ kpops deploy [OPTIONS] PIPELINE_PATH
+$ kpops deploy [OPTIONS] PIPELINE_PATHS...
 ```
 
 **Arguments**:
 
-* `PIPELINE_PATH`: Path to YAML with pipeline definition  [env var: KPOPS_PIPELINE_PATH;required]
+* `PIPELINE_PATHS...`: Paths to dir containing 'pipeline.yaml' or files named 'pipeline.yaml'.  [env var: KPOPS_PIPELINE_PATHS;required]
 
 **Options**:
 
@@ -83,12 +83,12 @@ Destroy pipeline steps
 **Usage**:
 
 ```console
-$ kpops destroy [OPTIONS] PIPELINE_PATH
+$ kpops destroy [OPTIONS] PIPELINE_PATHS...
 ```
 
 **Arguments**:
 
-* `PIPELINE_PATH`: Path to YAML with pipeline definition  [env var: KPOPS_PIPELINE_PATH;required]
+* `PIPELINE_PATHS...`: Paths to dir containing 'pipeline.yaml' or files named 'pipeline.yaml'.  [env var: KPOPS_PIPELINE_PATHS;required]
 
 **Options**:
 
@@ -109,18 +109,17 @@ Enrich pipeline steps with defaults. The enriched pipeline is used for all KPOps
 **Usage**:
 
 ```console
-$ kpops generate [OPTIONS] PIPELINE_PATH
+$ kpops generate [OPTIONS] PIPELINE_PATHS...
 ```
 
 **Arguments**:
 
-* `PIPELINE_PATH`: Path to YAML with pipeline definition  [env var: KPOPS_PIPELINE_PATH;required]
+* `PIPELINE_PATHS...`: Paths to dir containing 'pipeline.yaml' or files named 'pipeline.yaml'.  [env var: KPOPS_PIPELINE_PATHS;required]
 
 **Options**:
 
 * `--dotenv FILE`: Path to dotenv file. Multiple files can be provided. The files will be loaded in order, with each file overriding the previous one.  [env var: KPOPS_DOTENV_PATH]
 * `--config DIRECTORY`: Path to the dir containing config.yaml files  [env var: KPOPS_CONFIG_PATH; default: .]
-* `--output / --no-output`: Enable output printing  [default: output]
 * `--steps TEXT`: Comma separated list of steps to apply the command on  [env var: KPOPS_PIPELINE_STEPS]
 * `--filter-type [include|exclude]`: Whether the --steps option should include/exclude the steps  [default: FilterType.INCLUDE]
 * `--environment TEXT`: The environment you want to generate and deploy the pipeline to. Suffix your environment files with this value (e.g. defaults_development.yaml for environment=development).   [env var: KPOPS_ENVIRONMENT]
@@ -153,18 +152,17 @@ In addition to generate, render final resource representation for each pipeline
 **Usage**:
 
 ```console
-$ kpops manifest [OPTIONS] PIPELINE_PATH
+$ kpops manifest [OPTIONS] PIPELINE_PATHS...
 ```
 
 **Arguments**:
 
-* `PIPELINE_PATH`: Path to YAML with pipeline definition  [env var: KPOPS_PIPELINE_PATH;required]
+* `PIPELINE_PATHS...`: Paths to dir containing 'pipeline.yaml' or files named 'pipeline.yaml'.  [env var: KPOPS_PIPELINE_PATHS;required]
 
 **Options**:
 
 * `--dotenv FILE`: Path to dotenv file. Multiple files can be provided. The files will be loaded in order, with each file overriding the previous one.  [env var: KPOPS_DOTENV_PATH]
 * `--config DIRECTORY`: Path to the dir containing config.yaml files  [env var: KPOPS_CONFIG_PATH; default: .]
-* `--output / --no-output`: Enable output printing  [default: output]
 * `--steps TEXT`: Comma separated list of steps to apply the command on  [env var: KPOPS_PIPELINE_STEPS]
 * `--filter-type [include|exclude]`: Whether the --steps option should include/exclude the steps  [default: FilterType.INCLUDE]
 * `--environment TEXT`: The environment you want to generate and deploy the pipeline to. Suffix your environment files with this value (e.g. defaults_development.yaml for environment=development).   [env var: KPOPS_ENVIRONMENT]
@@ -178,12 +176,12 @@ Reset pipeline steps
 **Usage**:
 
 ```console
-$ kpops reset [OPTIONS] PIPELINE_PATH
+$ kpops reset [OPTIONS] PIPELINE_PATHS...
 ```
 
 **Arguments**:
 
-* `PIPELINE_PATH`: Path to YAML with pipeline definition  [env var: KPOPS_PIPELINE_PATH;required]
+* `PIPELINE_PATHS...`: Paths to dir containing 'pipeline.yaml' or files named 'pipeline.yaml'.  [env var: KPOPS_PIPELINE_PATHS;required]
 
 **Options**:
 

hooks/gen_docs/gen_docs_components.py

@@ -8,7 +8,7 @@
 import yaml
 
 from hooks import ROOT
-from kpops.cli.registry import _find_classes
+from kpops.api.registry import _find_classes
 from kpops.components import KafkaConnector, PipelineComponent
 from kpops.utils.colorify import redify, yellowify
 from kpops.utils.pydantic import issubclass_patched

hooks/gen_schema.py

@@ -5,8 +5,8 @@
 from pathlib import Path
 
 from hooks import ROOT
+from kpops.api.file_type import KpopsFileType
 from kpops.utils.gen_schema import (
-    SchemaScope,
     gen_config_schema,
     gen_defaults_schema,
     gen_pipeline_schema,
@@ -15,7 +15,7 @@
 PATH_TO_SCHEMA = ROOT / "docs/docs/schema"
 
 
-def gen_schema(scope: SchemaScope):
+def gen_schema(scope: KpopsFileType):
     """Generate the specified schema and save it to a file.
 
     The file is located in docs/docs/schema and is named ``<scope.value>.json``
@@ -24,16 +24,16 @@ def gen_schema(scope: SchemaScope):
     """
     with redirect_stdout(StringIO()) as f:
         match scope:
-            case SchemaScope.PIPELINE:
+            case KpopsFileType.PIPELINE:
                 gen_pipeline_schema()
-            case SchemaScope.DEFAULTS:
+            case KpopsFileType.DEFAULTS:
                 gen_defaults_schema()
-            case SchemaScope.CONFIG:
+            case KpopsFileType.CONFIG:
                 gen_config_schema()
         Path(PATH_TO_SCHEMA / f"{scope.value}.json").write_text(f.getvalue())
 
 
 if __name__ == "__main__":
-    gen_schema(SchemaScope.PIPELINE)
-    gen_schema(SchemaScope.DEFAULTS)
-    gen_schema(SchemaScope.CONFIG)
+    gen_schema(KpopsFileType.PIPELINE)
+    gen_schema(KpopsFileType.DEFAULTS)
+    gen_schema(KpopsFileType.CONFIG)

kpops/__init__.py

@@ -1,7 +1,7 @@
 __version__ = "5.1.1"
 
 # export public API functions
-from kpops.cli.main import clean, deploy, destroy, generate, init, manifest, reset
+from kpops.api import clean, deploy, destroy, generate, init, manifest, reset
 
 __all__ = (
     "generate",