Migrate to Pydantic v2

docs/docs/resources/variables/cli_env_vars.env

@@ -12,6 +12,9 @@ KPOPS_PIPELINE_BASE_DIR=.
 KPOPS_CONFIG_PATH=config.yaml
 # Path to defaults folder
 KPOPS_DEFAULT_PATH # No default value, not required
+# Path to dotenvfile. Multiple files can be provided. The files will
+# be loaded in order, with each file overriding the previous one.
+KPOPS_DOTENV_PATH # No default value, not required
 # Path to YAML with pipeline definition
 KPOPS_PIPELINE_PATH # No default value, required
 # Comma separated list of steps to apply the command on

docs/docs/resources/variables/cli_env_vars.md

@@ -1,9 +1,10 @@
 These variables are a lower priority alternative to the commands' flags. If a variable is set, the corresponding flag does not have to be specified in commands. Variables marked as required can instead be set as flags.
 
-|         Name          |Default Value|Required|                             Description                              |
-|-----------------------|-------------|--------|----------------------------------------------------------------------|
-|KPOPS_PIPELINE_BASE_DIR|.            |False   |Base directory to the pipelines (default is current working directory)|
-|KPOPS_CONFIG_PATH      |config.yaml  |False   |Path to the config.yaml file                                          |
-|KPOPS_DEFAULT_PATH     |             |False   |Path to defaults folder                                               |
-|KPOPS_PIPELINE_PATH    |             |True    |Path to YAML with pipeline definition                                 |
-|KPOPS_PIPELINE_STEPS   |             |False   |Comma separated list of steps to apply the command on                 |
+|         Name          |Default Value|Required|                                                           Description                                                            |
+|-----------------------|-------------|--------|----------------------------------------------------------------------------------------------------------------------------------|
+|KPOPS_PIPELINE_BASE_DIR|.            |False   |Base directory to the pipelines (default is current working directory)                                                            |
+|KPOPS_CONFIG_PATH      |config.yaml  |False   |Path to the config.yaml file                                                                                                      |
+|KPOPS_DEFAULT_PATH     |             |False   |Path to defaults folder                                                                                                           |
+|KPOPS_DOTENV_PATH      |             |False   |Path to dotenvfile. Multiple files can be provided. The files will be loaded in order, with each file overriding the previous one.|
+|KPOPS_PIPELINE_PATH    |             |True    |Path to YAML with pipeline definition                                                                                             |
+|KPOPS_PIPELINE_STEPS   |             |False   |Comma separated list of steps to apply the command on                                                                             |

docs/docs/resources/variables/config_env_vars.env

@@ -4,28 +4,51 @@
 # alternative to the settings in `config.yaml`. Variables marked as
 # required can instead be set in the pipeline config.
 #
+# defaults_path
+# The path to the folder containing the defaults.yaml file and the
+# environment defaults files. Paths can either be absolute or relative
+# to `config.yaml`
+defaults_path=.
 # environment
 # The environment you want to generate and deploy the pipeline to.
 # Suffix your environment files with this value (e.g.
 # defaults_development.yaml for environment=development).
-KPOPS_ENVIRONMENT # No default value, required
+environment=PydanticUndefined
 # kafka_brokers
 # The comma separated Kafka brokers address.
-KPOPS_KAFKA_BROKERS # No default value, required
-# url
-# Address of the Schema Registry.
-KPOPS_SCHEMA_REGISTRY_URL=http://localhost:8081
-# url
-# Address of the Kafka REST Proxy.
-KPOPS_KAFKA_REST_URL=http://localhost:8082
-# url
-# Address of Kafka Connect.
-KPOPS_KAFKA_CONNECT_URL=http://localhost:8083
+kafka_brokers=PydanticUndefined
+# defaults_filename_prefix
+# The name of the defaults file and the prefix of the defaults
+# environment file.
+defaults_filename_prefix=defaults
+# topic_name_config
+# Configure the topic name variables you can use in the pipeline
+# definition.
+topic_name_config=default_output_topic_name='${pipeline_name}-${component_name}' default_error_topic_name='${pipeline_name}-${component_name}-error'
+# schema_registry
+# Configuration for Schema Registry.
+schema_registry=enabled=False url=Url('http://localhost:8081/')
+# kafka_rest
+# Configuration for Kafka REST Proxy.
+kafka_rest=url=Url('http://localhost:8082/')
+# kafka_connect
+# Configuration for Kafka Connect.
+kafka_connect=url=Url('http://localhost:8083/')
 # timeout
 # The timeout in seconds that specifies when actions like deletion or
 # deploy timeout.
-KPOPS_TIMEOUT=300
+timeout=300
+# create_namespace
+# Flag for `helm upgrade --install`. Create the release namespace if
+# not present.
+create_namespace=False
+# helm_config
+# Global flags for Helm.
+helm_config=context=None debug=False api_version=None
+# helm_diff_config
+# Configure Helm Diff.
+helm_diff_config=ignore=set()
 # retain_clean_jobs
 # Whether to retain clean up jobs in the cluster or uninstall the,
 # after completion.
-KPOPS_RETAIN_CLEAN_JOBS=False
+retain_clean_jobs=False

docs/docs/resources/variables/config_env_vars.md

@@ -1,11 +1,17 @@
 These variables are a lower priority alternative to the settings in `config.yaml`. Variables marked as required can instead be set in the pipeline config.
 
-|          Name           |    Default Value    |Required|                                                                                Description                                                                                 |  Setting name   |
-|-------------------------|---------------------|--------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------|
-|KPOPS_ENVIRONMENT        |                     |True    |The environment you want to generate and deploy the pipeline to. Suffix your environment files with this value (e.g. defaults_development.yaml for environment=development).|environment      |
-|KPOPS_KAFKA_BROKERS      |                     |True    |The comma separated Kafka brokers address.                                                                                                                                  |kafka_brokers    |
-|KPOPS_SCHEMA_REGISTRY_URL|http://localhost:8081|False   |Address of the Schema Registry.                                                                                                                                             |url              |
-|KPOPS_KAFKA_REST_URL     |http://localhost:8082|False   |Address of the Kafka REST Proxy.                                                                                                                                            |url              |
-|KPOPS_KAFKA_CONNECT_URL  |http://localhost:8083|False   |Address of Kafka Connect.                                                                                                                                                   |url              |
-|KPOPS_TIMEOUT            |300                  |False   |The timeout in seconds that specifies when actions like deletion or deploy timeout.                                                                                         |timeout          |
-|KPOPS_RETAIN_CLEAN_JOBS  |False                |False   |Whether to retain clean up jobs in the cluster or uninstall the, after completion.                                                                                          |retain_clean_jobs|
+|          Name          |                                                          Default Value                                                           |Required|                                                                                Description                                                                                 |      Setting name      |
+|------------------------|----------------------------------------------------------------------------------------------------------------------------------|--------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------|
+|defaults_path           |.                                                                                                                                 |False   |The path to the folder containing the defaults.yaml file and the environment defaults files. Paths can either be absolute or relative to `config.yaml`                      |defaults_path           |
+|environment             |PydanticUndefined                                                                                                                 |False   |The environment you want to generate and deploy the pipeline to. Suffix your environment files with this value (e.g. defaults_development.yaml for environment=development).|environment             |
+|kafka_brokers           |PydanticUndefined                                                                                                                 |False   |The comma separated Kafka brokers address.                                                                                                                                  |kafka_brokers           |
+|defaults_filename_prefix|defaults                                                                                                                          |False   |The name of the defaults file and the prefix of the defaults environment file.                                                                                              |defaults_filename_prefix|
+|topic_name_config       |default_output_topic_name='${pipeline_name}-${component_name}' default_error_topic_name='${pipeline_name}-${component_name}-error'|False   |Configure the topic name variables you can use in the pipeline definition.                                                                                                  |topic_name_config       |
+|schema_registry         |enabled=False url=Url('http://localhost:8081/')                                                                                   |False   |Configuration for Schema Registry.                                                                                                                                          |schema_registry         |
+|kafka_rest              |url=Url('http://localhost:8082/')                                                                                                 |False   |Configuration for Kafka REST Proxy.                                                                                                                                         |kafka_rest              |
+|kafka_connect           |url=Url('http://localhost:8083/')                                                                                                 |False   |Configuration for Kafka Connect.                                                                                                                                            |kafka_connect           |
+|timeout                 |300                                                                                                                               |False   |The timeout in seconds that specifies when actions like deletion or deploy timeout.                                                                                         |timeout                 |
+|create_namespace        |False                                                                                                                             |False   |Flag for `helm upgrade --install`. Create the release namespace if not present.                                                                                             |create_namespace        |
+|helm_config             |context=None debug=False api_version=None                                                                                         |False   |Global flags for Helm.                                                                                                                                                      |helm_config             |
+|helm_diff_config        |ignore=set()                                                                                                                      |False   |Configure Helm Diff.                                                                                                                                                        |helm_diff_config        |
+|retain_clean_jobs       |False                                                                                                                             |False   |Whether to retain clean up jobs in the cluster or uninstall the, after completion.                                                                                          |retain_clean_jobs       |

docs/docs/schema/config.json

@@ -5,15 +5,33 @@
             "description": "Global Helm configuration.",
             "properties": {
                 "api_version": {
+                    "anyOf": [
+                        {
+                            "type": "string"
+                        },
+                        {
+                            "type": "null"
+                        }
+                    ],
+                    "default": null,
                     "description": "Kubernetes API version used for Capabilities.APIVersions",
-                    "title": "API version",
-                    "type": "string"
+                    "title": "API version"
                 },
                 "context": {
+                    "anyOf": [
+                        {
+                            "type": "string"
+                        },
+                        {
+                            "type": "null"
+                        }
+                    ],
+                    "default": null,
                     "description": "Name of kubeconfig context (`--kube-context`)",
-                    "example": "dev-storage",
-                    "title": "Context",
-                    "type": "string"
+                    "examples": [
+                        "dev-storage"
+                    ],
+                    "title": "Context"
                 },
                 "debug": {
                     "default": false,
@@ -29,7 +47,9 @@
             "properties": {
                 "ignore": {
                     "description": "Set of keys that should not be checked.",
-                    "example": "- name\n- imageTag",
+                    "examples": [
+                        "- name\n- imageTag"
+                    ],
                     "items": {
                         "type": "string"
                     },