feat: specify environment via cli

bakdata · Dec 1, 2023 · a442888 · a442888
1 parent e4c154a
commit a442888
Show file tree

Hide file tree

Showing 10 changed files with 75 additions and 44 deletions.
diff --git a/docs/docs/resources/variables/cli_env_vars.env b/docs/docs/resources/variables/cli_env_vars.env
@@ -15,6 +15,11 @@ KPOPS_DEFAULT_PATH # No default value, not required
 # Path to dotenv file. Multiple files can be provided. The files will
 # be loaded in order, with each file overriding the previous one.
 KPOPS_DOTENV_PATH # No default value, not required
+# The environment you want to generate and deploy the pipeline to.
+# Suffix your environment files with this value (e.g.
+# defaults_development.yaml for environment=development). To be
+# defined only in the default config definition, i.e. `config.yaml`.
+KPOPS_ENVIRONMENT # No default value, not required
 # Path to YAML with pipeline definition
 KPOPS_PIPELINE_PATH # No default value, required
 # Comma separated list of steps to apply the command on

diff --git a/docs/docs/resources/variables/cli_env_vars.md b/docs/docs/resources/variables/cli_env_vars.md
@@ -1,10 +1,11 @@
 These variables are a lower priority alternative to the commands' flags. If a variable is set, the corresponding flag does not have to be specified in commands. Variables marked as required can instead be set as flags.
 
-|         Name          |Default Value|Required|                                                            Description                                                            |
-|-----------------------|-------------|--------|-----------------------------------------------------------------------------------------------------------------------------------|
-|KPOPS_PIPELINE_BASE_DIR|.            |False   |Base directory to the pipelines (default is current working directory)                                                             |
-|KPOPS_CONFIG_PATH      |.            |False   |Path to the dir containing config.yaml files                                                                                       |
-|KPOPS_DEFAULT_PATH     |             |False   |Path to defaults folder                                                                                                            |
-|KPOPS_DOTENV_PATH      |             |False   |Path to dotenv file. Multiple files can be provided. The files will be loaded in order, with each file overriding the previous one.|
-|KPOPS_PIPELINE_PATH    |             |True    |Path to YAML with pipeline definition                                                                                              |
-|KPOPS_PIPELINE_STEPS   |             |False   |Comma separated list of steps to apply the command on                                                                              |
+|         Name          |Default Value|Required|                                                                                                                     Description                                                                                                                     |
+|-----------------------|-------------|--------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+|KPOPS_PIPELINE_BASE_DIR|.            |False   |Base directory to the pipelines (default is current working directory)                                                                                                                                                                               |
+|KPOPS_CONFIG_PATH      |.            |False   |Path to the dir containing config.yaml files                                                                                                                                                                                                         |
+|KPOPS_DEFAULT_PATH     |             |False   |Path to defaults folder                                                                                                                                                                                                                              |
+|KPOPS_DOTENV_PATH      |             |False   |Path to dotenv file. Multiple files can be provided. The files will be loaded in order, with each file overriding the previous one.                                                                                                                  |
+|KPOPS_ENVIRONMENT      |             |False   |The environment you want to generate and deploy the pipeline to. Suffix your environment files with this value (e.g. defaults_development.yaml for environment=development). To be defined only in the default config definition, i.e. `config.yaml`.|
+|KPOPS_PIPELINE_PATH    |             |True    |Path to YAML with pipeline definition                                                                                                                                                                                                                |
+|KPOPS_PIPELINE_STEPS   |             |False   |Comma separated list of steps to apply the command on                                                                                                                                                                                                |
diff --git a/docs/docs/user/references/cli-commands.md b/docs/docs/user/references/cli-commands.md
@@ -47,6 +47,7 @@ $ kpops clean [OPTIONS] PIPELINE_PATH [COMPONENTS_MODULE]
 * `--filter-type [include|exclude]`: Whether the --steps option should include/exclude the steps  [default: include]
 * `--dry-run / --execute`: Whether to dry run the command or execute it  [default: dry-run]
 * `--verbose / --no-verbose`: Enable verbose printing  [default: no-verbose]
+* `--environment TEXT`: The environment you want to generate and deploy the pipeline to. Suffix your environment files with this value (e.g. defaults_development.yaml for environment=development). To be defined only in the default config definition, i.e. `config.yaml`.  [env var: KPOPS_ENVIRONMENT]
 * `--help`: Show this message and exit.
 
 ## `kpops deploy`
@@ -74,6 +75,7 @@ $ kpops deploy [OPTIONS] PIPELINE_PATH [COMPONENTS_MODULE]
 * `--filter-type [include|exclude]`: Whether the --steps option should include/exclude the steps  [default: include]
 * `--dry-run / --execute`: Whether to dry run the command or execute it  [default: dry-run]
 * `--verbose / --no-verbose`: Enable verbose printing  [default: no-verbose]
+* `--environment TEXT`: The environment you want to generate and deploy the pipeline to. Suffix your environment files with this value (e.g. defaults_development.yaml for environment=development). To be defined only in the default config definition, i.e. `config.yaml`.  [env var: KPOPS_ENVIRONMENT]
 * `--help`: Show this message and exit.
 
 ## `kpops destroy`
@@ -101,6 +103,7 @@ $ kpops destroy [OPTIONS] PIPELINE_PATH [COMPONENTS_MODULE]
 * `--filter-type [include|exclude]`: Whether the --steps option should include/exclude the steps  [default: include]
 * `--dry-run / --execute`: Whether to dry run the command or execute it  [default: dry-run]
 * `--verbose / --no-verbose`: Enable verbose printing  [default: no-verbose]
+* `--environment TEXT`: The environment you want to generate and deploy the pipeline to. Suffix your environment files with this value (e.g. defaults_development.yaml for environment=development). To be defined only in the default config definition, i.e. `config.yaml`.  [env var: KPOPS_ENVIRONMENT]
 * `--help`: Show this message and exit.
 
 ## `kpops generate`
@@ -128,6 +131,7 @@ $ kpops generate [OPTIONS] PIPELINE_PATH [COMPONENTS_MODULE]
 * `--steps TEXT`: Comma separated list of steps to apply the command on  [env var: KPOPS_PIPELINE_STEPS]
 * `--filter-type [include|exclude]`: Whether the --steps option should include/exclude the steps  [default: include]
 * `--verbose / --no-verbose`: Enable verbose printing  [default: no-verbose]
+* `--environment TEXT`: The environment you want to generate and deploy the pipeline to. Suffix your environment files with this value (e.g. defaults_development.yaml for environment=development). To be defined only in the default config definition, i.e. `config.yaml`.  [env var: KPOPS_ENVIRONMENT]
 * `--help`: Show this message and exit.
 
 ## `kpops reset`
@@ -155,6 +159,7 @@ $ kpops reset [OPTIONS] PIPELINE_PATH [COMPONENTS_MODULE]
 * `--filter-type [include|exclude]`: Whether the --steps option should include/exclude the steps  [default: include]
 * `--dry-run / --execute`: Whether to dry run the command or execute it  [default: dry-run]
 * `--verbose / --no-verbose`: Enable verbose printing  [default: no-verbose]
+* `--environment TEXT`: The environment you want to generate and deploy the pipeline to. Suffix your environment files with this value (e.g. defaults_development.yaml for environment=development). To be defined only in the default config definition, i.e. `config.yaml`.  [env var: KPOPS_ENVIRONMENT]
 * `--help`: Show this message and exit.
 
 ## `kpops schema`

diff --git a/kpops/cli/main.py b/kpops/cli/main.py
@@ -113,8 +113,9 @@ class FilterType(str, Enum):
     help="Custom Python module containing your project-specific components",
 )
 
-ENVIRONMENT: str | None = typer.Argument(
+ENVIRONMENT: str | None = typer.Option(
     default=None,
+    envvar=f"{ENV_PREFIX}ENVIRONMENT",
     help=KpopsConfig.model_fields["environment"].description,
 )
 
@@ -212,10 +213,15 @@ def log_action(action: str, pipeline_component: PipelineComponent):
 
 
 def create_kpops_config(
-    config: Path, defaults: Optional[Path], verbose: bool, dotenv: Optional[list[Path]]
+    config: Path,
+    defaults: Optional[Path],
+    verbose: bool,
+    dotenv: Optional[list[Path]],
+    environment: Optional[str],
 ) -> KpopsConfig:
     setup_logging_level(verbose)
     YamlConfigSettingsSource.path_to_settings = config
+    YamlConfigSettingsSource.environment = environment
     kpops_config = KpopsConfig(
         _env_file=dotenv  # pyright: ignore[reportGeneralTypeIssues]
     )
@@ -270,9 +276,9 @@ def generate(
     steps: Optional[str] = PIPELINE_STEPS,
     filter_type: FilterType = FILTER_TYPE,
     verbose: bool = VERBOSE_OPTION,
-    # environment: Optional[str] = ENVIRONMENT
+    environment: Optional[str] = ENVIRONMENT,
 ) -> Pipeline:
-    kpops_config = create_kpops_config(config, defaults, verbose, dotenv)
+    kpops_config = create_kpops_config(config, defaults, verbose, dotenv, environment)
     pipeline = setup_pipeline(
         pipeline_base_dir, pipeline_path, components_module, kpops_config
     )
@@ -307,9 +313,9 @@ def deploy(
     filter_type: FilterType = FILTER_TYPE,
     dry_run: bool = DRY_RUN,
     verbose: bool = VERBOSE_OPTION,
-    # environment: Optional[str] = ENVIRONMENT
+    environment: Optional[str] = ENVIRONMENT,
 ) -> None:
-    kpops_config = create_kpops_config(config, defaults, verbose, dotenv)
+    kpops_config = create_kpops_config(config, defaults, verbose, dotenv, environment)
     pipeline = setup_pipeline(
         pipeline_base_dir, pipeline_path, components_module, kpops_config
     )
@@ -334,9 +340,9 @@ def destroy(
     filter_type: FilterType = FILTER_TYPE,
     dry_run: bool = DRY_RUN,
     verbose: bool = VERBOSE_OPTION,
-    # environment: Optional[str] = ENVIRONMENT
+    environment: Optional[str] = ENVIRONMENT,
 ) -> None:
-    kpops_config = create_kpops_config(config, defaults, verbose, dotenv)
+    kpops_config = create_kpops_config(config, defaults, verbose, dotenv, environment)
     pipeline = setup_pipeline(
         pipeline_base_dir, pipeline_path, components_module, kpops_config
     )
@@ -360,9 +366,9 @@ def reset(
     filter_type: FilterType = FILTER_TYPE,
     dry_run: bool = DRY_RUN,
     verbose: bool = VERBOSE_OPTION,
-    # environment: Optional[str] = ENVIRONMENT
+    environment: Optional[str] = ENVIRONMENT,
 ) -> None:
-    kpops_config = create_kpops_config(config, defaults, verbose, dotenv)
+    kpops_config = create_kpops_config(config, defaults, verbose, dotenv, environment)
     pipeline = setup_pipeline(
         pipeline_base_dir, pipeline_path, components_module, kpops_config
     )
@@ -387,9 +393,9 @@ def clean(
     filter_type: FilterType = FILTER_TYPE,
     dry_run: bool = DRY_RUN,
     verbose: bool = VERBOSE_OPTION,
-    # environment: Optional[str] = ENVIRONMENT
+    environment: Optional[str] = ENVIRONMENT,
 ) -> None:
-    kpops_config = create_kpops_config(config, defaults, verbose, dotenv)
+    kpops_config = create_kpops_config(config, defaults, verbose, dotenv, environment)
     pipeline = setup_pipeline(
         pipeline_base_dir, pipeline_path, components_module, kpops_config
     )

diff --git a/kpops/config.py b/kpops/config.py
@@ -142,8 +142,8 @@ def settings_customise_sources(
     ):
         return (
             env_settings,
+            dotenv_settings,
             init_settings,
             YamlConfigSettingsSource(settings_cls),
-            dotenv_settings,
             file_secret_settings,
         )
diff --git a/kpops/utils/pydantic.py b/kpops/utils/pydantic.py
@@ -128,11 +128,9 @@ def __init__(self, settings_cls):
         elif self.path_to_settings.is_file():
             msg = f"Path to config directory {self.path_to_settings} must point to a directory."
             raise ValueError(msg)
-        # Collect settings files
         default_settings = self.__load_settings(
             self.path_to_settings / f"{self.settings_file_base_name}.yaml"
         )
-        self.environment = self.environment or default_settings.get("environment")
         env_settings = (
             self.__load_settings(
                 self.path_to_settings
@@ -141,12 +139,6 @@ def __init__(self, settings_cls):
             if self.environment
             else {}
         )
-        if env_settings.get("environment"):
-            env_settings["environment"] = self.environment
-            self.log.warning(
-                f"Ignoring environment setting in {self.path_to_settings}, "
-                "it must not be specified in an environment-specific config."
-            )
         self.settings = update_nested_pair(env_settings, default_settings)
 
     @staticmethod

diff --git a/tests/pipeline/resources/multi-config/.env b/tests/pipeline/resources/multi-config/.env
@@ -0,0 +1 @@
+KPOPS_environment="no_env"
diff --git a/tests/pipeline/resources/multi-config/config_env.yaml b/tests/pipeline/resources/multi-config/config_env.yaml
@@ -1,4 +1,4 @@
 environment: "correct"
 schema_registry:
   enabled: true
-  url: "http://correct:8081"
+  url: "http://env:8081"
diff --git a/tests/pipeline/resources/multi-config/config_no_env.yaml b/tests/pipeline/resources/multi-config/config_no_env.yaml
@@ -0,0 +1,3 @@
+schema_registry:
+  enabled: true
+  url: "http://noenv:8081"
diff --git a/tests/pipeline/test_pipeline.py b/tests/pipeline/test_pipeline.py
@@ -396,7 +396,7 @@ def test_with_custom_config_with_absolute_defaults_path(
         config_dict["defaults_path"] = str(
             (RESOURCE_PATH / "no-topics-defaults").absolute(),
         )
-        temp_config_path = RESOURCE_PATH / "custom-config/temp_config.yaml"
+        temp_config_path = RESOURCE_PATH / "custom-config/config_custom.yaml"
         try:
             with temp_config_path.open("w") as abs_config_yaml:
                 yaml.dump(config_dict, abs_config_yaml)
@@ -408,7 +408,7 @@ def test_with_custom_config_with_absolute_defaults_path(
                     str(PIPELINE_BASE_DIR_PATH),
                     str(RESOURCE_PATH / "custom-config/pipeline.yaml"),
                     "--config",
-                    str(temp_config_path),
+                    str(temp_config_path.parent),
                 ],
                 catch_exceptions=False,
             )
@@ -505,7 +505,10 @@ def test_nested_config_env_vars(self, monkeypatch: pytest.MonkeyPatch):
             == "http://somename:1234/"
         )
 
-    def test_env_specific_config(self, caplog: pytest.LogCaptureFixture):
+    def test_env_specific_config_env_def_in_env_var(
+        self, monkeypatch: pytest.MonkeyPatch
+    ):
+        monkeypatch.setenv(name="KPOPS_ENVIRONMENT", value="no_env")
         config_path = str(RESOURCE_PATH / "multi-config")
         result = runner.invoke(
             app,
@@ -525,18 +528,33 @@ def test_env_specific_config(self, caplog: pytest.LogCaptureFixture):
         enriched_pipeline: dict = yaml.safe_load(result.stdout)
         assert (
             enriched_pipeline["components"][0]["app"]["streams"]["schemaRegistryUrl"]
-            == "http://correct:8081/"
+            == "http://noenv:8081/"
+        )
+
+    def test_env_specific_config_env_def_in_cli(self):
+        config_path = str(RESOURCE_PATH / "multi-config")
+        result = runner.invoke(
+            app,
+            [
+                "generate",
+                "--pipeline-base-dir",
+                str(PIPELINE_BASE_DIR_PATH),
+                str(RESOURCE_PATH / "custom-config/pipeline.yaml"),
+                "--config",
+                config_path,
+                "--defaults",
+                str(RESOURCE_PATH),
+                "--environment",
+                "no_env",
+            ],
+            catch_exceptions=False,
+        )
+        assert result.exit_code == 0
+        enriched_pipeline: dict = yaml.safe_load(result.stdout)
+        assert (
+            enriched_pipeline["components"][0]["app"]["streams"]["schemaRegistryUrl"]
+            == "http://noenv:8081/"
         )
-        assert caplog.record_tuples == [
-            (
-                "root",
-                logging.WARNING,
-                (
-                    f"Ignoring environment setting in {config_path}, "
-                    "it must not be specified in an environment-specific config."
-                ),
-            )
-        ]
 
     def test_model_serialization(self, snapshot: SnapshotTest):
         """Test model serialization of component containing pathlib.Path attribute."""