Merge remote-tracking branch 'origin/main' into ci/replace-black-with…

…-ruff

.gitignore

@@ -6,3 +6,4 @@
 pipelines/
 defaults/
 site/
+scratch*

CHANGELOG.md

@@ -1,4 +1,21 @@
 # Changelog
+## [2.0.10](https://github.com/bakdata/kpops/releases/tag/2.0.10) - Release Date: [2023-10-12]
+
+### 🌀 Miscellaneous
+
+- Fix environment variables documentation generation - [#362](https://github.com/bakdata/kpops/pull/362)
+
+- Introduce ruff - [#363](https://github.com/bakdata/kpops/pull/363)
+
+- Print details on connector name mismatch error - [#369](https://github.com/bakdata/kpops/pull/369)
+
+- Enable transparent OS environment lookups from internal environment - [#368](https://github.com/bakdata/kpops/pull/368)
+
+
+
+
+
+
 ## [2.0.9](https://github.com/bakdata/kpops/releases/tag/2.0.9) - Release Date: [2023-09-19]
 
 ### 🐛 Fixes

kpops/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "2.0.9"
+__version__ = "2.0.10"
 
 # export public API functions
 from kpops.cli.main import clean, deploy, destroy, generate, reset

kpops/components/base_components/kafka_connector.py

@@ -87,7 +87,7 @@ def connector_config_should_have_component_name(
         component_name = values["prefix"] + values["name"]
         connector_name: str | None = app.get("name")
         if connector_name is not None and connector_name != component_name:
-            msg = "Connector name should be the same as component name"
+            msg = f"Connector name '{connector_name}' should be the same as component name '{component_name}'"
             raise ValueError(msg)
         app["name"] = component_name
         return app

kpops/components/streams_bootstrap/producer/producer_app.py

@@ -1,4 +1,4 @@
-from __future__ import annotations
+# from __future__ import annotations
 
 from pydantic import Field
 from typing_extensions import override

kpops/components/streams_bootstrap/streams/streams_app.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 from pydantic import Field
 from typing_extensions import override
 

kpops/utils/environment.py

@@ -1,36 +1,42 @@
 import os
-import platform
 from collections import UserDict
-from collections.abc import Callable
+from collections.abc import ItemsView, KeysView, MutableMapping, ValuesView
 
 
-class Environment(UserDict):
-    def __init__(self, mapping=None, /, **kwargs) -> None:
-        transformation = Environment.__get_transformation()
-        if mapping is not None:
-            mapping = {transformation(key): value for key, value in mapping.items()}
-        else:
+class Environment(UserDict[str, str]):
+    """Internal environment wrapping OS environment."""
+
+    def __init__(
+        self, mapping: MutableMapping[str, str] | None = None, /, **kwargs: str
+    ) -> None:
+        self._global = os.environ
+        if mapping is None:
             mapping = {}
         if kwargs:
-            mapping.update(
-                {transformation(key): value for key, value in kwargs.items()}
-            )
+            mapping.update(**kwargs)
         super().__init__(mapping)
 
-    @staticmethod
-    def __key_camel_case_transform(key: str) -> str:
-        return key.lower()
+    def __getitem__(self, key: str) -> str:
+        try:
+            return self.data[key]
+        except KeyError:
+            return self._global[key]
+
+    def __contains__(self, key: object) -> bool:
+        return super().__contains__(key) or self._global.__contains__(key)
+
+    @property
+    def _dict(self) -> dict[str, str]:
+        return {**self._global, **self.data}
+
+    def keys(self) -> KeysView[str]:
+        return KeysView(self._dict)
 
-    @staticmethod
-    def __key_identity_transform(key: str) -> str:
-        return key
+    def values(self) -> ValuesView[str]:
+        return ValuesView(self._dict)
 
-    @staticmethod
-    def __get_transformation() -> Callable[[str], str]:
-        if platform.system() == "Windows":
-            return Environment.__key_camel_case_transform
-        else:
-            return Environment.__key_identity_transform
+    def items(self) -> ItemsView[str, str]:
+        return ItemsView(self._dict)
 
 
-ENV = Environment(os.environ)
+ENV = Environment()

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "kpops"
-version = "2.0.9"
+version = "2.0.10"
 description = "KPOps is a tool to deploy Kafka pipelines to Kubernetes"
 authors = ["bakdata <[email protected]>"]
 license = "MIT"
@@ -158,7 +158,7 @@ split-on-trailing-comma = false
 extend-immutable-calls = ["typer.Argument"]
 
 [tool.ruff.flake8-type-checking]
-runtime-evaluated-base-classes = ["pydantic.BaseModel", "kpops.components.base_components.kafka_app.KafkaApp"]
+runtime-evaluated-base-classes = ["pydantic.BaseModel"]
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]

tests/components/test_kafka_connector.py

@@ -1,3 +1,4 @@
+import re
 from pathlib import Path
 from unittest.mock import MagicMock
 
@@ -85,7 +86,10 @@ def test_connector_config_name_override(
         assert connector.app.name == CONNECTOR_FULL_NAME
 
         with pytest.raises(
-            ValueError, match="Connector name should be the same as component name"
+            ValueError,
+            match=re.escape(
+                f"Connector name 'different-name' should be the same as component name '{CONNECTOR_FULL_NAME}'"
+            ),
         ):
             KafkaConnector(
                 name=CONNECTOR_NAME,
@@ -96,7 +100,10 @@ def test_connector_config_name_override(
             )
 
         with pytest.raises(
-            ValueError, match="Connector name should be the same as component name"
+            ValueError,
+            match=re.escape(
+                f"Connector name '' should be the same as component name '{CONNECTOR_FULL_NAME}'"
+            ),
         ):
             KafkaConnector(
                 name=CONNECTOR_NAME,

tests/pipeline/resources/pipeline-with-env-defaults/defaults.yaml

@@ -1,5 +1,5 @@
 kubernetes-app:
-  name: "${component_type}"
+  name: ${component_type}
   namespace: example-namespace
 kafka-app:
   app:
@@ -24,7 +24,7 @@ streams-app: # inherits from kafka-app
           cleanup.policy: compact,delete
 
 kafka-connector:
-  name: "sink-connector"
+  name: sink-connector
   app:
     batch.size: "2000"
     behavior.on.malformed.documents: "warn"
@@ -34,6 +34,5 @@ kafka-connector:
     key.ignore: "false"
     linger.ms: "5000"
     max.buffered.records: "20000"
-    name: "sink-connector"
     read.timeout.ms: "120000"
     tasks.max: "1"

tests/pipeline/resources/pipeline-with-env-defaults/defaults_development.yaml

@@ -1,3 +1,3 @@
 kubernetes-app:
-  name: "${component_type}-development"
+  name: ${component_type}-development
   namespace: development-namespace

tests/utils/test_environment.py

@@ -1,117 +1,81 @@
-from unittest.mock import patch
+import os
+from collections.abc import ItemsView, KeysView, ValuesView
+from unittest.mock import ANY
 
 import pytest
 
 from kpops.utils.environment import Environment
 
 
-@pytest.fixture()
-def fake_environment_windows():
-    return {"MY": "fake", "ENVIRONMENT": "here"}
+@pytest.fixture(autouse=True)
+def environment(monkeypatch: pytest.MonkeyPatch) -> Environment:
+    for key in os.environ:
+        monkeypatch.delenv(key)
+    monkeypatch.setenv("MY", "fake")
+    monkeypatch.setenv("ENVIRONMENT", "here")
+    return Environment()
 
 
-@pytest.fixture()
-def fake_environment_linux():
-    return {"my": "fake", "environment": "here"}
+def test_get_item(environment: Environment):
+    assert environment["MY"] == "fake"
+    assert environment["ENVIRONMENT"] == "here"
 
 
-@patch("platform.system")
-def test_normal_behaviour_get_item(system, fake_environment_linux):
-    system.return_value = "Linux"
-    environment = Environment(fake_environment_linux)
-
-    assert environment["my"] == "fake"
-    assert environment["environment"] == "here"
-
-
-@patch("platform.system")
-def test_normal_behaviour_get_item_as_kwargs(system, fake_environment_linux):
-    system.return_value = "Linux"
-    environment = Environment(**fake_environment_linux)
-
-    assert environment["my"] == "fake"
-    assert environment["environment"] == "here"
-
-
-@patch("platform.system")
-def test_normal_behaviour_keys_transformation(system, fake_environment_linux):
-    system.return_value = "Linux"
-    environment = Environment(fake_environment_linux)
-    keys = set(environment.keys())
-
-    assert "my" in keys
-    assert "environment" in keys
-
-
-@patch("platform.system")
-def test_normal_behaviour_set_key(system, fake_environment_linux):
-    system.return_value = "Linux"
-    environment = Environment(fake_environment_linux)
+def test_set_item(environment: Environment):
     environment["extra"] = "key"
 
     keys = set(environment.keys())
-    assert "my" in keys
-    assert "environment" in keys
+    assert "MY" in keys
+    assert "ENVIRONMENT" in keys
     assert "extra" in keys
     assert environment["extra"] == "key"
 
 
-@patch("platform.system")
-def test_windows_behaviour_set_key(system, fake_environment_windows):
-    system.return_value = "Windows"
-    environment = Environment(fake_environment_windows)
-    environment["extra"] = "key"
-
-    keys = set(environment.keys())
-    assert "my" in keys
-    assert "environment" in keys
-    assert "extra" in keys
-    assert environment["extra"] == "key"
-
-
-@patch("platform.system")
-def test_normal_behaviour_keys_transformation_kwargs(system, fake_environment_linux):
-    system.return_value = "Linux"
-    environment = Environment(**fake_environment_linux)
-
-    keys = set(environment.keys())
-    assert "my" in keys
-    assert "environment" in keys
-
-
-@patch("platform.system")
-def test_windows_behaviour_keys_transformation(system, fake_environment_windows):
-    system.return_value = "Windows"
-    environment = Environment(fake_environment_windows)
-
-    keys = set(environment.keys())
-    assert "my" in keys
-    assert "environment" in keys
-
-
-@patch("platform.system")
-def test_windows_behaviour_keys_transformation_as_kwargs(
-    system, fake_environment_windows
-):
-    system.return_value = "Windows"
-    environment = Environment(**fake_environment_windows)
-    keys = set(environment.keys())
-    assert "my" in keys
-    assert "environment" in keys
-
-
-@patch("platform.system")
-def test_windows_behaviour_get_item(system, fake_environment_windows):
-    system.return_value = "Windows"
-
-    environment = Environment(fake_environment_windows)
-    assert environment["my"] == "fake"
-    assert environment["environment"] == "here"
-
-
-@patch("platform.system")
-def test_windows_behaviour_get_item_as_kwargs(system, fake_environment_windows):
-    system.return_value = "Windows"
-    environment = Environment(**fake_environment_windows)
-    assert environment["my"] == "fake"
-    assert environment["environment"] == "here"
+def test_update_os_environ(environment: Environment):
+    with pytest.raises(KeyError):
+        environment["TEST"]
+    os.environ["TEST"] = "test"
+    assert "TEST" in environment
+    assert environment["TEST"] == "test"
+    keys = environment.keys()
+    assert isinstance(keys, KeysView)
+    assert "TEST" in keys
+    values = environment.values()
+    assert isinstance(values, ValuesView)
+    assert "test" in values
+    items = environment.items()
+    assert isinstance(items, ItemsView)
+    d = dict(items)
+    assert d["TEST"] == "test"
+
+
+def test_mapping():
+    environment = Environment({"kwarg1": "value1", "kwarg2": "value2"})
+    assert environment["MY"] == "fake"
+    assert environment["ENVIRONMENT"] == "here"
+    assert environment["kwarg1"] == "value1"
+    assert environment["kwarg2"] == "value2"
+
+
+def test_kwargs():
+    environment = Environment(kwarg1="value1", kwarg2="value2")
+    assert environment["MY"] == "fake"
+    assert environment["ENVIRONMENT"] == "here"
+    assert environment["kwarg1"] == "value1"
+    assert environment["kwarg2"] == "value2"
+
+
+def test_dict(environment: Environment):
+    assert environment._dict == {
+        "MY": "fake",
+        "ENVIRONMENT": "here",
+        "PYTEST_CURRENT_TEST": ANY,
+    }
+
+
+def test_dict_unpacking(environment: Environment):
+    assert {**environment} == {
+        "MY": "fake",
+        "ENVIRONMENT": "here",
+        "PYTEST_CURRENT_TEST": ANY,
+    }