diff --git a/hooks/gen_docs/gen_docs_components.py b/hooks/gen_docs/gen_docs_components.py index b6c733267..ed28dfe6e 100644 --- a/hooks/gen_docs/gen_docs_components.py +++ b/hooks/gen_docs/gen_docs_components.py @@ -9,7 +9,10 @@ from hooks import ROOT from kpops.api.registry import Registry -from kpops.components import KafkaConnector, PipelineComponent +from kpops.components.base_components.kafka_connector import KafkaConnector +from kpops.components.base_components.pipeline_component import ( + PipelineComponent, +) from kpops.utils.colorify import redify, yellowify from kpops.utils.pydantic import issubclass_patched from kpops.utils.yaml import load_yaml_file diff --git a/kpops/__init__.py b/kpops/__init__.py deleted file mode 100644 index b0b3c6651..000000000 --- a/kpops/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -__version__ = "6.0.1" - -# export public API functions -from kpops.api import clean, deploy, destroy, generate, init, manifest, reset - -__all__ = ( - "generate", - "manifest", - "deploy", - "destroy", - "reset", - "clean", - "init", -) diff --git a/kpops/api/__init__.py b/kpops/api/__init__.py index aa7133fa2..5a3a81102 100644 --- a/kpops/api/__init__.py +++ b/kpops/api/__init__.py @@ -4,7 +4,6 @@ from pathlib import Path from typing import TYPE_CHECKING -import kpops from kpops.api.logs import log, log_action from kpops.api.options import FilterType from kpops.api.registry import Registry @@ -23,8 +22,8 @@ from kpops.utils.cli_commands import init_project if TYPE_CHECKING: - from kpops.components import PipelineComponent from kpops.components.base_components.models.resource import Resource + from kpops.components.base_components.pipeline_component import PipelineComponent from kpops.config import KpopsConfig @@ -68,7 +67,7 @@ def manifest( environment: str | None = None, verbose: bool = False, ) -> list[Resource]: - pipeline = kpops.generate( + pipeline = generate( pipeline_path=pipeline_path, dotenv=dotenv, config=config, @@ -95,7 +94,7 @@ def deploy( verbose: bool = True, parallel: bool = False, ): - pipeline = kpops.generate( + pipeline = generate( pipeline_path=pipeline_path, dotenv=dotenv, config=config, @@ -131,7 +130,7 @@ def destroy( verbose: bool = True, parallel: bool = False, ): - pipeline = kpops.generate( + pipeline = generate( pipeline_path=pipeline_path, dotenv=dotenv, config=config, @@ -169,7 +168,7 @@ def reset( verbose: bool = True, parallel: bool = False, ): - pipeline = kpops.generate( + pipeline = generate( pipeline_path=pipeline_path, dotenv=dotenv, config=config, @@ -206,7 +205,7 @@ def clean( verbose: bool = True, parallel: bool = False, ): - pipeline = kpops.generate( + pipeline = generate( pipeline_path=pipeline_path, dotenv=dotenv, config=config, diff --git a/kpops/api/logs.py b/kpops/api/logs.py index e9a833aba..979b5a36c 100644 --- a/kpops/api/logs.py +++ b/kpops/api/logs.py @@ -6,7 +6,7 @@ import typer if TYPE_CHECKING: - from kpops.components import PipelineComponent + from kpops.components.base_components.pipeline_component import PipelineComponent class CustomFormatter(logging.Formatter): diff --git a/kpops/api/options.py b/kpops/api/options.py index dc116bd35..22fda2542 100644 --- a/kpops/api/options.py +++ b/kpops/api/options.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from kpops.components import PipelineComponent + from kpops.components.base_components.pipeline_component import PipelineComponent from kpops.pipeline import ComponentFilterPredicate diff --git a/kpops/api/registry.py b/kpops/api/registry.py index 592f481d2..7a5b4401d 100644 --- a/kpops/api/registry.py +++ b/kpops/api/registry.py @@ -11,14 +11,13 @@ from types import ModuleType from typing import TYPE_CHECKING, TypeVar -from kpops import __name__ from kpops.api.exception import ClassNotFoundError from kpops.components.base_components.pipeline_component import PipelineComponent if TYPE_CHECKING: from collections.abc import Iterator -KPOPS_MODULE = __name__ + "." +KPOPS_MODULE = "kpops." T = TypeVar("T") ClassDict = dict[str, type[T]] # type -> class diff --git a/kpops/cli/main.py b/kpops/cli/main.py index c84c4c403..3e669df54 100644 --- a/kpops/cli/main.py +++ b/kpops/cli/main.py @@ -5,8 +5,7 @@ import typer -import kpops -from kpops import __version__ +import kpops.api as kpops from kpops.api.file_type import KpopsFileType from kpops.api.options import FilterType from kpops.cli.utils import collect_pipeline_paths @@ -306,7 +305,7 @@ def clean( def version_callback(show_version: bool) -> None: if show_version: - typer.echo(f"KPOps {__version__}") + # typer.echo(f"KPOps {__version__}") # FIXME: read version from package raise typer.Exit diff --git a/kpops/components/__init__.py b/kpops/components/__init__.py deleted file mode 100644 index 3800b16d5..000000000 --- a/kpops/components/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -from kpops.components.base_components.helm_app import HelmApp -from kpops.components.base_components.kafka_app import KafkaApp -from kpops.components.base_components.kafka_connector import ( - KafkaConnector, - KafkaSinkConnector, - KafkaSourceConnector, -) -from kpops.components.base_components.kubernetes_app import KubernetesApp -from kpops.components.base_components.pipeline_component import PipelineComponent -from kpops.components.streams_bootstrap import StreamsBootstrap -from kpops.components.streams_bootstrap.producer.producer_app import ProducerApp -from kpops.components.streams_bootstrap.streams.streams_app import StreamsApp - -__all__ = ( - "HelmApp", - "KafkaApp", - "KafkaConnector", - "KafkaSinkConnector", - "KafkaSourceConnector", - "KubernetesApp", - "StreamsBootstrap", - "ProducerApp", - "StreamsApp", - "PipelineComponent", - "StreamsApp", - "ProducerApp", -) diff --git a/kpops/components/base_components/__init__.py b/kpops/components/base_components/__init__.py index e69de29bb..ff94dde1f 100644 --- a/kpops/components/base_components/__init__.py +++ b/kpops/components/base_components/__init__.py @@ -0,0 +1,19 @@ +from kpops.components.base_components.helm_app import HelmApp +from kpops.components.base_components.kafka_app import KafkaApp +from kpops.components.base_components.kafka_connector import ( + KafkaConnector, + KafkaSinkConnector, + KafkaSourceConnector, +) +from kpops.components.base_components.kubernetes_app import KubernetesApp +from kpops.components.base_components.pipeline_component import PipelineComponent + +__all__ = ( + "HelmApp", + "KafkaApp", + "KafkaConnector", + "KafkaSinkConnector", + "KafkaSourceConnector", + "KubernetesApp", + "PipelineComponent", +) diff --git a/kpops/components/base_components/kafka_app.py b/kpops/components/base_components/kafka_app.py index f52ca6e78..0ab4806b9 100644 --- a/kpops/components/base_components/kafka_app.py +++ b/kpops/components/base_components/kafka_app.py @@ -13,7 +13,7 @@ from kpops.components.base_components.helm_app import HelmAppValues from kpops.components.base_components.models.topic import KafkaTopic, KafkaTopicStr from kpops.components.base_components.pipeline_component import PipelineComponent -from kpops.components.streams_bootstrap import StreamsBootstrap +from kpops.components.common.streams_bootstrap import StreamsBootstrap from kpops.utils.docstring import describe_attr from kpops.utils.pydantic import ( CamelCaseConfigModel, diff --git a/kpops/components/streams_bootstrap/__init__.py b/kpops/components/streams_bootstrap/__init__.py index 1b02b091b..b4eb34b2f 100644 --- a/kpops/components/streams_bootstrap/__init__.py +++ b/kpops/components/streams_bootstrap/__init__.py @@ -1,31 +1,9 @@ -from abc import ABC - -from pydantic import Field - -from kpops.component_handlers.helm_wrapper.model import HelmRepoConfig -from kpops.components.base_components.helm_app import HelmApp -from kpops.utils.docstring import describe_attr - -STREAMS_BOOTSTRAP_HELM_REPO = HelmRepoConfig( - repository_name="bakdata-streams-bootstrap", - url="https://bakdata.github.io/streams-bootstrap/", +from kpops.components.common.streams_bootstrap import StreamsBootstrap +from kpops.components.streams_bootstrap.producer.producer_app import ProducerApp +from kpops.components.streams_bootstrap.streams.streams_app import StreamsApp + +__all__ = ( + "StreamsBootstrap", + "StreamsApp", + "ProducerApp", ) -STREAMS_BOOTSTRAP_VERSION = "2.9.0" - - -class StreamsBootstrap(HelmApp, ABC): - """Base for components with a streams-bootstrap Helm chart. - - :param repo_config: Configuration of the Helm chart repo to be used for - deploying the component, defaults to streams-bootstrap Helm repo - :param version: Helm chart version, defaults to "2.9.0" - """ - - repo_config: HelmRepoConfig = Field( - default=STREAMS_BOOTSTRAP_HELM_REPO, - description=describe_attr("repo_config", __doc__), - ) - version: str | None = Field( - default=STREAMS_BOOTSTRAP_VERSION, - description=describe_attr("version", __doc__), - ) diff --git a/kpops/components/streams_bootstrap/producer/producer_app.py b/kpops/components/streams_bootstrap/producer/producer_app.py index a9c06f37e..9674dd7da 100644 --- a/kpops/components/streams_bootstrap/producer/producer_app.py +++ b/kpops/components/streams_bootstrap/producer/producer_app.py @@ -12,7 +12,7 @@ OutputTopicTypes, TopicConfig, ) -from kpops.components.streams_bootstrap import StreamsBootstrap +from kpops.components.common.streams_bootstrap import StreamsBootstrap from kpops.components.streams_bootstrap.app_type import AppType from kpops.components.streams_bootstrap.producer.model import ProducerAppValues from kpops.utils.docstring import describe_attr diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index 9bd5d87c5..979c3c4c9 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -5,13 +5,10 @@ from typing_extensions import override from kpops.component_handlers.kubernetes.pvc_handler import PVCHandler -from kpops.components import HelmApp -from kpops.components.base_components.kafka_app import ( - KafkaApp, - KafkaAppCleaner, -) +from kpops.components.base_components.helm_app import HelmApp +from kpops.components.base_components.kafka_app import KafkaApp, KafkaAppCleaner from kpops.components.base_components.models.topic import KafkaTopic -from kpops.components.streams_bootstrap import StreamsBootstrap +from kpops.components.common.streams_bootstrap import StreamsBootstrap from kpops.components.streams_bootstrap.app_type import AppType from kpops.components.streams_bootstrap.streams.model import ( StreamsAppValues, diff --git a/kpops/utils/gen_schema.py b/kpops/utils/gen_schema.py index 6cf326ed0..4535c05e7 100644 --- a/kpops/utils/gen_schema.py +++ b/kpops/utils/gen_schema.py @@ -21,7 +21,7 @@ ) from kpops.api.registry import Registry -from kpops.components import ( +from kpops.components.base_components.pipeline_component import ( PipelineComponent, ) from kpops.config import KpopsConfig diff --git a/tests/api/test_registry.py b/tests/api/test_registry.py index 46e62f17f..e582b2c18 100644 --- a/tests/api/test_registry.py +++ b/tests/api/test_registry.py @@ -8,18 +8,18 @@ from kpops.api.exception import ClassNotFoundError from kpops.api.registry import Registry, _find_classes, _iter_namespace, find_class from kpops.component_handlers.schema_handler.schema_provider import SchemaProvider -from kpops.components import ( - HelmApp, - KafkaApp, +from kpops.components.base_components.helm_app import HelmApp +from kpops.components.base_components.kafka_app import KafkaApp +from kpops.components.base_components.kafka_connector import ( KafkaConnector, KafkaSinkConnector, KafkaSourceConnector, - KubernetesApp, - PipelineComponent, - ProducerApp, - StreamsApp, - StreamsBootstrap, ) +from kpops.components.base_components.kubernetes_app import KubernetesApp +from kpops.components.base_components.pipeline_component import PipelineComponent +from kpops.components.common.streams_bootstrap import StreamsBootstrap +from kpops.components.streams_bootstrap.producer.producer_app import ProducerApp +from kpops.components.streams_bootstrap.streams.streams_app import StreamsApp from tests.cli.resources.custom_module import CustomSchemaProvider diff --git a/tests/cli/test_init.py b/tests/cli/test_init.py index 8d4790d23..3109d16fc 100644 --- a/tests/cli/test_init.py +++ b/tests/cli/test_init.py @@ -3,7 +3,7 @@ from pytest_snapshot.plugin import Snapshot from typer.testing import CliRunner -import kpops +import kpops.api as kpops from kpops.cli.main import app from kpops.utils.cli_commands import create_config diff --git a/tests/cli/test_schema_generation.py b/tests/cli/test_schema_generation.py index a15028b5b..dc30f0543 100644 --- a/tests/cli/test_schema_generation.py +++ b/tests/cli/test_schema_generation.py @@ -10,7 +10,7 @@ from kpops.api.registry import Registry from kpops.cli.main import app -from kpops.components import PipelineComponent +from kpops.components.base_components.pipeline_component import PipelineComponent from kpops.utils.docstring import describe_attr RESOURCE_PATH = Path(__file__).parent / "resources" diff --git a/tests/components/test_kafka_sink_connector.py b/tests/components/test_kafka_sink_connector.py index 51b30a61d..1f9747280 100644 --- a/tests/components/test_kafka_sink_connector.py +++ b/tests/components/test_kafka_sink_connector.py @@ -12,8 +12,10 @@ KafkaConnectorConfig, KafkaConnectorType, ) -from kpops.components import KafkaSinkConnector -from kpops.components.base_components.kafka_connector import KafkaConnectorResetter +from kpops.components.base_components.kafka_connector import ( + KafkaConnectorResetter, + KafkaSinkConnector, +) from kpops.components.base_components.models.from_section import ( FromSection, FromTopic, diff --git a/tests/components/test_producer_app.py b/tests/components/test_producer_app.py index 4f7184ead..346913e57 100644 --- a/tests/components/test_producer_app.py +++ b/tests/components/test_producer_app.py @@ -7,13 +7,15 @@ from kpops.component_handlers import ComponentHandlers from kpops.component_handlers.helm_wrapper.model import HelmUpgradeInstallFlags from kpops.component_handlers.helm_wrapper.utils import create_helm_release_name -from kpops.components import ProducerApp from kpops.components.base_components.models.topic import ( KafkaTopic, OutputTopicTypes, TopicConfig, ) -from kpops.components.streams_bootstrap.producer.producer_app import ProducerAppCleaner +from kpops.components.streams_bootstrap.producer.producer_app import ( + ProducerApp, + ProducerAppCleaner, +) from kpops.config import KpopsConfig, TopicNameConfig from tests.components import PIPELINE_BASE_DIR diff --git a/tests/components/test_streams_app.py b/tests/components/test_streams_app.py index cb340174a..a72328d5e 100644 --- a/tests/components/test_streams_app.py +++ b/tests/components/test_streams_app.py @@ -12,7 +12,6 @@ HelmUpgradeInstallFlags, ) from kpops.component_handlers.helm_wrapper.utils import create_helm_release_name -from kpops.components import StreamsApp from kpops.components.base_components.models import TopicName from kpops.components.base_components.models.to_section import ( ToSection, @@ -27,6 +26,7 @@ StreamsAppAutoScaling, ) from kpops.components.streams_bootstrap.streams.streams_app import ( + StreamsApp, StreamsAppCleaner, ) from kpops.config import KpopsConfig, TopicNameConfig diff --git a/tests/components/test_streams_bootstrap.py b/tests/components/test_streams_bootstrap.py index a82fca8a9..d014d2b32 100644 --- a/tests/components/test_streams_bootstrap.py +++ b/tests/components/test_streams_bootstrap.py @@ -10,7 +10,7 @@ HelmUpgradeInstallFlags, ) from kpops.component_handlers.helm_wrapper.utils import create_helm_release_name -from kpops.components.streams_bootstrap import StreamsBootstrap +from kpops.components.common.streams_bootstrap import StreamsBootstrap from kpops.config import KpopsConfig from tests.components import PIPELINE_BASE_DIR diff --git a/tests/pipeline/test_components/components.py b/tests/pipeline/test_components/components.py index 36f24f938..4502ea69e 100644 --- a/tests/pipeline/test_components/components.py +++ b/tests/pipeline/test_components/components.py @@ -5,17 +5,15 @@ Schema, SchemaProvider, ) -from kpops.components import ( - KafkaSinkConnector, - PipelineComponent, - ProducerApp, - StreamsApp, -) +from kpops.components.base_components.kafka_connector import KafkaSinkConnector from kpops.components.base_components.models import ModelName, ModelVersion, TopicName from kpops.components.base_components.models.to_section import ( ToSection, ) from kpops.components.base_components.models.topic import OutputTopicTypes, TopicConfig +from kpops.components.base_components.pipeline_component import PipelineComponent +from kpops.components.streams_bootstrap.producer.producer_app import ProducerApp +from kpops.components.streams_bootstrap.streams.streams_app import StreamsApp class ScheduledProducer(ProducerApp): ... diff --git a/tests/pipeline/test_components_without_schema_handler/components.py b/tests/pipeline/test_components_without_schema_handler/components.py index 1d54a9f7a..9646e569b 100644 --- a/tests/pipeline/test_components_without_schema_handler/components.py +++ b/tests/pipeline/test_components_without_schema_handler/components.py @@ -1,13 +1,11 @@ from typing_extensions import override from kpops.component_handlers.kafka_connect.model import KafkaConnectorConfig -from kpops.components import ( - KafkaSinkConnector, - PipelineComponent, - ProducerApp, - StreamsApp, -) +from kpops.components.base_components.kafka_connector import KafkaSinkConnector from kpops.components.base_components.models.topic import OutputTopicTypes +from kpops.components.base_components.pipeline_component import PipelineComponent +from kpops.components.streams_bootstrap.producer.producer_app import ProducerApp +from kpops.components.streams_bootstrap.streams.streams_app import StreamsApp class ScheduledProducer(ProducerApp): ... diff --git a/tests/pipeline/test_example.py b/tests/pipeline/test_example.py index 1101b5795..7a03ae2ee 100644 --- a/tests/pipeline/test_example.py +++ b/tests/pipeline/test_example.py @@ -5,7 +5,7 @@ from pytest_snapshot.plugin import Snapshot from typer.testing import CliRunner -import kpops +import kpops.api as kpops runner = CliRunner() diff --git a/tests/pipeline/test_generate.py b/tests/pipeline/test_generate.py index a209071e3..f0daf13d7 100644 --- a/tests/pipeline/test_generate.py +++ b/tests/pipeline/test_generate.py @@ -9,11 +9,12 @@ from pytest_snapshot.plugin import Snapshot from typer.testing import CliRunner -import kpops +import kpops.api as kpops from kpops.api.exception import ParsingException, ValidationError from kpops.api.file_type import KpopsFileType from kpops.cli.main import FilterType, app -from kpops.components import KafkaSinkConnector, PipelineComponent +from kpops.components.base_components.kafka_connector import KafkaSinkConnector +from kpops.components.base_components.pipeline_component import PipelineComponent PIPELINE_YAML = KpopsFileType.PIPELINE.as_yaml_file() @@ -66,32 +67,12 @@ def test_python_api_filter_exclude(self, log_info: MagicMock): ) def test_load_pipeline(self, snapshot: Snapshot): - result = runner.invoke( - app, - [ - "generate", - str(RESOURCE_PATH / "first-pipeline" / PIPELINE_YAML), - ], - catch_exceptions=False, - ) - - assert result.exit_code == 0, result.stdout - - snapshot.assert_match(result.stdout, PIPELINE_YAML) + pipeline = kpops.generate(RESOURCE_PATH / "first-pipeline" / PIPELINE_YAML) + snapshot.assert_match(pipeline.to_yaml(), PIPELINE_YAML) def test_load_pipeline_with_folder_path(self, snapshot: Snapshot): - result = runner.invoke( - app, - [ - "generate", - str(RESOURCE_PATH / "pipeline-folders"), - ], - catch_exceptions=False, - ) - - assert result.exit_code == 0, result.stdout - - snapshot.assert_match(result.stdout, "pipeline.yaml") + pipeline = kpops.generate(RESOURCE_PATH / "pipeline-folders") + snapshot.assert_match(pipeline.to_yaml(), "pipeline.yaml") def test_load_pipeline_with_multiple_pipeline_paths(self, snapshot: Snapshot): path_1 = RESOURCE_PATH / "pipeline-folders/pipeline-1/pipeline.yaml" diff --git a/tests/pipeline/test_manifest.py b/tests/pipeline/test_manifest.py index 445e528cc..45f3e3a94 100644 --- a/tests/pipeline/test_manifest.py +++ b/tests/pipeline/test_manifest.py @@ -7,7 +7,7 @@ from pytest_snapshot.plugin import Snapshot from typer.testing import CliRunner -import kpops +import kpops.api as kpops from kpops.cli.main import app from kpops.component_handlers.helm_wrapper.helm import Helm from kpops.component_handlers.helm_wrapper.model import HelmConfig, Version diff --git a/tests/pipeline/test_pipeline.py b/tests/pipeline/test_pipeline.py index b5c741ad0..61d338dcf 100644 --- a/tests/pipeline/test_pipeline.py +++ b/tests/pipeline/test_pipeline.py @@ -7,9 +7,9 @@ from kpops.component_handlers import ( ComponentHandlers, ) -from kpops.components import PipelineComponent from kpops.components.base_components.models.from_section import FromSection from kpops.components.base_components.models.to_section import ToSection +from kpops.components.base_components.pipeline_component import PipelineComponent from kpops.pipeline import Pipeline PREFIX = "example-prefix-"