Skip to content

Commit

Permalink
Generate developer docs for Python API (#503)
Browse files Browse the repository at this point in the history
Co-authored-by: Salomon Popp <[email protected]>
  • Loading branch information
sujuka99 and disrupted authored Jul 4, 2024
1 parent 6a44b69 commit c7ceecb
Show file tree
Hide file tree
Showing 6 changed files with 203 additions and 10 deletions.
16 changes: 16 additions & 0 deletions docs/docs/developer/api.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# Python API

<!-- dprint-ignore-start -->

::: kpops.api
options:
filters:
- "!^_"

::: kpops.pipeline.Pipeline
options:
filters:
- "!^_"
- "!^model_config$"

<!-- dprint-ignore-end -->
17 changes: 17 additions & 0 deletions docs/mkdocs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,22 @@ plugins:
exclude:
- resources/*
- glightbox
- mkdocstrings:
handlers:
python:
paths: [..]
options:
docstring_style: sphinx
show_if_no_docstring: true
line_length: 60
separate_signature: true
show_signature_annotations: true
docstring_section_style: spacy
show_root_heading: true
# The options below are not yet publicly available,
# but would be a nice addition when released.
#show_inheritance_diagram: true # https://mkdocstrings.github.io/python/usage/configuration/general/?h=show_inheri#show_inheritance_diagram
#modernize_annotations: true # https://mkdocstrings.github.io/python/usage/configuration/signatures/#modernize_annotations

extra:
version:
Expand Down Expand Up @@ -120,6 +136,7 @@ nav:
- GitHub Actions: user/references/ci-integration/github-actions.md
- Developer Guide:
- Getting Started: developer/getting-started.md
- API: developer/api.md
- Contributing: developer/contributing.md
- Code base:
- Auto generation: developer/auto-generation.md
96 changes: 92 additions & 4 deletions kpops/api/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,13 +37,24 @@ def generate(
environment: str | None = None,
verbose: bool = False,
) -> Pipeline:
"""Generate enriched pipeline representation.
:param pipeline_path: Path to pipeline definition yaml file.
:param dotenv: Paths to dotenv files.
:param config: Path to the dir containing config.yaml files.
:param steps: Set of steps (components) to apply the command on.
:param filter_type: Whether `steps` should include/exclude the steps.
:param environment: The environment to generate and deploy the pipeline to.
:param verbose: Enable verbose printing.
:return: Generated `Pipeline` object.
"""
kpops_config = KpopsConfig.create(
config,
dotenv,
environment,
verbose,
)
pipeline = create_pipeline(pipeline_path, kpops_config, environment)
pipeline = _create_pipeline(pipeline_path, kpops_config, environment)
log.info(f"Picked up pipeline '{pipeline_path.parent.name}'")
if steps:
component_names = steps
Expand All @@ -68,6 +79,17 @@ def manifest(
environment: str | None = None,
verbose: bool = False,
) -> list[Resource]:
"""Generate pipeline, return final resource representations for each step.
:param pipeline_path: Path to pipeline definition yaml file.
:param dotenv: Paths to dotenv files.
:param config: Path to the dir containing config.yaml files.
:param steps: Set of steps (components) to apply the command on.
:param filter_type: Whether `steps` should include/exclude the steps.
:param environment: The environment to generate and deploy the pipeline to.
:param verbose: Enable verbose printing.
:return: Resources.
"""
pipeline = kpops.generate(
pipeline_path=pipeline_path,
dotenv=dotenv,
Expand Down Expand Up @@ -95,6 +117,18 @@ def deploy(
verbose: bool = True,
parallel: bool = False,
):
"""Deploy pipeline steps.
:param pipeline_path: Path to pipeline definition yaml file.
:param dotenv: Paths to dotenv files.
:param config: Path to the dir containing config.yaml files.
:param steps: Set of steps (components) to apply the command on.
:param filter_type: Whether `steps` should include/exclude the steps.
:param dry_run: Whether to dry run the command or execute it.
:param environment: The environment to generate and deploy the pipeline to.
:param verbose: Enable verbose printing.
:param parallel: Enable or disable parallel execution of pipeline steps.
"""
pipeline = kpops.generate(
pipeline_path=pipeline_path,
dotenv=dotenv,
Expand Down Expand Up @@ -131,6 +165,18 @@ def destroy(
verbose: bool = True,
parallel: bool = False,
):
"""Destroy pipeline steps.
:param pipeline_path: Path to pipeline definition yaml file.
:param dotenv: Paths to dotenv files.
:param config: Path to the dir containing config.yaml files.
:param steps: Set of steps (components) to apply the command on.
:param filter_type: Whether `steps` should include/exclude the steps.
:param dry_run: Whether to dry run the command or execute it.
:param environment: The environment to generate and deploy the pipeline to.
:param verbose: Enable verbose printing.
:param parallel: Enable or disable parallel execution of pipeline steps.
"""
pipeline = kpops.generate(
pipeline_path=pipeline_path,
dotenv=dotenv,
Expand Down Expand Up @@ -169,6 +215,18 @@ def reset(
verbose: bool = True,
parallel: bool = False,
):
"""Reset pipeline steps.
:param pipeline_path: Path to pipeline definition yaml file.
:param dotenv: Paths to dotenv files.
:param config: Path to the dir containing config.yaml files.
:param steps: Set of steps (components) to apply the command on.
:param filter_type: Whether `steps` should include/exclude the steps.
:param dry_run: Whether to dry run the command or execute it.
:param environment: The environment to generate and deploy the pipeline to.
:param verbose: Enable verbose printing.
:param parallel: Enable or disable parallel execution of pipeline steps.
"""
pipeline = kpops.generate(
pipeline_path=pipeline_path,
dotenv=dotenv,
Expand Down Expand Up @@ -206,6 +264,18 @@ def clean(
verbose: bool = True,
parallel: bool = False,
):
"""Clean pipeline steps.
:param pipeline_path: Path to pipeline definition yaml file.
:param dotenv: Paths to dotenv files.
:param config: Path to the dir containing config.yaml files.
:param steps: Set of steps (components) to apply the command on.
:param filter_type: Whether `steps` should include/exclude the steps.
:param dry_run: Whether to dry run the command or execute it.
:param environment: The environment to generate and deploy the pipeline to.
:param verbose: Enable verbose printing.
:param parallel: Enable or disable parallel execution of pipeline steps.
"""
pipeline = kpops.generate(
pipeline_path=pipeline_path,
dotenv=dotenv,
Expand Down Expand Up @@ -236,6 +306,12 @@ def init(
path: Path,
config_include_opt: bool = False,
):
"""Initiate a default empty project.
:param path: Directory in which the project should be initiated.
:param conf_incl_opt: Whether to include non-required settings
in the generated config file.
"""
if not path.exists():
path.mkdir(parents=False)
elif next(path.iterdir(), False):
Expand All @@ -244,22 +320,34 @@ def init(
init_project(path, config_include_opt)


def create_pipeline(
def _create_pipeline(
pipeline_path: Path,
kpops_config: KpopsConfig,
environment: str | None,
) -> Pipeline:
"""Create pipeline.
:param pipeline_path: Path to pipeline definition yaml file.
:param config: KPOps Config.
:param environment: The environment to generate and deploy the pipeline to.
:return: Created `Pipeline` object.
"""
registry = Registry()
if kpops_config.components_module:
registry.find_components(kpops_config.components_module)
registry.find_components("kpops.components")

handlers = setup_handlers(kpops_config)
handlers = _setup_handlers(kpops_config)
parser = PipelineGenerator(kpops_config, registry, handlers)
return parser.load_yaml(pipeline_path, environment)


def setup_handlers(config: KpopsConfig) -> ComponentHandlers:
def _setup_handlers(config: KpopsConfig) -> ComponentHandlers:
"""Set up handlers for a component.
:param config: KPOps config.
:return: Handlers for a component.
"""
schema_handler = SchemaHandler.load_schema_handler(config)
connector_handler = KafkaConnectHandler.from_kpops_config(config)
proxy_wrapper = ProxyWrapper(config.kafka_rest)
Expand Down
77 changes: 74 additions & 3 deletions poetry.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ mkdocs-material = "^9.0.0"
mkdocs-glightbox = "^0.3.1"
mkdocs-exclude-search = "^0.6.5"
mike = "^1.1.2"
mkdocstrings = { extras = ["python"], version = "^0.25.1" }

[tool.poetry_bumpversion.file."kpops/__init__.py"]

Expand Down
6 changes: 3 additions & 3 deletions tests/api/test_handlers.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from pytest_mock import MockerFixture

from kpops.api import setup_handlers
from kpops.api import _setup_handlers
from kpops.component_handlers import ComponentHandlers
from kpops.component_handlers.kafka_connect.kafka_connect_handler import (
KafkaConnectHandler,
Expand Down Expand Up @@ -35,7 +35,7 @@ def test_set_up_handlers_with_no_schema_handler(mocker: MockerFixture):
topic_handler=topic_handler,
)

actual_handlers = setup_handlers(config)
actual_handlers = _setup_handlers(config)

connector_handler_mock.from_kpops_config.assert_called_once_with(config)

Expand Down Expand Up @@ -72,7 +72,7 @@ def test_set_up_handlers_with_schema_handler(mocker: MockerFixture):
topic_handler=topic_handler,
)

actual_handlers = setup_handlers(config)
actual_handlers = _setup_handlers(config)

schema_handler_mock.load_schema_handler.assert_called_once_with(config)

Expand Down

0 comments on commit c7ceecb

Please sign in to comment.