diff --git a/.github/actions/update-docs/action.yml b/.github/actions/update-docs/action.yml index 48911af5f..bc7877497 100644 --- a/.github/actions/update-docs/action.yml +++ b/.github/actions/update-docs/action.yml @@ -1,8 +1,7 @@ name: "Update documentation in gh-pages" description: | - Compile markdown documents to html and deploy to docs branch. If a semver tag is given, this action strips the patch. + Compile markdown documents to html and deploy to docs branch. If a semver version is given, this action strips the patch. It then pushes the . as the latest alias to the documentation branch with mike. - In case no tag is given, it pushes to 'dev'. inputs: username: @@ -14,8 +13,12 @@ inputs: token: description: "GitHub Token (must be a PAT for repository dispatch)" required: true - tag: - description: "Version tag" + version: + description: "Version name to be deployed by mike" + required: true + release: + description: "Determines if the set version is a stable and latest version, otherwise it is a dev version. (Default false)" + default: 'false' required: false runs: @@ -27,23 +30,30 @@ runs: poetry-version: "1.5.1" python-version: "3.10" - - name: Update documentation branch with mike + - name: Install docs dependencies shell: bash - env: - USERNAME: ${{ inputs.username }} - EMAIL: ${{ inputs.email }} - TOKEN: ${{ inputs.token }} - NEW_TAG: ${{ inputs.tag }} run: | poetry install --with docs - git config --local user.name ${USERNAME} - git config --local user.email ${EMAIL} - git config --local user.password ${TOKEN} - git pull # make sure docs branch is up-to-date - if [ -z "$NEW_TAG" ]; then - poetry run mike deploy dev --push --rebase --config-file ./docs/mkdocs.yml - else - new_tag=${NEW_TAG%.*} - poetry run mike deploy "$new_tag" latest --update-aliases --push --rebase --config-file ./docs/mkdocs.yml - fi + - name: Update ${{ github.head_ref }} branch + shell: bash + run: | + git config --local user.name ${{ inputs.username }} + git config --local user.email ${{ inputs.email }} + git config --local user.password ${{ inputs.token }} + + git pull + + - name: Deploy ${{ inputs.version }} version of the documentation with mike + shell: bash + if: ${{ inputs.release == 'false' }} + run: | + poetry run mike deploy ${{ inputs.version }} --push --rebase --config-file ./docs/mkdocs.yml + + - name: Deploy ${{ inputs.version }} version (latest) of the documentation with mike + shell: bash + if: ${{ inputs.release == 'true' }} + run: | + sem_version=${{ inputs.version }} + major_minor_version=${sem_version%.*} + poetry run mike deploy "$major_minor_version" latest --update-aliases --push --rebase --config-file ./docs/mkdocs.yml diff --git a/.github/ruff-matcher.json b/.github/ruff-matcher.json new file mode 100644 index 000000000..bc3b10738 --- /dev/null +++ b/.github/ruff-matcher.json @@ -0,0 +1,17 @@ +{ + "problemMatcher": [ + { + "owner": "ruff", + "pattern": [ + { + "regexp": "^(.*):(\\d+):(\\d+):\\s([\\da-zA-Z]+)\\s(.*)$", + "file": 1, + "line": 2, + "column": 3, + "code": 4, + "message": 5 + } + ] + } + ] +} diff --git a/.github/scripts/__init__.py b/.github/scripts/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2b0cda131..fe87e8436 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -35,11 +35,16 @@ jobs: - name: Install dependencies run: poetry install --no-interaction - - name: Lint (flake8) - run: poetry run pre-commit run flake8 --all-files --show-diff-on-failure - - - name: Order of imports (isort) - run: poetry run pre-commit run isort --all-files --show-diff-on-failure + - name: Lint (ruff) + shell: bash + run: | + if [[ "$RUNNER_OS" == "Linux" && "${{ matrix.python-version }}" == "3.10" ]] + then + echo "::add-matcher::.github/ruff-matcher.json" + poetry run ruff check . --config pyproject.toml --output-format text --no-fix + else + poetry run pre-commit run ruff --all-files --show-diff-on-failure + fi; - name: Formatting (black) run: poetry run pre-commit run black --all-files --show-diff-on-failure @@ -59,16 +64,15 @@ jobs: - name: Generate pipeline definitions run: poetry run pre-commit run gen-docs-components --all-files --show-diff-on-failure - # TODO: enable when PEP 604 incompatibilty is in typer is resolved https://github.com/tiangolo/typer/issues/348 - # See https://github.com/tiangolo/typer/pull/522 - # - name: Syntax (pyupgrade) - # run: poetry run pre-commit run --hook-stage manual pyupgrade --all-files - - name: Test run: poetry run pytest tests - name: Install docs dependencies run: poetry install --with docs + + - name: Check markdown formatting + uses: dprint/check@v2.2 + if: runner.os == 'Linux' - name: Test docs build (mkdocs) run: poetry run mkdocs build -f docs/mkdocs.yml @@ -79,3 +83,46 @@ jobs: uses: bakdata/ci-templates/.github/workflows/python-poetry-publish-snapshot.yaml@v1.25.0 secrets: pypi-token: ${{ secrets.TEST_PYPI_TOKEN }} + + publish-docs-from-main: + runs-on: ubuntu-22.04 + if: ${{ github.ref == 'refs/heads/main' }} + needs: [test] + steps: + - uses: actions/checkout@v3 + + - name: Publish docs from main branch + uses: ./.github/actions/update-docs + with: + username: ${{ secrets.GH_USERNAME }} + email: ${{ secrets.GH_EMAIL }} + token: ${{ secrets.GH_TOKEN }} + version: main + + publish-dev-docs-from-pr: + runs-on: ubuntu-22.04 + if: ${{ github.event_name == 'pull_request' }} + needs: [test] + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.head_ref }} + + # Checks to see if any files in the PR match one of the listed file types. + # This will return true if there's a file in docs folder that was added, deleted, or modified in the PR. + - name: Check if files in docs folder have changed + uses: dorny/paths-filter@v2 + id: docs-changes + with: + filters: | + docs: + - added|deleted|modified: 'docs/**' + + - name: Publish dev docs from PR + if: steps.docs-changes.outputs.docs == 'true' + uses: ./.github/actions/update-docs + with: + username: ${{ secrets.GH_USERNAME }} + email: ${{ secrets.GH_EMAIL }} + token: ${{ secrets.GH_TOKEN }} + version: dev diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 1341075b0..0fe1f0573 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -6,6 +6,7 @@ on: - main paths: - .github/** + - actions/** jobs: actionlint: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index b34323896..9171e6c70 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -39,7 +39,8 @@ jobs: - name: Update gh-pages uses: ./.github/actions/update-docs with: - tag: ${{ needs.create-github-release-push-tag.outputs.release-version }} username: ${{ secrets.GH_USERNAME }} email: ${{ secrets.GH_EMAIL }} token: ${{ secrets.GH_TOKEN }} + version: ${{ needs.create-github-release-push-tag.outputs.release-version }} + release: true diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f92a0c936..8c709b20a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,12 +1,14 @@ repos: - - repo: https://github.com/pycqa/isort - rev: 5.12.0 - hooks: - - id: isort - args: ["--settings", "setup.cfg"] - exclude: ^tests/.*snapshots/ - repo: local hooks: + - id: ruff + name: ruff + entry: ruff check . + args: [ --config, pyproject.toml, --fix, --show-fixes, --exit-non-zero-on-fix ] + language: system + types_or: [python] + require_serial: true # run once for all files + pass_filenames: false - id: black name: black entry: black @@ -14,14 +16,6 @@ repos: types_or: [python, pyi] require_serial: true # run once for all files exclude: ^tests/.*snapshots/ - - repo: https://github.com/pycqa/flake8 - rev: 5.0.4 - hooks: - - id: flake8 - args: ["--config", "setup.cfg"] - exclude: ^tests/.*snapshots/ - - repo: local - hooks: - id: pyright name: pyright entry: pyright @@ -29,15 +23,6 @@ repos: types: [python] require_serial: true # run once for all files exclude: ^tests/.*snapshots/ - - repo: https://github.com/asottile/pyupgrade - rev: v3.1.0 - hooks: - - id: pyupgrade - stages: [manual] - args: ["--py310-plus"] - exclude: ^tests/.*snapshots/ - - repo: local - hooks: - id: gen-schema name: gen-schema entry: python hooks/gen_schema.py @@ -81,3 +66,10 @@ repos: | kpops/components/.*\.py )$ require_serial: true + - id: dprint + name: dprint + entry: dprint fmt + language: system + types: [markdown] + require_serial: true + pass_filenames: false diff --git a/CHANGELOG.md b/CHANGELOG.md index 876f40080..c16a9033b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,117 @@ # Changelog +## [2.0.9](https://github.com/bakdata/kpops/releases/tag/2.0.9) - Release Date: [2023-09-19] + +### 🐛 Fixes + +- Fix Kafka connect config name for deletion - [#361](https://github.com/bakdata/kpops/pull/361) + + +### 📝 Documentation + +- Fix link to kpops-examples - [#357](https://github.com/bakdata/kpops/pull/357) + + + + + + +## [2.0.8](https://github.com/bakdata/kpops/releases/tag/2.0.8) - Release Date: [2023-09-06] + +### 🐛 Fixes + +- Fix config.yaml overriding environment variables - [#353](https://github.com/bakdata/kpops/pull/353) + + +### 🏭 Refactor + +- Refactor component prefix & name - [#326](https://github.com/bakdata/kpops/pull/326) + +- Remove unnecessary condition during inflate - [#328](https://github.com/bakdata/kpops/pull/328) + + + + + + +## [2.0.7](https://github.com/bakdata/kpops/releases/tag/2.0.7) - Release Date: [2023-08-31] + +### 🐛 Fixes + +- Print only rendered templates when `--template` flag is set - [#350](https://github.com/bakdata/kpops/pull/350) + + +### 📝 Documentation + +- Add migration guide - [#352](https://github.com/bakdata/kpops/pull/352) + + + + + + +## [2.0.6](https://github.com/bakdata/kpops/releases/tag/2.0.6) - Release Date: [2023-08-30] + +### 🏭 Refactor + +- Simplify deployment with local Helm charts - [#349](https://github.com/bakdata/kpops/pull/349) + + + + + + +## [2.0.5](https://github.com/bakdata/kpops/releases/tag/2.0.5) - Release Date: [2023-08-30] + +### 🐛 Fixes + +- Fix versioning of docs when releasing - [#346](https://github.com/bakdata/kpops/pull/346) + + + + + + +## [2.0.4](https://github.com/bakdata/kpops/releases/tag/2.0.4) - Release Date: [2023-08-29] + +### 🐛 Fixes + +- Fix GitHub ref variable for pushing docs to main branch - [#343](https://github.com/bakdata/kpops/pull/343) + + +### 📝 Documentation + +- Add `dprint` as the markdown formatter - [#337](https://github.com/bakdata/kpops/pull/337) + +- Publish pre-release docs for PRs & main branch - [#339](https://github.com/bakdata/kpops/pull/339) + +- Align docs colours - [#345](https://github.com/bakdata/kpops/pull/345) + + +### 🌀 Miscellaneous + +- Exclude abstract components from pipeline schema - [#332](https://github.com/bakdata/kpops/pull/332) + + + + + + +## [2.0.3](https://github.com/bakdata/kpops/releases/tag/2.0.3) - Release Date: [2023-08-24] + +### 🐛 Fixes + +- Fix GitHub action error in non-Python projects - [#340](https://github.com/bakdata/kpops/pull/340) + + +### 🌀 Miscellaneous + +- Lint GitHub action - [#342](https://github.com/bakdata/kpops/pull/342) + + + + + + ## [2.0.2](https://github.com/bakdata/kpops/releases/tag/2.0.2) - Release Date: [2023-08-23] ### 📝 Documentation diff --git a/README.md b/README.md index 17707f0b3..9dd25fd9c 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ [![Build status](https://github.com/bakdata/kpops/actions/workflows/ci.yaml/badge.svg)](https://github.com/bakdata/kpops/actions/workflows/ci.yaml) [![pypi](https://img.shields.io/pypi/v/kpops.svg)](https://pypi.org/project/kpops) [![versions](https://img.shields.io/pypi/pyversions/kpops.svg)](https://github.com/bakdata/kpops) +[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) [![license](https://img.shields.io/github/license/bakdata/kpops.svg)](https://github.com/bakdata/kpops/blob/main/LICENSE) ## Key features @@ -21,7 +22,7 @@ the [documentation](https://bakdata.github.io/kpops/latest). ## Install KPOps -KPOps comes as a [PyPI package](https://pypi.org/project/kpops/). +KPOps comes as a [PyPI package](https://pypi.org/project/kpops/). You can install it with [pip](https://github.com/pypa/pip): ```shell diff --git a/actions/kpops-runner/action.yaml b/actions/kpops-runner/action.yaml index 851726968..ff2533251 100644 --- a/actions/kpops-runner/action.yaml +++ b/actions/kpops-runner/action.yaml @@ -45,23 +45,35 @@ inputs: runs: using: "composite" steps: - - name: Set up Python ${{ inputs.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ inputs.python-version }} - cache: 'pip' - - name: Set up Helm uses: azure/setup-helm@v3 with: version: ${{ inputs.helm-version }} token: ${{ inputs.token }} + - name: Create temporary requirements.txt for caching + shell: bash + id: requirements + run: | + TEMP_FILE="${{ github.action_path }}/kpops-runner-action-requirements.txt" + echo "kpops${{ inputs.kpops-version != 'latest' && format('=={0}', inputs.kpops-version) || '' }}" > "$TEMP_FILE" + cat "$TEMP_FILE" + echo "path=$TEMP_FILE">> $GITHUB_OUTPUT + + - name: Set up Python ${{ inputs.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python-version }} + cache: pip + cache-dependency-path: '' + # FIXME: https://github.com/actions/setup-python/issues/361 + # "${{ steps.requirements.outputs.path }}" + - name: Install KPOps shell: bash run: | - echo "::group::pip install kpops package" - pip install kpops${{ inputs.kpops-version != 'latest' && format('=={0}', inputs.kpops-version) || '' }} + echo "::group::install kpops package" + pip install -r "${{ steps.requirements.outputs.path }}" echo "::endgroup::" - name: ${{ inputs.command }} ${{ inputs.pipeline }} pipeline diff --git a/docs/docs/dev/development/autogeneration.md b/docs/docs/dev/development/autogeneration.md deleted file mode 100644 index b19e95c42..000000000 --- a/docs/docs/dev/development/autogeneration.md +++ /dev/null @@ -1,47 +0,0 @@ -# Autogeneration - -Autogeneration happens mostly in the `pre-commit` hooks in `/kpops/hooks`. - -## Generation scripts and their respective files - -### Documentation (`gen_docs/*`) - -#### Variables (`./kpops/docs/docs/resources/variables/*`) - -- `cli_env_vars.env` -- All CLI environment variables in a `dotenv` file. -- `cli_env_vars.md` -- All CLI environment variables in a table. -- `config_env_vars.env` -- Almost all pipeline config environment variables in a `dotenv` file. The script checks for each field in `PipelineConfig` from `./kpops/kpops/cli/pipeline_config.py` whether it has an `env` attribute defined. The script is currently unable to visit the classes of fields like `topic_name_config`, hence any environment variables defined there would remain unknown to it. -- `config_env_vars.env` -- Almost all pipeline config environment variables in a table. -- `variable_substitution.yaml` -- A copy of `./tests/pipeline/resources/component-type-substitution/pipeline.yaml` used as an example of substition. - -#### CLI commands (`./kpops/docs/docs/user/references/cli-commands.md`) - -Generated by `typer-cli` from the code in `./kpops/kpops/cli/main.py`. It is called with Python's `subprocess` module. - -#### Pipeline and defaults example definitions (`./kpops/docs/docs/resources/pipeline-components/` and `./kpops/docs/docs/resources/pipeline-defaults/`) - -Generates example `pipeline.yaml` and `defaults.yaml` for each individual component, stores them and also concatenates them into 1 big pipeline definition and 1 big pipeline defaults definition. - - -User input - -- `headers/*\.yaml` -- The top of each example. Includes a description comment, `type` and `name`. The headers for `pipeline.yaml` reside in the `pipeline-components` dir and the `defaults.yaml` headers reside in the `pipeline-defaults` dir. The names of the files must be equal to the respective component `type`. -- `sections/*\.yaml` -- Each YAML file contains a single section (component attribute) definition. The intention is to keep the minimal set of definitions there from which any component definition can be built. The names of the files must be equal to the respective component `type` and the attribute name. The sections are used for both `defaults.yaml` and `pipeline.yaml` generation and reside in the `pipeline-components` dir. - - -Generated - -- `pipeline-components/dependencies/*` -Cached information about KPOps components - - `pipeline_component_dependencies.yaml` -- Specifies per component which files in the `sections` dir should be used for the `pipeline.yaml` generation. - - `defaults_pipeline_component_dependencies.yaml` -- Specifies per component which files in the `sections` dir should be used for the `defaults.yaml` generation. - - `kpops_structure.yaml` -- Specifies the inheritance hierarchy of the components and what sections exist in each component. -- `pipeline-components/*\.yaml` -- All single-component pipeline definitions and one big (complete) `pipeline.yaml` that contains all of them. -- `pipeline-defaults/*\.yaml` -- All single-component defaults definitions and one big (complete) `defaults.yaml` that contains all of them. - -### Editor integration (`gen_schema.py`) - -#### Schemas (`./kpops/docs/docs/schema/*`) - -- `config.json` -- `pipeline.json` diff --git a/docs/docs/developer/auto-generation.md b/docs/docs/developer/auto-generation.md new file mode 100644 index 000000000..249f52b77 --- /dev/null +++ b/docs/docs/developer/auto-generation.md @@ -0,0 +1,45 @@ +# Auto generation + +Auto generation happens mostly with [`pre-commit`](https://pre-commit.com/) hooks. You can find the pre-commit configuration [here](https://github.com/bakdata/kpops/blob/main/.pre-commit-config.yaml). These pre-commit hooks call different [Python scripts](https://github.com/bakdata/kpops/tree/main/hooks) to auto generate code for the documentation. + +## Generation scripts and their respective files + +### [Documentation](https://github.com/bakdata/kpops/tree/main/hooks/gen_docs) + +#### [Variables](https://github.com/bakdata/kpops/tree/main/docs/docs/resources/variables) + +- `cli_env_vars.env` -- All CLI environment variables in a `dotenv` file. +- `cli_env_vars.md` -- All CLI environment variables in a table. +- `config_env_vars.env` -- Almost all pipeline config environment variables in a `dotenv` file. The script checks for each field in [`PipelineConfig`](https://github.com/bakdata/kpops/blob/main/kpops/cli/pipeline_config.py) whether it has an `env` attribute defined. The script is currently unable to visit the classes of fields like `topic_name_config`, hence any environment variables defined there would remain unknown to it. +- `config_env_vars.env` -- Almost all pipeline config environment variables in a table. +- `variable_substitution.yaml` -- A copy of `./tests/pipeline/resources/component-type-substitution/pipeline.yaml` used as an example of substitution. + +#### [CLI commands](../user/references/cli-commands.md) + +Generated by `typer-cli` from the code in [`main.py`](https://github.com/bakdata/kpops/blob/main/kpops/cli/main.py). It is called with Python's `subprocess` module. + +#### [Pipeline](https://github.com/bakdata/kpops/tree/main/docs/docs/resources/pipeline-components) and [defaults](https://github.com/bakdata/kpops/tree/main/docs/docs/resources/pipeline-defaults) example definitions + +Generates example `pipeline.yaml` and `defaults.yaml` for each individual component, stores them and also concatenates them into 1 big pipeline definition and 1 big pipeline defaults definition. + +User input + +- `headers/*\.yaml` -- The top of each example. Includes a description comment, `type` and `name`. The headers for `pipeline.yaml` reside in the `pipeline-components` dir and the `defaults.yaml` headers reside in the `pipeline-defaults` dir. The names of the files must be equal to the respective component `type`. +- `sections/*\.yaml` -- Each YAML file contains a single section (component attribute) definition. The intention is to keep the minimal set of definitions there from which any component definition can be built. The names of the files must be equal to the respective component `type` and the attribute name. The sections are used for both `defaults.yaml` and `pipeline.yaml` generation and reside in the `pipeline-components` dir. + +Generated + +- `pipeline-components/dependencies/*` + Cached information about KPOps components + - `pipeline_component_dependencies.yaml` -- Specifies per component which files in the `sections` dir should be used for the `pipeline.yaml` generation. + - `defaults_pipeline_component_dependencies.yaml` -- Specifies per component which files in the `sections` dir should be used for the `defaults.yaml` generation. + - `kpops_structure.yaml` -- Specifies the inheritance hierarchy of the components and what sections exist in each component. +- `pipeline-components/*\.yaml` -- All single-component pipeline definitions and one big (complete) `pipeline.yaml` that contains all of them. +- `pipeline-defaults/*\.yaml` -- All single-component defaults definitions and one big (complete) `defaults.yaml` that contains all of them. + +### [Editor integration](https://github.com/bakdata/kpops/blob/main/hooks/gen_schema.py) + +#### [Schemas](https://github.com/bakdata/kpops/tree/main/docs/docs/schema) + +- [config.json](https://github.com/bakdata/kpops/blob/main/docs/docs/schema/config.json) +- [pipeline.json](https://github.com/bakdata/kpops/blob/main/docs/docs/schema/pipeline.json) diff --git a/docs/docs/developer/formatting.md b/docs/docs/developer/formatting.md new file mode 100644 index 000000000..54ea1b6d9 --- /dev/null +++ b/docs/docs/developer/formatting.md @@ -0,0 +1,12 @@ +# Formatting + +## Markdown + +To ensure a consistent markdown style, we use [dprint](https://dprint.dev) to check and reformat. + +```shell +dprint fmt +``` + +Use the [official documentation](https://dprint.dev/setup/) to set up dprint. +The configuration can be found [here](https://github.com/bakdata/kpops/blob/main/dprint.json). diff --git a/docs/overrides/home.html b/docs/docs/overrides/home.html similarity index 85% rename from docs/overrides/home.html rename to docs/docs/overrides/home.html index 4cf65d0f1..a69b1d4ca 100644 --- a/docs/overrides/home.html +++ b/docs/docs/overrides/home.html @@ -13,19 +13,10 @@ diff --git a/docs/docs/resources/architecture/components-hierarchy.md b/docs/docs/resources/architecture/components-hierarchy.md index 4ce7b68f2..fec927ddf 100644 --- a/docs/docs/resources/architecture/components-hierarchy.md +++ b/docs/docs/resources/architecture/components-hierarchy.md @@ -1,6 +1,5 @@ ```mermaid flowchart BT - PipelineComponent --> BaseDefaultsComponent KubernetesApp --> PipelineComponent KafkaConnector --> PipelineComponent KafkaApp --> KubernetesApp @@ -17,4 +16,5 @@ flowchart BT click KafkaSourceConnector "../kafka-source-connector" click KafkaSinkConnector "../kafka-sink-connector" ``` +

KPOps component hierarchy

diff --git a/docs/docs/resources/examples/defaults.md b/docs/docs/resources/examples/defaults.md index 2b4525493..1c26ec43d 100644 --- a/docs/docs/resources/examples/defaults.md +++ b/docs/docs/resources/examples/defaults.md @@ -2,18 +2,26 @@ ## [ATM Fraud Pipeline](https://github.com/bakdata/kpops/tree/main/examples/bakdata/atm-fraud-detection){target=_blank} + + ??? example "defaults.yaml" ```yaml - --8<-- - https://raw.githubusercontent.com/bakdata/kpops/main/examples/bakdata/atm-fraud-detection/defaults.yaml - --8<-- + --8<-- + https://raw.githubusercontent.com/bakdata/kpops/main/examples/bakdata/atm-fraud-detection/defaults.yaml + --8<-- ``` -## [Word-count Pipeline](https://github.com/bakdata/kpops-examples/tree/main/word-count/deployment/kpops/defaults){target=_blank} + + +## [Word-count Pipeline](https://github.com/bakdata/kpops-examples/tree/main/word-count/deployment/kpops){target=_blank} + + ??? example "defaults.yaml" ```yaml - --8<-- - https://raw.githubusercontent.com/bakdata/kpops-examples/main/word-count/deployment/kpops/defaults/defaults.yaml - --8<-- + --8<-- + https://raw.githubusercontent.com/bakdata/kpops-examples/main/word-count/deployment/kpops/defaults.yaml + --8<-- ``` + + diff --git a/docs/docs/resources/examples/pipeline.md b/docs/docs/resources/examples/pipeline.md index 3bbec8039..4ef2632c8 100644 --- a/docs/docs/resources/examples/pipeline.md +++ b/docs/docs/resources/examples/pipeline.md @@ -2,18 +2,26 @@ ## [ATM Fraud Pipeline](https://github.com/bakdata/kpops/tree/main/examples/bakdata/atm-fraud-detection){target=_blank} + + ??? example "pipeline.yaml" ```yaml - --8<-- - https://raw.githubusercontent.com/bakdata/kpops/main/examples/bakdata/atm-fraud-detection/pipeline.yaml - --8<-- + --8<-- + https://raw.githubusercontent.com/bakdata/kpops/main/examples/bakdata/atm-fraud-detection/pipeline.yaml + --8<-- ``` + + ## [Word-count Pipeline](https://github.com/bakdata/kpops-examples/tree/main/word-count/deployment/kpops){target=_blank} + + ??? example "pipeline.yaml" ```yaml - --8<-- - https://raw.githubusercontent.com/bakdata/kpops-examples/main/word-count/deployment/kpops/pipeline.yaml - --8<-- + --8<-- + https://raw.githubusercontent.com/bakdata/kpops-examples/main/word-count/deployment/kpops/pipeline.yaml + --8<-- ``` + + diff --git a/docs/docs/resources/pipeline-components/pipeline.md b/docs/docs/resources/pipeline-components/pipeline.md index b4b518ebb..e8d0b752b 100644 --- a/docs/docs/resources/pipeline-components/pipeline.md +++ b/docs/docs/resources/pipeline-components/pipeline.md @@ -3,7 +3,7 @@ [:material-download:](./pipeline.yaml) ```yaml - --8<-- - ./docs/resources/pipeline-components/pipeline.yaml - --8<-- +--8<-- +./docs/resources/pipeline-components/pipeline.yaml +--8<-- ``` diff --git a/docs/docs/resources/pipeline-defaults/defaults.md b/docs/docs/resources/pipeline-defaults/defaults.md index 6f71a3cd6..914bca3e4 100644 --- a/docs/docs/resources/pipeline-defaults/defaults.md +++ b/docs/docs/resources/pipeline-defaults/defaults.md @@ -3,7 +3,7 @@ [:material-download:](./defaults.yaml) ```yaml - --8<-- - ./docs/resources/pipeline-defaults/defaults.yaml - --8<-- +--8<-- +./docs/resources/pipeline-defaults/defaults.yaml +--8<-- ``` diff --git a/docs/docs/resources/setup/kafka.yaml b/docs/docs/resources/setup/kafka.yaml new file mode 100644 index 000000000..753b29152 --- /dev/null +++ b/docs/docs/resources/setup/kafka.yaml @@ -0,0 +1,109 @@ +cp-zookeeper: + enabled: true + servers: 1 + imageTag: 7.1.3 + heapOptions: "-Xms124M -Xmx124M" + overrideGroupId: k8kafka + fullnameOverride: "k8kafka-cp-zookeeper" + resources: + requests: + cpu: 50m + memory: 0.2G + limits: + cpu: 250m + memory: 0.2G + prometheus: + jmx: + enabled: false + +cp-kafka: + enabled: true + brokers: 1 + imageTag: 7.1.3 + podManagementPolicy: Parallel + configurationOverrides: + "auto.create.topics.enable": false + "offsets.topic.replication.factor": 1 + "transaction.state.log.replication.factor": 1 + "transaction.state.log.min.isr": 1 + "confluent.metrics.reporter.topic.replicas": 1 + resources: + requests: + cpu: 50m + memory: 0.5G + limits: + cpu: 250m + memory: 0.5G + prometheus: + jmx: + enabled: false + persistence: + enabled: false + +cp-schema-registry: + enabled: true + imageTag: 7.1.3 + fullnameOverride: "k8kafka-cp-schema-registry" + overrideGroupId: k8kafka + kafka: + bootstrapServers: "PLAINTEXT://k8kafka-cp-kafka-headless:9092" + resources: + requests: + cpu: 50m + memory: 0.25G + limits: + cpu: 250m + memory: 0.25G + prometheus: + jmx: + enabled: false + +cp-kafka-connect: + enabled: true + replicaCount: 1 + image: k3d-kpops-registry.localhost:12345/kafka-connect-jdbc + imageTag: 7.1.3 + fullnameOverride: "k8kafka-cp-kafka-connect" + overrideGroupId: k8kafka + kafka: + bootstrapServers: "PLAINTEXT://k8kafka-cp-kafka-headless:9092" + heapOptions: "-Xms256M -Xmx256M" + resources: + requests: + cpu: 500m + memory: 0.25G + limits: + cpu: 500m + memory: 0.25G + configurationOverrides: + "consumer.max.poll.records": "10" + "consumer.max.poll.interval.ms": "900000" + "config.storage.replication.factor": "1" + "offset.storage.replication.factor": "1" + "status.storage.replication.factor": "1" + cp-schema-registry: + url: http://k8kafka-cp-schema-registry:8081 + prometheus: + jmx: + enabled: false + +cp-kafka-rest: + enabled: true + imageTag: 7.1.3 + fullnameOverride: "k8kafka-cp-rest" + heapOptions: "-Xms256M -Xmx256M" + resources: + requests: + cpu: 50m + memory: 0.25G + limits: + cpu: 250m + memory: 0.5G + prometheus: + jmx: + enabled: false + +cp-ksql-server: + enabled: false +cp-control-center: + enabled: false diff --git a/docs/docs/resources/variables/cli_env_vars.md b/docs/docs/resources/variables/cli_env_vars.md index 66615a185..763cb936e 100644 --- a/docs/docs/resources/variables/cli_env_vars.md +++ b/docs/docs/resources/variables/cli_env_vars.md @@ -1,4 +1,3 @@ - These variables are a lower priority alternative to the commands' flags. If a variable is set, the corresponding flag does not have to be specified in commands. Variables marked as required can instead be set as flags. | Name |Default Value|Required| Description | diff --git a/docs/docs/resources/variables/config_env_vars.md b/docs/docs/resources/variables/config_env_vars.md index 9f3b89926..5cfe21105 100644 --- a/docs/docs/resources/variables/config_env_vars.md +++ b/docs/docs/resources/variables/config_env_vars.md @@ -1,4 +1,3 @@ - These variables are a lower priority alternative to the settings in `config.yaml`. Variables marked as required can instead be set in the pipeline config. | Name | Default Value |Required| Description | Setting name | diff --git a/docs/docs/schema/config.json b/docs/docs/schema/config.json index b2e37e0ec..c1de43d6d 100644 --- a/docs/docs/schema/config.json +++ b/docs/docs/schema/config.json @@ -1,7 +1,7 @@ { "$defs": { "HelmConfig": { - "description": "Global Helm configuration", + "description": "Global Helm configuration.", "properties": { "api_version": { "anyOf": [ diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json index 317a915b9..f7ec539c5 100644 --- a/docs/docs/schema/pipeline.json +++ b/docs/docs/schema/pipeline.json @@ -2,7 +2,7 @@ "$defs": { "FromSection": { "additionalProperties": false, - "description": "Holds multiple input topics", + "description": "Holds multiple input topics.", "properties": { "components": { "additionalProperties": { @@ -28,7 +28,7 @@ }, "FromTopic": { "additionalProperties": false, - "description": "Input topic", + "description": "Input topic.", "properties": { "role": { "anyOf": [ @@ -60,7 +60,7 @@ "type": "object" }, "HelmRepoConfig": { - "description": "Helm repository configuration", + "description": "Helm repository configuration.", "properties": { "repo_auth_flags": { "allOf": [ @@ -96,7 +96,7 @@ "type": "object" }, "InputTopicTypes": { - "description": "Input topic types\n\nINPUT (input topic), PATTERN (extra-topic-pattern or input-topic-pattern)", + "description": "Input topic types.\n\nINPUT (input topic), PATTERN (extra-topic-pattern or input-topic-pattern)", "enum": [ "input", "pattern" @@ -104,237 +104,12 @@ "title": "InputTopicTypes", "type": "string" }, - "KafkaApp": { - "additionalProperties": true, - "description": "Base component for Kafka-based components.\nProducer or streaming apps should inherit from this class.", - "properties": { - "app": { - "allOf": [ - { - "$ref": "#/definitions/KafkaAppConfig" - } - ], - "description": "Application-specific settings" - }, - "from": { - "anyOf": [ - { - "$ref": "#/definitions/FromSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) and/or components from which the component will read input", - "title": "From" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline_name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/definitions/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" - }, - "to": { - "anyOf": [ - { - "$ref": "#/definitions/ToSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) into which the component will write output" - }, - "version": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": "2.9.0", - "description": "Helm chart version", - "title": "Version" - } - }, - "required": [ - "name", - "namespace", - "app" - ], - "title": "KafkaApp", - "type": "object" - }, - "KafkaAppConfig": { - "additionalProperties": true, - "description": "Settings specific to Kafka Apps", - "properties": { - "nameOverride": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Override name with this value", - "title": "Nameoverride" - }, - "streams": { - "allOf": [ - { - "$ref": "#/definitions/KafkaStreamsConfig" - } - ], - "description": "Kafka streams config" - } - }, - "required": [ - "streams" - ], - "title": "KafkaAppConfig", - "type": "object" - }, - "KafkaConnector": { - "additionalProperties": true, - "description": "Base class for all Kafka connectors\nShould only be used to set defaults", - "properties": { - "app": { - "allOf": [ - { - "$ref": "#/definitions/KafkaConnectorConfig" - } - ], - "description": "Application-specific settings" - }, - "from": { - "anyOf": [ - { - "$ref": "#/definitions/FromSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) and/or components from which the component will read input", - "title": "From" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline_name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/definitions/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-kafka-connect-resetter", - "url": "https://bakdata.github.io/kafka-connect-resetter/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" - }, - "resetter_values": { - "description": "Overriding Kafka Connect Resetter Helm values. E.g. to override the Image Tag etc.", - "title": "Resetter Values", - "type": "object" - }, - "to": { - "anyOf": [ - { - "$ref": "#/definitions/ToSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) into which the component will write output" - }, - "version": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": "1.0.4", - "description": "Helm chart version", - "title": "Version" - } - }, - "required": [ - "name", - "namespace", - "app" - ], - "title": "KafkaConnector", - "type": "object" - }, "KafkaConnectorConfig": { "additionalProperties": true, "additional_properties": { "type": "string" }, - "description": "Settings specific to Kafka Connectors", + "description": "Settings specific to Kafka Connectors.", "properties": { "connector.class": { "title": "Connector.Class", @@ -348,8 +123,7 @@ "type": "object" }, "KafkaSinkConnector": { - "additionalProperties": true, - "description": "Kafka sink connector model", + "description": "Kafka sink connector model.", "properties": { "app": { "allOf": [ @@ -416,23 +190,21 @@ "anyOf": [ { "$ref": "#/definitions/ToSection" - }, - { - "type": "null" } ], - "default": null, - "description": "Topic(s) into which the component will write output" + "description": "Topic(s) into which the component will write output", + "title": "To" }, - "version": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } + "type": { + "default": "kafka-sink-connector", + "description": "Kafka sink connector model.", + "enum": [ + "kafka-sink-connector" ], + "title": "Component type", + "type": "string" + }, + "version": { "default": "1.0.4", "description": "Helm chart version", "title": "Version" @@ -447,8 +219,7 @@ "type": "object" }, "KafkaSourceConnector": { - "additionalProperties": true, - "description": "Kafka source connector model", + "description": "Kafka source connector model.", "properties": { "app": { "allOf": [ @@ -528,23 +299,21 @@ "anyOf": [ { "$ref": "#/definitions/ToSection" - }, - { - "type": "null" } ], - "default": null, - "description": "Topic(s) into which the component will write output" + "description": "Topic(s) into which the component will write output", + "title": "To" }, - "version": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } + "type": { + "default": "kafka-source-connector", + "description": "Kafka source connector model.", + "enum": [ + "kafka-source-connector" ], + "title": "Component type", + "type": "string" + }, + "version": { "default": "1.0.4", "description": "Helm chart version", "title": "Version" @@ -558,35 +327,6 @@ "title": "KafkaSourceConnector", "type": "object" }, - "KafkaStreamsConfig": { - "additionalProperties": true, - "description": "Kafka Streams config", - "properties": { - "brokers": { - "description": "Brokers", - "title": "Brokers", - "type": "string" - }, - "schemaRegistryUrl": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "URL of the schema registry", - "title": "Schemaregistryurl" - } - }, - "required": [ - "brokers" - ], - "title": "KafkaStreamsConfig", - "type": "object" - }, "KubernetesApp": { "additionalProperties": true, "description": "Base class for all Kubernetes apps.\nAll built-in components are Kubernetes apps, except for the Kafka connectors.", @@ -665,21 +405,19 @@ "required": [ "name", "namespace", - "app", - "repo_config" + "app" ], "title": "KubernetesApp", "type": "object" }, "KubernetesAppConfig": { - "additionalProperties": true, - "description": "Settings specific to Kubernetes Apps", + "description": "Settings specific to Kubernetes Apps.", "properties": {}, "title": "KubernetesAppConfig", "type": "object" }, "OutputTopicTypes": { - "description": "Types of output topic\n\nOUTPUT (output topic), ERROR (error topic)", + "description": "Types of output topic.\n\nOUTPUT (output topic), ERROR (error topic)", "enum": [ "output", "error" @@ -687,56 +425,8 @@ "title": "OutputTopicTypes", "type": "string" }, - "PipelineComponent": { - "additionalProperties": true, - "description": "Base class for all components", - "properties": { - "from": { - "anyOf": [ - { - "$ref": "#/definitions/FromSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) and/or components from which the component will read input", - "title": "From" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "prefix": { - "default": "${pipeline_name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "to": { - "anyOf": [ - { - "$ref": "#/definitions/ToSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) into which the component will write output" - } - }, - "required": [ - "name" - ], - "title": "PipelineComponent", - "type": "object" - }, "ProducerApp": { - "additionalProperties": true, - "description": "Producer component\nThis producer holds configuration to use as values for the streams bootstrap producer helm chart. Note that the producer does not support error topics.", + "description": "Producer component.\nThis producer holds configuration to use as values for the streams bootstrap producer helm chart. Note that the producer does not support error topics.", "properties": { "app": { "allOf": [ @@ -791,23 +481,21 @@ "anyOf": [ { "$ref": "#/definitions/ToSection" - }, - { - "type": "null" } ], - "default": null, - "description": "Topic(s) into which the component will write output" + "description": "Topic(s) into which the component will write output", + "title": "To" }, - "version": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } + "type": { + "default": "producer-app", + "description": "Producer component.\nThis producer holds configuration to use as values for the streams bootstrap producer helm chart. Note that the producer does not support error topics.", + "enum": [ + "producer-app" ], + "title": "Component type", + "type": "string" + }, + "version": { "default": "2.9.0", "description": "Helm chart version", "title": "Version" @@ -822,8 +510,7 @@ "type": "object" }, "ProducerStreamsConfig": { - "additionalProperties": true, - "description": "Kafka Streams settings specific to Producer", + "description": "Kafka Streams settings specific to Producer.", "properties": { "brokers": { "description": "Brokers", @@ -873,8 +560,7 @@ "type": "object" }, "ProducerValues": { - "additionalProperties": true, - "description": "Settings specific to producers", + "description": "Settings specific to producers.", "properties": { "nameOverride": { "anyOf": [ @@ -905,7 +591,7 @@ "type": "object" }, "RepoAuthFlags": { - "description": "Authorisation-related flags for `helm repo`", + "description": "Authorisation-related flags for `helm repo`.", "properties": { "ca_file": { "anyOf": [ @@ -972,8 +658,7 @@ "type": "object" }, "StreamsApp": { - "additionalProperties": true, - "description": "StreamsApp component that configures a streams bootstrap app", + "description": "StreamsApp component that configures a streams bootstrap app.", "properties": { "app": { "allOf": [ @@ -1035,23 +720,21 @@ "anyOf": [ { "$ref": "#/definitions/ToSection" - }, - { - "type": "null" } ], - "default": null, - "description": "Topic(s) into which the component will write output" + "description": "Topic(s) into which the component will write output", + "title": "To" }, - "version": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } + "type": { + "default": "streams-app", + "description": "StreamsApp component that configures a streams bootstrap app.", + "enum": [ + "streams-app" ], + "title": "Component type", + "type": "string" + }, + "version": { "default": "2.9.0", "description": "Helm chart version", "title": "Version" @@ -1066,8 +749,7 @@ "type": "object" }, "StreamsAppAutoScaling": { - "additionalProperties": true, - "description": "Kubernetes Event-driven Autoscaling config", + "description": "Kubernetes Event-driven Autoscaling config.", "properties": { "consumerGroup": { "description": "Name of the consumer group used for checking the offset on the topic and processing the related lag.", @@ -1185,8 +867,7 @@ "type": "object" }, "StreamsConfig": { - "additionalProperties": true, - "description": "Streams Bootstrap streams section", + "description": "Streams Bootstrap streams section.", "properties": { "brokers": { "description": "Brokers", @@ -1301,8 +982,7 @@ "type": "object" }, "ToSection": { - "additionalProperties": false, - "description": "Holds multiple output topics", + "description": "Holds multiple output topics.", "properties": { "models": { "additionalProperties": { @@ -1328,7 +1008,7 @@ }, "TopicConfig": { "additionalProperties": false, - "description": "Configure an output topic", + "description": "Configure an output topic.", "properties": { "configs": { "additionalProperties": { @@ -1432,24 +1112,15 @@ "items": { "discriminator": { "mapping": { - "kafka-app": "#/definitions/KafkaApp", - "kafka-connector": "#/definitions/KafkaConnector", "kafka-sink-connector": "#/definitions/KafkaSinkConnector", "kafka-source-connector": "#/definitions/KafkaSourceConnector", "kubernetes-app": "#/definitions/KubernetesApp", - "pipeline-component": "#/definitions/PipelineComponent", "producer-app": "#/definitions/ProducerApp", "streams-app": "#/definitions/StreamsApp" }, "propertyName": "type" }, "oneOf": [ - { - "$ref": "#/definitions/KafkaApp" - }, - { - "$ref": "#/definitions/KafkaConnector" - }, { "$ref": "#/definitions/KafkaSinkConnector" }, @@ -1459,9 +1130,6 @@ { "$ref": "#/definitions/KubernetesApp" }, - { - "$ref": "#/definitions/PipelineComponent" - }, { "$ref": "#/definitions/ProducerApp" }, diff --git a/docs/docs/stylesheets/extra.css b/docs/docs/stylesheets/extra.css new file mode 100644 index 000000000..ae2953c82 --- /dev/null +++ b/docs/docs/stylesheets/extra.css @@ -0,0 +1,24 @@ +:root > * { + /* The primary color of KPOps docs */ + --md-primary-fg-color: #a599ff; + + /* The second color of KPOps docs */ + --md-accent-fg-color: #B0A5FF; + + /* Create custom color variable to use for the buttons */ + --kpops-button-color: #7068AD; + + /* The footer color needs to be set sparately */ + --md-footer-bg-color: var(--md-primary-fg-color); + + /* Color the links with the primary color */ + --md-typeset-a-color: var(--md-primary-fg-color); +} + +.md-typeset .md-button, .md-button--primary { + background-color: var(--kpops-button-color); +} + +.md-typeset .md-button:hover, .md-button--primary:hover { + border-color: white; +} diff --git a/docs/docs/user/core-concepts/components/kafka-app.md b/docs/docs/user/core-concepts/components/kafka-app.md index 392da332e..e69153c5e 100644 --- a/docs/docs/user/core-concepts/components/kafka-app.md +++ b/docs/docs/user/core-concepts/components/kafka-app.md @@ -10,6 +10,8 @@ Subclass of [_KubernetesApp_](kubernetes-app.md). ### Configuration + + ??? example "`pipeline.yaml`" ```yaml @@ -18,6 +20,8 @@ Subclass of [_KubernetesApp_](kubernetes-app.md). --8<-- ``` + + ### Operations #### deploy diff --git a/docs/docs/user/core-concepts/components/kafka-sink-connector.md b/docs/docs/user/core-concepts/components/kafka-sink-connector.md index 8596555af..ab3ceece7 100644 --- a/docs/docs/user/core-concepts/components/kafka-sink-connector.md +++ b/docs/docs/user/core-concepts/components/kafka-sink-connector.md @@ -8,6 +8,8 @@ Lets other systems pull data from Apache Kafka. ### Configuration + + ??? example "`pipeline.yaml`" ```yaml @@ -16,6 +18,8 @@ Lets other systems pull data from Apache Kafka. --8<-- ``` + + ### Operations #### deploy diff --git a/docs/docs/user/core-concepts/components/kafka-source-connector.md b/docs/docs/user/core-concepts/components/kafka-source-connector.md index 409dade9f..d15a32bc1 100644 --- a/docs/docs/user/core-concepts/components/kafka-source-connector.md +++ b/docs/docs/user/core-concepts/components/kafka-source-connector.md @@ -8,6 +8,8 @@ Manages source connectors in your Kafka Connect cluster. ### Configuration + + ??? example "`pipeline.yaml`" ```yaml @@ -16,6 +18,8 @@ Manages source connectors in your Kafka Connect cluster. --8<-- ``` + + ### Operations #### deploy diff --git a/docs/docs/user/core-concepts/components/kubernetes-app.md b/docs/docs/user/core-concepts/components/kubernetes-app.md index aa6e7e280..4a28dbe0e 100644 --- a/docs/docs/user/core-concepts/components/kubernetes-app.md +++ b/docs/docs/user/core-concepts/components/kubernetes-app.md @@ -6,6 +6,8 @@ Can be used to deploy any app in Kubernetes using Helm, for example, a REST serv ### Configuration + + ??? example "`pipeline.yaml`" ```yaml @@ -14,6 +16,8 @@ Can be used to deploy any app in Kubernetes using Helm, for example, a REST serv --8<-- ``` + + ### Operations #### deploy diff --git a/docs/docs/user/core-concepts/components/producer-app.md b/docs/docs/user/core-concepts/components/producer-app.md index e5f707411..1f55fa6d9 100644 --- a/docs/docs/user/core-concepts/components/producer-app.md +++ b/docs/docs/user/core-concepts/components/producer-app.md @@ -8,6 +8,8 @@ Configures a [streams-bootstrap](https://github.com/bakdata/streams-bootstrap){t ### Configuration + + ??? example "`pipeline.yaml`" ```yaml @@ -16,6 +18,8 @@ Configures a [streams-bootstrap](https://github.com/bakdata/streams-bootstrap){t --8<-- ``` + + ### Operations #### deploy diff --git a/docs/docs/user/core-concepts/components/streams-app.md b/docs/docs/user/core-concepts/components/streams-app.md index 362e2c1b1..ac881ade2 100644 --- a/docs/docs/user/core-concepts/components/streams-app.md +++ b/docs/docs/user/core-concepts/components/streams-app.md @@ -10,6 +10,8 @@ Configures a ### Configuration + + ??? example "`pipeline.yaml`" ```yaml @@ -18,6 +20,8 @@ Configures a --8<-- ``` + + ### Operations #### deploy diff --git a/docs/docs/user/core-concepts/config.md b/docs/docs/user/core-concepts/config.md index 163c13cd8..dfc308fb4 100644 --- a/docs/docs/user/core-concepts/config.md +++ b/docs/docs/user/core-concepts/config.md @@ -6,6 +6,8 @@ Consider enabling [KPOps' editor integration](../references/editor-integration.m To learn about any of the available settings, take a look at the example below. + + ??? example "`config.yaml`" ```yaml @@ -15,4 +17,6 @@ To learn about any of the available settings, take a look at the example below. ``` !!! note "Environment-specific pipeline definitions" - Similarly to [defaults](defaults.md#configuration), it is possible to have an unlimited amount of additional environment-specific pipeline definitions. The naming convention is the same: add a suffix of the form `_{environment}` to the filename. \ No newline at end of file + Similarly to [defaults](defaults.md#configuration), it is possible to have an unlimited amount of additional environment-specific pipeline definitions. The naming convention is the same: add a suffix of the form `_{environment}` to the filename. + + diff --git a/docs/docs/user/core-concepts/defaults.md b/docs/docs/user/core-concepts/defaults.md index 2b129d2cf..c07c2ce34 100644 --- a/docs/docs/user/core-concepts/defaults.md +++ b/docs/docs/user/core-concepts/defaults.md @@ -18,10 +18,14 @@ It is possible to set specific `defaults` for each `environment` by adding files It is important to note that `defaults_{environment}.yaml` overrides only the settings that are explicitly set to be different from the ones in the base `defaults` file. + + !!! tip `defaults` is the default value of `defaults_filename_prefix`. Together with `defaults_path` and `environment` it can be changed in [`config.yaml`](../config/#__codelineno-0-16) + + ## Components @@ -30,70 +34,98 @@ The `defaults` codeblocks in this section contain the full set of settings that ### [KubernetesApp](./components/kubernetes-app.md) + + ??? example "`defaults.yaml`" ```yaml - --8<-- - ./docs/resources/pipeline-defaults/defaults-kubernetes-app.yaml - --8<-- + --8<-- + ./docs/resources/pipeline-defaults/defaults-kubernetes-app.yaml + --8<-- ``` + + ### [KafkaApp](./components/kafka-app.md) + + ??? example "`defaults.yaml`" ```yaml - --8<-- - ./docs/resources/pipeline-defaults/defaults-kafka-app.yaml - --8<-- + --8<-- + ./docs/resources/pipeline-defaults/defaults-kafka-app.yaml + --8<-- ``` + + ### [StreamsApp](./components/streams-app.md) + + ??? example "`defaults.yaml`" ```yaml - --8<-- - ./docs/resources/pipeline-defaults/defaults-streams-app.yaml - --8<-- + --8<-- + ./docs/resources/pipeline-defaults/defaults-streams-app.yaml + --8<-- ``` + + ### [ProducerApp](./components/producer-app.md) + + ??? example "`defaults.yaml`" ```yaml - --8<-- - ./docs/resources/pipeline-defaults/defaults-producer.yaml - --8<-- + --8<-- + ./docs/resources/pipeline-defaults/defaults-producer.yaml + --8<-- ``` + + ### [KafkaConnector](./components/kafka-connector.md) + + ??? example "`defaults.yaml`" ```yaml - --8<-- - ./docs/resources/pipeline-defaults/defaults-kafka-connector.yaml - --8<-- + --8<-- + ./docs/resources/pipeline-defaults/defaults-kafka-connector.yaml + --8<-- ``` + + ### [KafkaSourceConnector](./components/kafka-source-connector.md) + + ??? example "`defaults.yaml`" ```yaml - --8<-- - ./docs/resources/pipeline-defaults/defaults-kafka-source-connector.yaml - --8<-- + --8<-- + ./docs/resources/pipeline-defaults/defaults-kafka-source-connector.yaml + --8<-- ``` + + ### [KafkaSinkConnector](./components/kafka-sink-connector.md) + + ??? example "`defaults.yaml`" ```yaml - --8<-- - ./docs/resources/pipeline-defaults/defaults-kafka-sink-connector.yaml - --8<-- + --8<-- + ./docs/resources/pipeline-defaults/defaults-kafka-sink-connector.yaml + --8<-- ``` + + diff --git a/docs/docs/user/core-concepts/variables/environment_variables.md b/docs/docs/user/core-concepts/variables/environment_variables.md index 38664f7d9..2a57aabea 100644 --- a/docs/docs/user/core-concepts/variables/environment_variables.md +++ b/docs/docs/user/core-concepts/variables/environment_variables.md @@ -2,6 +2,8 @@ Environment variables can be set by using the [export](https://www.unix.com/man-page/linux/1/export/){target=_blank} command in Linux or the [set](https://learn.microsoft.com/en-us/windows-server/administration/windows-commands/set_1){target=_blank} command in Windows. + + !!! tip "dotenv files" Support for `.env` files is on the [roadmap](https://github.com/bakdata/kpops/issues/20), @@ -9,12 +11,16 @@ Environment variables can be set by using the [export](https://www.unix.com/man- use one and export the contents manually is with the following command: `#!sh export $(xargs < .env)`. This would work in `bash` suppose there are no spaces inside the values. + + ## Config --8<-- ./docs/resources/variables/config_env_vars.md --8<-- + + ??? example "config_env_vars.env" ```py title="Exhaustive list of all config-related environment variables" @@ -23,12 +29,16 @@ Environment variables can be set by using the [export](https://www.unix.com/man- --8<-- ``` + + ## CLI --8<-- ./docs/resources/variables/cli_env_vars.md --8<-- + + ??? example "cli_env_vars.env" ```py title="Exhaustive list of all cli-related environment variables" @@ -36,3 +46,5 @@ Environment variables can be set by using the [export](https://www.unix.com/man- ./docs/resources/variables/cli_env_vars.env --8<-- ``` + + diff --git a/docs/docs/user/core-concepts/variables/substitution.md b/docs/docs/user/core-concepts/variables/substitution.md index 2fac8aa83..4ef9b1b25 100644 --- a/docs/docs/user/core-concepts/variables/substitution.md +++ b/docs/docs/user/core-concepts/variables/substitution.md @@ -8,6 +8,8 @@ These variables can be used in a component's definition to refer to any of its a All of them are prefixed with `component_` and follow the following form: `component_{attribute_name}`. If the attribute itself contains attributes, they can be referred to like this: `component_{attribute_name}_{subattribute_name}`. + + ??? Example ```yaml --8<-- @@ -15,14 +17,20 @@ All of them are prefixed with `component_` and follow the following form: `compo --8<-- ``` + + ## Pipeline-config-specific variables These variables include all fields in the [config](../config.md) and refer to the pipeline configuration that is independent of the components. + + !!! info Aliases `error_topic_name` is an alias for `topic_name_config_default_error_topic_name` `output_topic_name` is an alias for `topic_name_config_default_output_topic_name` + + ## Environment variables Environment variables such as `$PATH` can be used in the pipeline definition and defaults without any transformation following the form `${ENV_VAR_NAME}`. This, of course, includes variables like the ones relevant to the [KPOps cli](../../references/cli-commands.md) that are exported by the user. @@ -33,19 +41,14 @@ Environment variables such as `$PATH` can be used in the pipeline definition and These are special variables that refer to the name and path of a pipeline. -- `${pipeline_name}` - Concatenated path of the parent directory where pipeline.yaml is defined in. - For instance, `./data/pipelines/v1/pipeline.yaml`, here the value for the variable would be `data-pipelines-v1`. +- `${pipeline_name}`: Concatenated path of the parent directory where pipeline.yaml is defined in. + For instance, `./data/pipelines/v1/pipeline.yaml`, here the value for the variable would be `data-pipelines-v1`. -- `${pipeline_name_}` - Similar to the previous variable, each `` contains a part of the path to the `pipeline.yaml` file. - Consider the previous example, `${pipeline_name_0}` would be `data`, `${pipeline_name_1}` would be `pipelines`, and `${pipeline_name_2}` equals to `v1`. +- `${pipeline_name_}`: Similar to the previous variable, each `` contains a part of the path to the `pipeline.yaml` file. + Consider the previous example, `${pipeline_name_0}` would be `data`, `${pipeline_name_1}` would be `pipelines`, and `${pipeline_name_2}` equals to `v1`. ## Advanced use cases -1. **Refer to default component field values** -As long as a value is assigned to a component attribute, it is possible to refer to it with a placeholder. To see all component fields, take a look at the [pipeline schema](../../../schema/pipeline.json). -2. **Chaining variables** -It is possible to chain any number of variables, see the [example](#component-specific-variables) above. -3. **Cross-component substitution** -[YAML](https://yaml.org/){target=_blank} is quite an intricate language and with some of its [magic](https://yaml.org/spec/1.2.2/#692-node-anchors){target=_blank} one could write cross-component references. +1. **Refer to default component field values**: As long as a value is assigned to a component attribute, it is possible to refer to it with a placeholder. To see all component fields, take a look at the [pipeline schema](../../../schema/pipeline.json). +2. **Chaining variables**: It is possible to chain any number of variables, see the [example](#component-specific-variables) above. +3. **Cross-component substitution**: [YAML](https://yaml.org/){target=_blank} is quite an intricate language and with some of its [magic](https://yaml.org/spec/1.2.2/#692-node-anchors){target=_blank} one could write cross-component references. diff --git a/docs/docs/user/examples/atm-fraud-pipeline.md b/docs/docs/user/examples/atm-fraud-pipeline.md index ddd72a317..06c249f3d 100644 --- a/docs/docs/user/examples/atm-fraud-pipeline.md +++ b/docs/docs/user/examples/atm-fraud-pipeline.md @@ -1,9 +1,9 @@ # ATM fraud detection pipeline -ATM fraud is a demo pipeline for ATM fraud detection. -The original by Confluent is written in KSQL -and outlined in this [blogpost](https://www.confluent.io/blog/atm-fraud-detection-apache-kafka-ksql/){target=_blank}. -The one used in this example is re-built from scratch using [bakdata](https://bakdata.com/){target=_blank}'s +ATM fraud is a demo pipeline for ATM fraud detection. +The original by Confluent is written in KSQL +and outlined in this [blogpost](https://www.confluent.io/blog/atm-fraud-detection-apache-kafka-ksql/){target=_blank}. +The one used in this example is re-built from scratch using [bakdata](https://bakdata.com/){target=_blank}'s [`streams-bootstrap`](https://github.com/bakdata/streams-bootstrap){target=_blank} library. ## What this will demonstrate @@ -22,18 +22,22 @@ Completed all steps in the [setup](../getting-started/setup.md). Deploy PostgreSQL using the [Bitnami Helm chart:](https://artifacthub.io/packages/helm/bitnami/postgresql){target=_blank} Add the helm repository: + ```shell helm repo add bitnami https://charts.bitnami.com/bitnami && \ helm repo update ``` Install the PostgreSQL with helm: + ```shell helm upgrade --install -f ./postgresql.yaml \ --namespace kpops \ postgresql bitnami/postgresql ``` + + ??? example "PostgreSQL Example Helm chart values (`postgresql.yaml`)" ```yaml auth: @@ -50,6 +54,8 @@ postgresql bitnami/postgresql enabled: true ``` + + ### ATM fraud detection example pipeline setup #### Port forwarding @@ -68,33 +74,38 @@ kubectl port-forward --namespace kpops service/k8kafka-cp-kafka-connect 8083:808 1. Export environment variables in your terminal: - ```shell - export DOCKER_REGISTRY=bakdata && \ - export NAMESPACE=kpops - ``` + ```shell + export DOCKER_REGISTRY=bakdata && \ + export NAMESPACE=kpops + ``` 2. Deploy the pipeline - ```shell - poetry run kpops deploy ./examples/bakdata/atm-fraud-detection/pipeline.yaml \ - --pipeline-base-dir ./examples \ - --config ./examples/bakdata/atm-fraud-detection/config.yaml \ - --execute - ``` - + ```shell + poetry run kpops deploy ./examples/bakdata/atm-fraud-detection/pipeline.yaml \ + --pipeline-base-dir ./examples \ + --config ./examples/bakdata/atm-fraud-detection/config.yaml \ + --execute + ``` + + + !!! Note You can use the `--dry-run` flag instead of the `--execute` flag and check the logs if your pipeline will be deployed correctly. + + ### Check if the deployment is successful -You can use the [Streams Explorer](https://github.com/bakdata/streams-explorer){target=_blank} to see the deployed pipeline. +You can use the [Streams Explorer](https://github.com/bakdata/streams-explorer){target=_blank} to see the deployed pipeline. To do so, port-forward the service in a separate terminal session using the command below: ```shell kubectl port-forward -n kpops service/streams-explorer 8080:8080 ``` -After that open [http://localhost:8080](http://localhost:8080){target=_blank} in your browser. + +After that open [http://localhost:8080](http://localhost:8080){target=_blank} in your browser. You should be able to see pipeline shown in the image below:
@@ -102,6 +113,8 @@ You should be able to see pipeline shown in the image below:
An overview of ATM fraud pipeline shown in Streams Explorer
+ + !!! Attention Kafka Connect needs some time to set up the connector. Moreover, Streams Explorer needs a while to scrape the information from Kafka connect. @@ -135,6 +148,8 @@ helm --namespace kpops uninstall postgresql --verbose \ --execute ``` + + !!! Note You can use the `--dry-run` flag instead of the `--execute` flag and check the logs if your pipeline will be destroyed correctly. @@ -142,16 +157,18 @@ helm --namespace kpops uninstall postgresql !!! Attention If you face any issues destroying this example see [Teardown](../getting-started/teardown.md) for manual deletion. + + ## Common errors - `deploy` fails: - 1. Read the error message. - 2. Try to correct the mistakes if there were any. Likely the configuration is not correct or the port-forwarding is not working as intended. - 3. Run `clean`. - 4. Run `deploy --dry-run` to avoid havig to `clean` again. If an error is dropped, start over from step 1. - 5. If the dry-run is succesful, run `deploy`. + 1. Read the error message. + 2. Try to correct the mistakes if there were any. Likely the configuration is not correct or the port-forwarding is not working as intended. + 3. Run `clean`. + 4. Run `deploy --dry-run` to avoid havig to `clean` again. If an error is dropped, start over from step 1. + 5. If the dry-run is succesful, run `deploy`. - `clean` fails: - 1. Read the error message. - 2. Try to correct the indicated mistakes if there were any. Likely the configuration is not correct or the port-forwarding is not working as intended. - 3. Run `clean`. - 4. If `clean` fails, follow the steps in [teardown](../getting-started/teardown.md). + 1. Read the error message. + 2. Try to correct the indicated mistakes if there were any. Likely the configuration is not correct or the port-forwarding is not working as intended. + 3. Run `clean`. + 4. If `clean` fails, follow the steps in [teardown](../getting-started/teardown.md). diff --git a/docs/docs/user/getting-started/quick-start.md b/docs/docs/user/getting-started/quick-start.md index 8b32bbbc8..3727f8ca8 100644 --- a/docs/docs/user/getting-started/quick-start.md +++ b/docs/docs/user/getting-started/quick-start.md @@ -20,18 +20,22 @@ Completed all steps in the [setup](../setup). Deploy Redis using the [Bitnami Helm chart:](https://artifacthub.io/packages/helm/bitnami/redis){target=_blank} Add the Helm repository: + ```shell helm repo add bitnami https://charts.bitnami.com/bitnami && \ helm repo update ``` Install Redis with Helm: + ```shell helm upgrade --install -f ./values-redis.yaml \ --namespace kpops \ redis bitnami/redis ``` + + ??? example "Redis example Helm chart values (`values-redis.yaml`)" ```yaml architecture: standalone @@ -44,6 +48,8 @@ redis bitnami/redis tag: 7.0.8 ``` + + ### Word-count example pipeline setup #### Port forwarding @@ -63,38 +69,42 @@ kubectl port-forward --namespace kpops service/k8kafka-cp-kafka-connect 8083:808 1. Copy the [configuration](https://github.com/bakdata/kpops-examples/tree/main/word-count/deployment/kpops){target=_blank} from the [kpops-examples repository](https://github.com/bakdata/kpops-examples/tree/main/word-count){target=_blank} into `kpops>examples>bakdata>word-count` like so: - ``` - kpops - ├── examples - | ├── bakdata - | | ├── word-count - | | | ├── config.yaml - | | | ├── defaults - | | | │   └── defaults.yaml - | | | └── pipeline.yaml - | | | - ``` + ``` + kpops + ├── examples + | ├── bakdata + | | ├── word-count + | | | ├── config.yaml + | | | ├── defaults + | | | │   └── defaults.yaml + | | | └── pipeline.yaml + | | | + ``` 2. Export environment variables in your terminal: - ```shell - export DOCKER_REGISTRY=bakdata && \ - export NAMESPACE=kpops - ``` + ```shell + export DOCKER_REGISTRY=bakdata && \ + export NAMESPACE=kpops + ``` 3. Deploy the pipeline - ```shell - kpops deploy ./examples/bakdata/word-count/pipeline.yaml \ - --pipeline-base-dir ./examples \ - --config ./examples/bakdata/word-count/config.yaml \ - --execute - ``` + ```shell + kpops deploy ./examples/bakdata/word-count/pipeline.yaml \ + --pipeline-base-dir ./examples \ + --config ./examples/bakdata/word-count/config.yaml \ + --execute + ``` + + !!! Note You can use the `--dry-run` flag instead of the `--execute` flag and check the logs if your pipeline will be deployed correctly. + + ### Check if the deployment is successful You can use the [Streams Explorer](https://github.com/bakdata/streams-explorer){target=_blank} to inspect the deployed pipeline. @@ -113,11 +123,15 @@ You should be able to see pipeline shown in the image below:
An overview of Word-count pipeline shown in Streams Explorer
+ + !!! Attention Kafka Connect needs some time to set up the connector. Moreover, Streams Explorer needs a while to scrape the information from Kafka Connect. Therefore, it might take a bit until you see the whole graph. + + ## Teardown resources ### Redis @@ -132,20 +146,23 @@ helm --namespace kpops uninstall redis 1. Export environment variables in your terminal. - ```shell - export DOCKER_REGISTRY=bakdata && \ - export NAMESPACE=kpops - ``` + ```shell + export DOCKER_REGISTRY=bakdata && \ + export NAMESPACE=kpops + ``` 2. Remove the pipeline - ```shell - kpops clean ./examples/bakdata/word-count/pipeline.yaml \ - --pipeline-base-dir ./examples \ - --config ./examples/bakdata/word-count/config.yaml \ - --verbose \ - --execute - ``` + ```shell + kpops clean ./examples/bakdata/word-count/pipeline.yaml \ + --pipeline-base-dir ./examples \ + --config ./examples/bakdata/word-count/config.yaml \ + --verbose \ + --execute + ``` + + + !!! Note You can use the `--dry-run` flag instead of the `--execute` flag and check the logs if your pipeline will be destroyed correctly. @@ -153,16 +170,18 @@ helm --namespace kpops uninstall redis !!! Attention If you face any issues destroying this example see [Teardown](../teardown) for manual deletion. + + ## Common errors - `deploy` fails: - 1. Read the error message. - 2. Try to correct the mistakes if there were any. Likely the configuration is not correct or the port-forwarding is not working as intended. - 3. Run `clean`. - 4. Run `deploy --dry-run` to avoid having to `clean` again. If an error is dropped, start over from step 1. - 5. If the dry-run is successful, run `deploy`. + 1. Read the error message. + 2. Try to correct the mistakes if there were any. Likely the configuration is not correct or the port-forwarding is not working as intended. + 3. Run `clean`. + 4. Run `deploy --dry-run` to avoid having to `clean` again. If an error is dropped, start over from step 1. + 5. If the dry-run is successful, run `deploy`. - `clean` fails: - 1. Read the error message. - 2. Try to correct the indicated mistakes if there were any. Likely the configuration is not correct or the port-forwarding is not working as intended. - 3. Run `clean`. - 4. If `clean` fails, follow the steps in [teardown](../teardown). + 1. Read the error message. + 2. Try to correct the indicated mistakes if there were any. Likely the configuration is not correct or the port-forwarding is not working as intended. + 3. Run `clean`. + 4. If `clean` fails, follow the steps in [teardown](../teardown). diff --git a/docs/docs/user/getting-started/setup.md b/docs/docs/user/getting-started/setup.md index f30ca4688..1c608b77e 100644 --- a/docs/docs/user/getting-started/setup.md +++ b/docs/docs/user/getting-started/setup.md @@ -18,26 +18,30 @@ If you don't have access to an existing Kubernetes cluster, this section will gu 1. You can install k3d with its installation script: - ```shell - wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/v5.4.6/install.sh | bash - ``` + ```shell + wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/v5.4.6/install.sh | bash + ``` - For other ways of installing k3d, you can have a look at their [installation guide](https://k3d.io/v5.4.6/#installation){target=_blank}. + For other ways of installing k3d, you can have a look at their [installation guide](https://k3d.io/v5.4.6/#installation){target=_blank}. 2. The [Kafka deployment](#deploy-kafka) needs a modified Docker image. In that case the image is built and pushed to a Docker registry that holds it. If you do not have access to an existing Docker registry, you can use k3d's Docker registry: - ```shell - k3d registry create kpops-registry.localhost --port 12345 - ``` + ```shell + k3d registry create kpops-registry.localhost --port 12345 + ``` 3. Now you can create a new cluster called `kpops` that uses the previously created Docker registry: - ```shell - k3d cluster create kpops --k3s-arg "--no-deploy=traefik@server:*" --registry-use k3d-kpops-registry.localhost:12345 - ``` + ```shell + k3d cluster create kpops --k3s-arg "--no-deploy=traefik@server:*" --registry-use k3d-kpops-registry.localhost:12345 + ``` + + - !!! Note - Creating a new k3d cluster automatically configures `kubectl` to connect to the local cluster by modifying your `~/.kube/config`. In case you manually set the `KUBECONFIG` variable or don't want k3d to modify your config, k3d offers [many other options](https://k3d.io/v5.4.6/usage/kubeconfig/#handling-kubeconfigs){target=_blank}. +!!! Note + Creating a new k3d cluster automatically configures `kubectl` to connect to the local cluster by modifying your `~/.kube/config`. In case you manually set the `KUBECONFIG` variable or don't want k3d to modify your config, k3d offers [many other options](https://k3d.io/v5.4.6/usage/kubeconfig/#handling-kubeconfigs){target=_blank}. + + You can check the cluster status with `kubectl get pods -n kube-system`. If all returned elements have a `STATUS` of `Running` or `Completed`, then the cluster is up and running. @@ -47,157 +51,55 @@ You can check the cluster status with `kubectl get pods -n kube-system`. If all 1. To allow connectivity to other systems [Kafka Connect](https://docs.confluent.io/platform/current/connect/index.html#kafka-connect){target=_blank} needs to be extended with drivers. You can install a [JDBC driver](https://docs.confluent.io/kafka-connectors/jdbc/current/jdbc-drivers.html){target=_blank} for Kafka Connect by creating a new Docker image: - 1. Create a `Dockerfile` with the following content: + 1. Create a `Dockerfile` with the following content: - ```dockerfile - FROM confluentinc/cp-kafka-connect:7.1.3 + ```dockerfile + FROM confluentinc/cp-kafka-connect:7.1.3 - RUN confluent-hub install --no-prompt confluentinc/kafka-connect-jdbc:10.6.0 - ``` + RUN confluent-hub install --no-prompt confluentinc/kafka-connect-jdbc:10.6.0 + ``` - 2. Build and push the modified image to your private Docker registry: + 2. Build and push the modified image to your private Docker registry: - ```shell - docker build . --tag localhost:12345/kafka-connect-jdbc:7.1.3 && \ - docker push localhost:12345/kafka-connect-jdbc:7.1.3 - ``` + ```shell + docker build . --tag localhost:12345/kafka-connect-jdbc:7.1.3 && \ + docker push localhost:12345/kafka-connect-jdbc:7.1.3 + ``` - Detailed instructions on building, tagging and pushing a docker image can be found in [Docker docs](https://docs.docker.com/){target=_blank}. + Detailed instructions on building, tagging and pushing a docker image can be found in [Docker docs](https://docs.docker.com/){target=_blank}. 2. Add Confluent's Helm chart repository and update the index: - ```shell - helm repo add confluentinc https://confluentinc.github.io/cp-helm-charts/ && - helm repo update - ``` + ```shell + helm repo add confluentinc https://confluentinc.github.io/cp-helm-charts/ && + helm repo update + ``` 3. Install Kafka, Zookeeper, Confluent's Schema Registry, Kafka Rest Proxy, and Kafka Connect. A single Helm chart installs all five components. Below you can find an example for the `--values ./kafka.yaml` file configuring the deployment accordingly. Deploy the services: - ```shell - helm upgrade \ - --install \ - --version 0.6.1 \ - --values ./kafka.yaml \ - --namespace kpops \ - --create-namespace \ - --wait \ - k8kafka confluentinc/cp-helm-charts + ```shell + helm upgrade \ + --install \ + --version 0.6.1 \ + --values ./kafka.yaml \ + --namespace kpops \ + --create-namespace \ + --wait \ + k8kafka confluentinc/cp-helm-charts + ``` + + + +??? example "Kafka Helm chart values (`kafka.yaml`)" + An example value configuration for Confluent's Helm chart. This configuration deploys a single Kafka Broker, a Schema Registry, Zookeeper, Kafka Rest Proxy, and Kafka Connect with minimal resources. + + ```yaml + --8<-- + ./docs/resources/setup/kafka.yaml + --8<-- ``` - ??? example "Kafka Helm chart values (`kafka.yaml`)" - An example value configuration for Confluent's Helm chart. This configuration deploys a single Kafka Broker, a Schema Registry, Zookeeper, Kafka Rest Proxy, and Kafka Connect with minimal resources. - - ```yaml - cp-zookeeper: - enabled: true - servers: 1 - imageTag: 7.1.3 - heapOptions: "-Xms124M -Xmx124M" - overrideGroupId: k8kafka - fullnameOverride: "k8kafka-cp-zookeeper" - resources: - requests: - cpu: 50m - memory: 0.2G - limits: - cpu: 250m - memory: 0.2G - prometheus: - jmx: - enabled: false - - cp-kafka: - enabled: true - brokers: 1 - imageTag: 7.1.3 - podManagementPolicy: Parallel - configurationOverrides: - "auto.create.topics.enable": false - "offsets.topic.replication.factor": 1 - "transaction.state.log.replication.factor": 1 - "transaction.state.log.min.isr": 1 - "confluent.metrics.reporter.topic.replicas": 1 - resources: - requests: - cpu: 50m - memory: 0.5G - limits: - cpu: 250m - memory: 0.5G - prometheus: - jmx: - enabled: false - persistence: - enabled: false - - cp-schema-registry: - enabled: true - imageTag: 7.1.3 - fullnameOverride: "k8kafka-cp-schema-registry" - overrideGroupId: k8kafka - kafka: - bootstrapServers: "PLAINTEXT://k8kafka-cp-kafka-headless:9092" - resources: - requests: - cpu: 50m - memory: 0.25G - limits: - cpu: 250m - memory: 0.25G - prometheus: - jmx: - enabled: false - - cp-kafka-connect: - enabled: true - replicaCount: 1 - image: k3d-kpops-registry.localhost:12345/kafka-connect-jdbc - imageTag: 7.1.3 - fullnameOverride: "k8kafka-cp-kafka-connect" - overrideGroupId: k8kafka - kafka: - bootstrapServers: "PLAINTEXT://k8kafka-cp-kafka-headless:9092" - heapOptions: "-Xms256M -Xmx256M" - resources: - requests: - cpu: 500m - memory: 0.25G - limits: - cpu: 500m - memory: 0.25G - configurationOverrides: - "consumer.max.poll.records": "10" - "consumer.max.poll.interval.ms": "900000" - "config.storage.replication.factor": "1" - "offset.storage.replication.factor": "1" - "status.storage.replication.factor": "1" - cp-schema-registry: - url: http://k8kafka-cp-schema-registry:8081 - prometheus: - jmx: - enabled: false - - cp-kafka-rest: - enabled: true - imageTag: 7.1.3 - fullnameOverride: "k8kafka-cp-rest" - heapOptions: "-Xms256M -Xmx256M" - resources: - requests: - cpu: 50m - memory: 0.25G - limits: - cpu: 250m - memory: 0.5G - prometheus: - jmx: - enabled: false - - cp-ksql-server: - enabled: false - cp-control-center: - enabled: false - ``` + ## Deploy Streams Explorer @@ -210,14 +112,16 @@ helm repo update Below you can find an example for the `--values ./streams-explorer.yaml` file configuring the deployment accordingly. Now, deploy the service: - ```shell - helm upgrade \ - --install \ - --version 0.2.3 \ - --values ./streams-explorer.yaml \ - --namespace kpops \ - streams-explorer streams-explorer/streams-explorer - ``` +```shell +helm upgrade \ + --install \ + --version 0.2.3 \ + --values ./streams-explorer.yaml \ + --namespace kpops \ + streams-explorer streams-explorer/streams-explorer +``` + + ??? example "Streams Explorer Helm chart values (`streams-explorer.yaml`)" An example value configuration for Steams Explorer Helm chart. @@ -237,6 +141,8 @@ Below you can find an example for the `--values ./streams-explorer.yaml` file co memory: 300Mi ``` + + ## Check the status of your deployments Now we will check if all the pods are running in our namespace. You can list all pods in the namespace with this command: diff --git a/docs/docs/user/getting-started/teardown.md b/docs/docs/user/getting-started/teardown.md index 7f8bb4d27..47c839a18 100644 --- a/docs/docs/user/getting-started/teardown.md +++ b/docs/docs/user/getting-started/teardown.md @@ -13,26 +13,26 @@ In case that doesn't work, the pipeline can always be taken down manually with ` 1. Export environment variables. - ```shell - export DOCKER_REGISTRY=bakdata && \ - export NAMESPACE=kpops - ``` + ```shell + export DOCKER_REGISTRY=bakdata && \ + export NAMESPACE=kpops + ``` -2. Navigate to the `examples` folder. +2. Navigate to the `examples` folder. Replace the `` with the example you want to tear down. For example the `atm-fraud-detection`. 3. Remove the pipeline - ```shell - # Uncomment 1 line to either destroy, reset or clean. + ```shell + # Uncomment 1 line to either destroy, reset or clean. - # poetry run kpops destroy /pipeline.yaml \ - # poetry run kpops reset /pipeline.yaml \ - # poetry run kpops clean /pipeline.yaml \ - --config /config.yaml \ - --execute - ``` + # poetry run kpops destroy /pipeline.yaml \ + # poetry run kpops reset /pipeline.yaml \ + # poetry run kpops clean /pipeline.yaml \ + --config /config.yaml \ + --execute + ``` ## Infrastructure @@ -42,11 +42,15 @@ Delete namespace: kubectl delete namespace kpops ``` + + !!! Note In case `kpops destroy` is not working one can uninstall the pipeline services one by one. This is equivalent to running `kpops destroy`. In case a clean uninstall (like the one `kpops clean` does) is needed, one needs to also delete the topics and schemas created by deployment of the pipeline. + + ## Local cluster Delete local cluster: diff --git a/docs/docs/user/migration-guide/v1-v2.md b/docs/docs/user/migration-guide/v1-v2.md new file mode 100644 index 000000000..c5936cbe5 --- /dev/null +++ b/docs/docs/user/migration-guide/v1-v2.md @@ -0,0 +1,177 @@ +# Migrate from V1 to V2 + +## [Derive component type automatically from class name](https://github.com/bakdata/kpops/pull/309) + +KPOps automatically infers the component `type` from the class name. Therefore, the `type` and `schema_type` attributes can be removed from your custom components. By convention the `type` would be the lower, and kebab cased name of the class. + +```diff +class MyCoolStreamApp(StreamsApp): +- type = "my-cool-stream-app" ++ ... +``` + +Because of this new convention `producer` has been renamed to `producer-app`. This must be addressed in your `pipeline.yaml` and `defaults.yaml`. + +```diff +- producer: ++ producer-app: + app: + streams: + outputTopic: output_topic + extraOutputTopics: + output_role1: output_topic1 + output_role2: output_topic2 +``` + +## [Refactor input/output types](https://github.com/bakdata/kpops/pull/232) + +### To section + +In the `to` section these have changed: + +- The default type is `output` +- If `role` is set, type is inferred to be `extra` +- The type `error` needs to be defined explicitly + +```diff + to: + topics: + ${pipeline_name}-topic-1: +- type: extra + role: "role-1" + ... + ${pipeline_name}-topic-2: +- type: output + ... + ${pipeline_name}-topic-3: + type: error + ... +``` + +### From section + +In the `from` section these have changed: + +- The default type is `input` +- `input-pattern` type is replaced by `pattern` +- If `role` is set, type is inferred to be `extra` +- If `role` is set, type is explicitly set to `pattern`, this would be inferred type `extra-pattern` + +```diff + from: + topics: + ${pipeline_name}-input-topic: +- type: input + ... + ${pipeline_name}-extra-topic: +- type: extra + role: topic-role + ... + ${pipeline_name}-input-pattern-topic: +- type: input-pattern ++ type: pattern + ... + ${pipeline_name}-extra-pattern-topic: +- type: extra-pattern ++ type: pattern + role: some-role + ... +``` + +## [Remove camel case conversion of internal models](https://github.com/bakdata/kpops/pull/308) + +All the internal KPOps models are now snake_case, and only Helm/Kubernetes values require camel casing. You can find an example of a `pipeline.yaml` in the following. Notice that the `app` section here remains untouched. + +```diff +... +type: streams-app + name: streams-app + namespace: namespace + app: + streams: + brokers: ${brokers} + schemaRegistryUrl: ${schema_registry_url} + autoscaling: + consumerGroup: consumer-group + lagThreshold: 0 + enabled: false + pollingInterval: 30 + + to: + topics: + ${pipeline_name}-output-topic: + type: error +- keySchema: key-schema ++ key_schema: key-schema +- valueSchema: value-schema ++ value_schema: value-schema + partitions_count: 1 + replication_factor: 1 + configs: + cleanup.policy: compact + models: + model: model + prefix: ${pipeline_name}- +- repoConfig: ++ repo_config: +- repositoryName: bakdata-streams-bootstrap ++ repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ +- repoAuthFlags: ++ repo_auth_flags: + username: user + password: pass + ca_file: /home/user/path/to/ca-file + insecure_skip_tls_verify: false + version: "1.0.4" +... +``` + +## [Refactor handling of Helm flags](https://github.com/bakdata/kpops/pull/319) + +If you are using the `KubernetesApp` class to define your own Kubernetes resource to deploy, the abstract function `get_helm_chart` that returns the chart for deploying the app using Helm is now a Python property and renamed to `helm_chart`. + +```diff +class MyCoolApp(KubernetesApp): + ++ @property + @override +- def get_helm_chart(self) -> str: ++ def helm_chart(self) -> str: + return "./charts/charts-folder" +``` + +## [Plural broker field in pipeline config](https://github.com/bakdata/kpops/pull/278) + +Since you can pass a comma separated string of broker address, the broker field in KPOps is now plural. The pluralization has affected multiple areas: + +#### config.yaml + +```diff + environment: development +- broker: "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092" ++ brokers: "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092" + kafka_connect_host: "http://localhost:8083" + kafka_rest_host: "http://localhost:8082" + schema_registry_url: "http://localhost:8081" +``` + +#### pipeline.yaml and default.yaml + +The variable is now called `brokers`. + +```diff +... + app: + streams: +- brokers: ${broker} ++ brokers: ${brokers} + schemaRegistryUrl: ${schema_registry_url} + nameOverride: override-with-this-name + imageTag: "1.0.0" +... +``` + +#### Environment variable + +Previously, if you set the environment variable `KPOPS_KAFKA_BROKER`, you need to replace that now with `KPOPS_KAFKA_BROKERS`. diff --git a/docs/docs/user/references/editor-integration.md b/docs/docs/user/references/editor-integration.md index 690206503..86b7e93d0 100644 --- a/docs/docs/user/references/editor-integration.md +++ b/docs/docs/user/references/editor-integration.md @@ -9,17 +9,20 @@ KPOps provides JSON schemas that enable autocompletion and validation for some o ## Usage -1. Install the -[yaml-language-server](https://github.com/redhat-developer/yaml-language-server#clients){target=_blank} in your editor of choice. (requires LSP support) +1. Install the [yaml-language-server](https://github.com/redhat-developer/yaml-language-server#clients){target=_blank} in your editor of choice. (requires LSP support) 2. Configure the extension with the settings below. + + ??? note "`settings.json`" ```json - --8<-- - ./docs/resources/editor_integration/settings.json - --8<-- + --8<-- + ./docs/resources/editor_integration/settings.json + --8<-- ``` !!! tip "Advanced usage" It is possible to generate schemas with the [`kpops schema`](./cli-commands.md#kpops-schema) command. Useful when using custom components or when using a pre-release version of KPOps. + + diff --git a/docs/docs/user/what-is-kpops.md b/docs/docs/user/what-is-kpops.md index cf16bb40d..ccac918c4 100644 --- a/docs/docs/user/what-is-kpops.md +++ b/docs/docs/user/what-is-kpops.md @@ -2,10 +2,9 @@ With a couple of easy commands in the shell and a [`pipeline.yaml`](#example) of under 30 lines, KPOps can not only [`deploy`](./references/cli-commands.md#kpops-deploy) a Kafka pipeline[^1] to a Kubernetes cluster, but also [`reset`](./references/cli-commands.md#kpops-reset), [`clean`](./references/cli-commands.md#kpops-clean) or [`destroy`](./references/cli-commands.md#kpops-destroy) it! -[^1]: - A Kafka pipeline can consist of consecutive [streaming applications](./core-concepts/components/streams-app.md), - [producers](./core-concepts/components/producer-app.md), - and [connectors](./core-concepts/components/kafka-connector.md). +[^1]: A Kafka pipeline can consist of consecutive [streaming applications](./core-concepts/components/streams-app.md), +[producers](./core-concepts/components/producer-app.md), +and [connectors](./core-concepts/components/kafka-connector.md). ## Key features @@ -24,7 +23,7 @@ With a couple of easy commands in the shell and a [`pipeline.yaml`](#example) of ```yaml title="Word-count pipeline.yaml" - --8<-- - https://raw.githubusercontent.com/bakdata/kpops-examples/main/word-count/deployment/kpops/pipeline.yaml - --8<-- +--8<-- +https://raw.githubusercontent.com/bakdata/kpops-examples/main/word-count/deployment/kpops/pipeline.yaml +--8<-- ``` diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 743568e0e..02aa207dc 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -4,15 +4,16 @@ site_url: https://bakdata.github.io/kpops/ remote_branch: gh-pages copyright: Copyright © 2023 bakdata +extra_css: + - stylesheets/extra.css + theme: name: "material" - custom_dir: overrides + custom_dir: docs/overrides language: "en" - palette: - primary: "custom" font: - text: "Roboto" - code: "Roboto Mono" + text: "SF Pro" + code: "DejaVu Sans Mono" features: - navigation.sections - navigation.tabs @@ -100,7 +101,13 @@ nav: - Environment variables: user/core-concepts/variables/environment_variables.md - Substitution: user/core-concepts/variables/substitution.md - References: + - Migration guide: + - Migrate from v1 to v2: user/migration-guide/v1-v2.md - CLI usage: user/references/cli-commands.md - Editor integration: user/references/editor-integration.md - CI integration: - GitHub Actions: user/references/ci-integration/github-actions.md + - Developer Guide: + - Auto generation: developer/auto-generation.md + - Formatting: developer/formatting.md + diff --git a/dprint.json b/dprint.json new file mode 100644 index 000000000..d702f9ea9 --- /dev/null +++ b/dprint.json @@ -0,0 +1,20 @@ +{ + "$schema": "https://dprint.dev/schemas/v0.json", + "incremental": true, + "markdown": {}, + "includes": [ + "**/*.{md}" + ], + "excludes": [ + ".pytest_cache/**", + ".mypy_cache/**", + "**/.venv/**", + "CHANGELOG.md", + "**/cli-commands.md", + "**/cli_env_vars.md", + "**/config_env_vars.md" + ], + "plugins": [ + "https://plugins.dprint.dev/markdown-0.16.0.wasm" + ] +} diff --git a/hooks/__init__.py b/hooks/__init__.py index 0ae8ea143..ef17ce38a 100644 --- a/hooks/__init__.py +++ b/hooks/__init__.py @@ -1,4 +1,4 @@ -"""KPOps pre-commit hooks""" +"""KPOps pre-commit hooks.""" from pathlib import Path PATH_ROOT = Path(__file__).parents[1] diff --git a/hooks/gen_docs/__init__.py b/hooks/gen_docs/__init__.py index 63b0a9366..5a0d63a28 100644 --- a/hooks/gen_docs/__init__.py +++ b/hooks/gen_docs/__init__.py @@ -1,30 +1,27 @@ """Documentation generation.""" -from collections.abc import Generator +from collections.abc import Iterator from enum import Enum -from typing import Any, Generic, TypeVar -_T = TypeVar("_T") +class IterableStrEnum(str, Enum): + """Polyfill that also introduces dict-like behavior. -class SuperEnum(Generic[_T], Enum): - """Adds constructors that return all items in a ``Generator``. - - Introduces constructors that return a ``Generator`` object + Introduces constructors that return a ``Iterator`` object either containing all items, only their names or their values. """ @classmethod - def items(cls) -> Generator[tuple[_T, Any], None, None]: + def items(cls) -> Iterator[tuple[str, str]]: """Return all item names and values in tuples.""" return ((e.name, e.value) for e in cls) @classmethod - def keys(cls) -> Generator[_T, None, None]: + def keys(cls) -> Iterator[str]: """Return all item names.""" return (e.name for e in cls) @classmethod - def values(cls) -> Generator[Any, None, None]: + def values(cls) -> Iterator[str]: """Return all item values.""" return (e.value for e in cls) diff --git a/hooks/gen_docs/gen_docs_cli_usage.py b/hooks/gen_docs/gen_docs_cli_usage.py index f03c4fd62..469274745 100644 --- a/hooks/gen_docs/gen_docs_cli_usage.py +++ b/hooks/gen_docs/gen_docs_cli_usage.py @@ -21,7 +21,7 @@ "--output", str(PATH_CLI_COMMANDS_DOC), ] - subprocess.run(typer_args) + subprocess.run(typer_args, check=True, capture_output=True) # Replace wrong title in CLI Usage doc with PATH_CLI_COMMANDS_DOC.open("r") as f: diff --git a/hooks/gen_docs/gen_docs_components.py b/hooks/gen_docs/gen_docs_components.py index 30a35d8d6..3fffd8c7b 100644 --- a/hooks/gen_docs/gen_docs_components.py +++ b/hooks/gen_docs/gen_docs_components.py @@ -213,9 +213,9 @@ def get_sections(component_name: str, *, exist_changes: bool) -> KpopsComponent: ] component_sections_not_inherited: list[ str - ] = DEFAULTS_PIPELINE_COMPONENT_DEPENDENCIES[ # type: ignore [reportGeneralTypeIssues] + ] = DEFAULTS_PIPELINE_COMPONENT_DEPENDENCIES[ component_file_name - ] + ] # type: ignore [reportGeneralTypeIssues] return KpopsComponent(component_sections, component_sections_not_inherited) diff --git a/hooks/gen_docs/gen_docs_env_vars.py b/hooks/gen_docs/gen_docs_env_vars.py index 5bd218643..ac88b82b6 100644 --- a/hooks/gen_docs/gen_docs_env_vars.py +++ b/hooks/gen_docs/gen_docs_env_vars.py @@ -2,12 +2,14 @@ import csv import shutil -from collections.abc import Callable +from collections.abc import Callable, Iterator from dataclasses import dataclass from pathlib import Path from textwrap import fill from typing import Any +from pydantic import BaseSettings +from pydantic.fields import ModelField from pytablewriter import MarkdownTableWriter from typer.models import ArgumentInfo, OptionInfo @@ -17,7 +19,7 @@ from typing_extensions import Self from hooks import PATH_ROOT -from hooks.gen_docs import SuperEnum +from hooks.gen_docs import IterableStrEnum from kpops.cli import main from kpops.cli.pipeline_config import PipelineConfig @@ -90,7 +92,7 @@ def from_record(cls, record: dict[str, Any]) -> Self: ) -class EnvVarAttrs(str, SuperEnum): +class EnvVarAttrs(IterableStrEnum): """The attr names are used as columns for the markdown tables.""" NAME = "Name" @@ -103,7 +105,7 @@ class EnvVarAttrs(str, SuperEnum): def csv_append_env_var( file: Path, name: str, - default_value, + default_value: Any, description: str | list[str] | None, *args, ) -> None: @@ -218,7 +220,7 @@ def write_csv_to_md_file( target: Path, title: str | None, description: str | None = None, - heading: str = "###", + heading: str | None = "###", ) -> None: """Write csv data from a file into a markdown file. @@ -227,11 +229,15 @@ def write_csv_to_md_file( :param title: Title for the table, optional """ + if heading: + heading += " " + else: + heading = "" with target.open("w+") as f: if title: - f.write(f"{heading} {title}\n") + f.write(f"{heading}{title}\n\n") if description: - f.write(f"\n{description}\n\n") + f.write(f"{description}\n\n") writer = MarkdownTableWriter() with source.open("r", newline="") as source_contents: writer.from_csv(source_contents.read()) @@ -239,7 +245,7 @@ def write_csv_to_md_file( writer.dump(output=f) -def __fill_csv_pipeline_config(target: Path) -> None: +def fill_csv_pipeline_config(target: Path) -> None: """Append all ``PipelineConfig``-related env vars to a ``.csv`` file. Finds all ``PipelineConfig``-related env vars and appends them to @@ -248,25 +254,38 @@ def __fill_csv_pipeline_config(target: Path) -> None: :param target: The path to the `.csv` file. Note that it must already contain the column names """ - # NOTE: This does not see nested fields, hence if there are env vars in a class like - # TopicConfig(), they wil not be listed. Possible fix with recursion. - config_fields = PipelineConfig.model_fields - for field_name, field_value in config_fields.items(): - config_field_description: str = ( - field_value.description + for field in collect_fields(PipelineConfig): + field_info = PipelineConfig.Config.get_field_info(field.name) + field_description: str = ( + field.field_info.description or "No description available, please refer to the pipeline config documentation." ) - config_field_default = field_value.default - csv_append_env_var( - target, - field_value.serialization_alias or field_name, - config_field_default, - config_field_description, - field_name, - ) + field_default = field.field_info.default + if config_env_var := field_info.get( + "env", + ) or field.field_info.extra.get("env"): + csv_append_env_var( + target, + config_env_var, + field_default, + field_description, + field.name, + ) + + +def collect_fields(settings: type[BaseSettings]) -> Iterator[ModelField]: + """Collect and yield all fields in a settings class. + + :param model: settings class + :yield: all settings including nested ones in settings classes + """ + for field in settings.__fields__.values(): + if issubclass(field_type := field.type_, BaseSettings): + yield from collect_fields(field_type) + yield field -def __fill_csv_cli(target: Path) -> None: +def fill_csv_cli(target: Path) -> None: """Append all CLI-commands-related env vars to a ``.csv`` file. Finds all CLI-commands-related env vars and appends them to a ``.csv`` @@ -278,7 +297,7 @@ def __fill_csv_cli(target: Path) -> None: var_in_main = getattr(main, var_in_main_name) if ( not var_in_main_name.startswith("__") - and isinstance(var_in_main, (OptionInfo, ArgumentInfo)) + and isinstance(var_in_main, OptionInfo | ArgumentInfo) and var_in_main.envvar ): cli_env_var_description: list[str] = [ @@ -311,7 +330,7 @@ def gen_vars( csv_file: Path, title_dotenv_file: str, description_dotenv_file: str, - columns: list, + columns: list[str], description_md_file: str, variable_extraction_function: Callable[[Path], None], ) -> None: @@ -326,7 +345,7 @@ def gen_vars( :param description_dotenv_file: The description to be written in the dotenv file :param columns: The column names in the table :param description_md_file: The description to be written in the markdown file - :param variable_extraction_function: Function that ooks for variables and appends + :param variable_extraction_function: Function that looks for variables and appends them to the temp csv file. """ # Overwrite/create the temp csv file @@ -369,7 +388,7 @@ def gen_vars( + DESCRIPTION_CONFIG_ENV_VARS, columns=list(EnvVarAttrs.values()), description_md_file=DESCRIPTION_CONFIG_ENV_VARS, - variable_extraction_function=__fill_csv_pipeline_config, + variable_extraction_function=fill_csv_pipeline_config, ) # Find all cli-related env variables, write them into a file gen_vars( @@ -381,5 +400,5 @@ def gen_vars( + DESCRIPTION_CLI_ENV_VARS, columns=list(EnvVarAttrs.values())[:-1], description_md_file=DESCRIPTION_CLI_ENV_VARS, - variable_extraction_function=__fill_csv_cli, + variable_extraction_function=fill_csv_cli, ) diff --git a/hooks/gen_schema.py b/hooks/gen_schema.py index e72f2bcf5..88d70b7fd 100644 --- a/hooks/gen_schema.py +++ b/hooks/gen_schema.py @@ -1,4 +1,4 @@ -"""Generates the stock KPOps editor integration schemas""" +"""Generates the stock KPOps editor integration schemas.""" from contextlib import redirect_stdout from io import StringIO from pathlib import Path @@ -10,7 +10,7 @@ def gen_schema(scope: SchemaScope): - """Generates the specified schema and saves it to a file. + """Generate the specified schema and save it to a file. The file is located in docs/docs/schema and is named ``.json`` diff --git a/justfile b/justfile index a907023d4..eb5116739 100644 --- a/justfile +++ b/justfile @@ -14,7 +14,3 @@ docs-config := base-directory / "docs/mkdocs.yml" # Serve current docs located in ./docs/docs serve-docs port="8000": poetry run mkdocs serve --config-file {{ docs-config }} --dev-addr localhost:{{ port }} - -# Serve docs deployed in branch 'documentation' -serve-deployed-docs port="8000": - mike serve --config-file {{ docs-config }} --dev-addr localhost:{{ port }} diff --git a/kpops/__init__.py b/kpops/__init__.py index e11400c24..20b037690 100644 --- a/kpops/__init__.py +++ b/kpops/__init__.py @@ -1,4 +1,4 @@ -__version__ = "2.0.2" +__version__ = "2.0.9" # export public API functions from kpops.cli.main import clean, deploy, destroy, generate, reset diff --git a/kpops/cli/main.py b/kpops/cli/main.py index 3abccb10a..572a7b82b 100644 --- a/kpops/cli/main.py +++ b/kpops/cli/main.py @@ -3,7 +3,7 @@ import logging from enum import Enum from pathlib import Path -from typing import TYPE_CHECKING, Iterator, Optional +from typing import TYPE_CHECKING, Optional import dtyper import typer @@ -24,6 +24,8 @@ from kpops.utils.gen_schema import SchemaScope, gen_config_schema, gen_pipeline_schema if TYPE_CHECKING: + from collections.abc import Iterator + from kpops.components.base_components import PipelineComponent LOG_DIVIDER = "#" * 100 @@ -31,7 +33,7 @@ app = dtyper.Typer(pretty_exceptions_enable=False) BASE_DIR_PATH_OPTION: Path = typer.Option( - default=Path("."), + default=Path(), exists=True, dir_okay=True, file_okay=False, @@ -145,14 +147,14 @@ def parse_steps(steps: str) -> set[str]: def get_step_names(steps_to_apply: list[PipelineComponent]) -> list[str]: - return [step.name.removeprefix(step.prefix) for step in steps_to_apply] + return [step.name for step in steps_to_apply] def filter_steps_to_apply( pipeline: Pipeline, steps: set[str], filter_type: FilterType ) -> list[PipelineComponent]: def is_in_steps(component: PipelineComponent) -> bool: - return component.name.removeprefix(component.prefix) in steps + return component.name in steps log.debug( f"KPOPS_PIPELINE_STEPS is defined with values: {steps} and filter type of {filter_type.value}" @@ -253,7 +255,9 @@ def generate( pipeline = setup_pipeline( pipeline_base_dir, pipeline_path, components_module, pipeline_config ) - pipeline.print_yaml() + + if not template: + pipeline.print_yaml() if template: steps_to_apply = get_steps_to_apply(pipeline, steps, filter_type) @@ -370,7 +374,7 @@ def clean( def version_callback(show_version: bool) -> None: if show_version: typer.echo(f"KPOps {__version__}") - raise typer.Exit() + raise typer.Exit @app.callback() diff --git a/kpops/cli/pipeline_config.py b/kpops/cli/pipeline_config.py index c37c880f0..ad66dfb27 100644 --- a/kpops/cli/pipeline_config.py +++ b/kpops/cli/pipeline_config.py @@ -1,4 +1,7 @@ +from __future__ import annotations + from pathlib import Path +from typing import TYPE_CHECKING, Any from pydantic import AliasChoices, Field from pydantic_settings import BaseSettings, PydanticBaseSettingsSource @@ -6,6 +9,11 @@ from kpops.cli.settings_sources import YamlConfigSettingsSource from kpops.component_handlers.helm_wrapper.model import HelmConfig, HelmDiffConfig +if TYPE_CHECKING: + from collections.abc import Callable + + from pydantic.env_settings import SettingsSourceCallable + ENV_PREFIX = "KPOPS_" diff --git a/kpops/cli/registry.py b/kpops/cli/registry.py index 410aa1be5..a97e2cd91 100644 --- a/kpops/cli/registry.py +++ b/kpops/cli/registry.py @@ -2,22 +2,24 @@ import importlib import inspect -import os import sys -from collections.abc import Iterator from dataclasses import dataclass, field -from typing import TypeVar +from pathlib import Path +from typing import TYPE_CHECKING, TypeVar from kpops import __name__ from kpops.cli.exception import ClassNotFoundError from kpops.components.base_components.pipeline_component import PipelineComponent +if TYPE_CHECKING: + from collections.abc import Iterator + KPOPS_MODULE = __name__ + "." T = TypeVar("T") ClassDict = dict[str, type[T]] # type -> class -sys.path.append(os.getcwd()) +sys.path.append(str(Path.cwd())) @dataclass @@ -27,9 +29,9 @@ class Registry: _classes: ClassDict[PipelineComponent] = field(default_factory=dict, init=False) def find_components(self, module_name: str) -> None: - """ - Find all PipelineComponent subclasses in module - :param module_name: name of the python module + """Find all PipelineComponent subclasses in module. + + :param module_name: name of the python module. """ for _class in _find_classes(module_name, PipelineComponent): self._classes[_class.type] = _class @@ -37,17 +39,16 @@ def find_components(self, module_name: str) -> None: def __getitem__(self, component_type: str) -> type[PipelineComponent]: try: return self._classes[component_type] - except KeyError: - raise ClassNotFoundError( - f"Could not find a component of type {component_type}" - ) + except KeyError as ke: + msg = f"Could not find a component of type {component_type}" + raise ClassNotFoundError(msg) from ke def find_class(module_name: str, baseclass: type[T]) -> type[T]: try: return next(_find_classes(module_name, baseclass)) - except StopIteration: - raise ClassNotFoundError + except StopIteration as e: + raise ClassNotFoundError from e def _find_classes(module_name: str, baseclass: type[T]) -> Iterator[type[T]]: diff --git a/kpops/component_handlers/__init__.py b/kpops/component_handlers/__init__.py index 988ca7ee7..fa296a574 100644 --- a/kpops/component_handlers/__init__.py +++ b/kpops/component_handlers/__init__.py @@ -2,11 +2,10 @@ from typing import TYPE_CHECKING -from kpops.component_handlers.kafka_connect.kafka_connect_handler import ( - KafkaConnectHandler, -) - if TYPE_CHECKING: + from kpops.component_handlers.kafka_connect.kafka_connect_handler import ( + KafkaConnectHandler, + ) from kpops.component_handlers.schema_handler.schema_handler import SchemaHandler from kpops.component_handlers.topic.handler import TopicHandler diff --git a/kpops/component_handlers/helm_wrapper/dry_run_handler.py b/kpops/component_handlers/helm_wrapper/dry_run_handler.py index 8e260f7df..2d28957b7 100644 --- a/kpops/component_handlers/helm_wrapper/dry_run_handler.py +++ b/kpops/component_handlers/helm_wrapper/dry_run_handler.py @@ -11,7 +11,7 @@ def __init__(self, helm: Helm, helm_diff: HelmDiff, namespace: str) -> None: self.namespace = namespace def print_helm_diff(self, stdout: str, helm_release_name: str, log: Logger) -> None: - """Print the diff of the last and current release of this component + """Print the diff of the last and current release of this component. :param stdout: The output of a Helm command that installs or upgrades the release :param helm_release_name: The Helm release name diff --git a/kpops/component_handlers/helm_wrapper/helm.py b/kpops/component_handlers/helm_wrapper/helm.py index e73ba7209..b1b101b41 100644 --- a/kpops/component_handlers/helm_wrapper/helm.py +++ b/kpops/component_handlers/helm_wrapper/helm.py @@ -4,8 +4,7 @@ import re import subprocess import tempfile -from collections.abc import Iterator -from typing import Iterable +from typing import TYPE_CHECKING import yaml @@ -20,6 +19,9 @@ Version, ) +if TYPE_CHECKING: + from collections.abc import Iterable, Iterator + log = logging.getLogger("Helm") @@ -29,16 +31,17 @@ def __init__(self, helm_config: HelmConfig) -> None: self._debug = helm_config.debug self._version = self.get_version() if self._version.major != 3: - raise RuntimeError( - f"The supported Helm version is 3.x.x. The current Helm version is {self._version.major}.{self._version.minor}.{self._version.patch}" - ) + msg = f"The supported Helm version is 3.x.x. The current Helm version is {self._version.major}.{self._version.minor}.{self._version.patch}" + raise RuntimeError(msg) def add_repo( self, repository_name: str, repository_url: str, - repo_auth_flags: RepoAuthFlags = RepoAuthFlags(), + repo_auth_flags: RepoAuthFlags | None = None, ) -> None: + if repo_auth_flags is None: + repo_auth_flags = RepoAuthFlags() command = [ "helm", "repo", @@ -50,7 +53,7 @@ def add_repo( try: self.__execute(command) - except Exception as e: + except (ReleaseNotFoundException, RuntimeError) as e: if ( len(e.args) == 1 and re.match( @@ -59,9 +62,9 @@ def add_repo( ) is not None ): - log.error(f"Could not add repository {repository_name}. {e}") + log.exception(f"Could not add repository {repository_name}.") else: - raise e + raise if self._version.minor > 7: self.__execute(["helm", "repo", "update", repository_name]) @@ -75,25 +78,25 @@ def upgrade_install( dry_run: bool, namespace: str, values: dict, - flags: HelmUpgradeInstallFlags = HelmUpgradeInstallFlags(), + flags: HelmUpgradeInstallFlags | None = None, ) -> str: - """Prepares and executes the `helm upgrade --install` command""" + """Prepare and execute the `helm upgrade --install` command.""" + if flags is None: + flags = HelmUpgradeInstallFlags() with tempfile.NamedTemporaryFile("w") as values_file: yaml.safe_dump(values, values_file) - command = ["helm"] - command.extend( - [ - "upgrade", - release_name, - chart, - "--install", - "--namespace", - namespace, - "--values", - values_file.name, - ] - ) + command = [ + "helm", + "upgrade", + release_name, + chart, + "--install", + "--namespace", + namespace, + "--values", + values_file.name, + ] command.extend(flags.to_command()) if dry_run: command.append("--dry-run") @@ -105,7 +108,7 @@ def uninstall( release_name: str, dry_run: bool, ) -> str | None: - """Prepares and executes the helm uninstall command""" + """Prepare and execute the helm uninstall command.""" command = [ "helm", "uninstall", @@ -128,7 +131,7 @@ def template( chart: str, namespace: str, values: dict, - flags: HelmTemplateFlags = HelmTemplateFlags(), + flags: HelmTemplateFlags | None = None, ) -> str: """From HELM: Render chart templates locally and display the output. @@ -143,6 +146,8 @@ def template( :param flags: the flags to be set for `helm template`, defaults to HelmTemplateFlags() :return: the output of `helm template` """ + if flags is None: + flags = HelmTemplateFlags() with tempfile.NamedTemporaryFile("w") as values_file: yaml.safe_dump(values, values_file) command = [ @@ -179,9 +184,8 @@ def get_version(self) -> Version: short_version = self.__execute(command) version_match = re.search(r"^v(\d+(?:\.\d+){0,2})", short_version) if version_match is None: - raise RuntimeError( - f"Could not parse the Helm version.\n\nHelm output:\n{short_version}" - ) + msg = f"Could not parse the Helm version.\n\nHelm output:\n{short_version}" + raise RuntimeError(msg) version = map(int, version_match.group(1).split(".")) return Version(*version) @@ -208,8 +212,8 @@ def __execute(self, command: list[str]) -> str: log.debug(f"Executing {' '.join(command)}") process = subprocess.run( command, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + check=True, + capture_output=True, text=True, ) Helm.parse_helm_command_stderr_output(process.stderr) @@ -230,7 +234,7 @@ def parse_helm_command_stderr_output(stderr: str) -> None: for line in stderr.splitlines(): lower = line.lower() if "release: not found" in lower: - raise ReleaseNotFoundException() + raise ReleaseNotFoundException elif "error" in lower: raise RuntimeError(stderr) elif "warning" in lower: diff --git a/kpops/component_handlers/helm_wrapper/helm_diff.py b/kpops/component_handlers/helm_wrapper/helm_diff.py index e778a7df2..26de5613a 100644 --- a/kpops/component_handlers/helm_wrapper/helm_diff.py +++ b/kpops/component_handlers/helm_wrapper/helm_diff.py @@ -1,6 +1,5 @@ import logging -from collections.abc import Iterator -from typing import Iterable +from collections.abc import Iterable, Iterator from kpops.component_handlers.helm_wrapper.model import HelmDiffConfig, HelmTemplate from kpops.utils.dict_differ import Change, render_diff @@ -17,7 +16,7 @@ def calculate_changes( current_release: Iterable[HelmTemplate], new_release: Iterable[HelmTemplate], ) -> Iterator[Change[dict]]: - """Compare 2 releases and generate a Change object for each difference + """Compare 2 releases and generate a Change object for each difference. :param current_release: Iterable containing HelmTemplate objects for the current release :param new_release: Iterable containing HelmTemplate objects for the new release diff --git a/kpops/component_handlers/helm_wrapper/model.py b/kpops/component_handlers/helm_wrapper/model.py index 5ff258ddf..66100d7ef 100644 --- a/kpops/component_handlers/helm_wrapper/model.py +++ b/kpops/component_handlers/helm_wrapper/model.py @@ -1,6 +1,6 @@ +from collections.abc import Iterator from dataclasses import dataclass from pathlib import Path -from typing import Iterator import yaml from pydantic import BaseModel, ConfigDict, Field @@ -172,7 +172,8 @@ def parse_source(source: str) -> str: # Source: chart/templates/serviceaccount.yaml """ if not source.startswith(HELM_SOURCE_PREFIX): - raise ParseError("Not a valid Helm template source") + msg = "Not a valid Helm template source" + raise ParseError(msg) return source.removeprefix(HELM_SOURCE_PREFIX).strip() @classmethod @@ -197,9 +198,9 @@ def __iter__(self) -> Iterator[str]: @property def manifest(self) -> str: - """ - Reads the manifest section of Helm stdout. `helm upgrade --install` output message contains three sections - in the following order: + """Reads the manifest section of Helm stdout. + + `helm upgrade --install` output message contains three sections in the following order: - HOOKS - MANIFEST diff --git a/kpops/component_handlers/helm_wrapper/utils.py b/kpops/component_handlers/helm_wrapper/utils.py index d39536041..7ad76b93a 100644 --- a/kpops/component_handlers/helm_wrapper/utils.py +++ b/kpops/component_handlers/helm_wrapper/utils.py @@ -7,11 +7,11 @@ def trim_release_name(name: str, suffix: str = "") -> str: - """ - Trim Helm release name while preserving suffix. + """Trim Helm release name while preserving suffix. + :param name: The release name including optional suffix :param suffix: The release suffix to preserve - :return: Truncated release name + :return: Truncated release name. """ if len(name) > RELEASE_NAME_MAX_LEN: new_name = name[: (RELEASE_NAME_MAX_LEN - len(suffix))] + suffix diff --git a/kpops/component_handlers/kafka_connect/connect_wrapper.py b/kpops/component_handlers/kafka_connect/connect_wrapper.py index 7dff05c2d..aba8b2fc1 100644 --- a/kpops/component_handlers/kafka_connect/connect_wrapper.py +++ b/kpops/component_handlers/kafka_connect/connect_wrapper.py @@ -20,9 +20,7 @@ class ConnectWrapper: - """ - Wraps Kafka Connect APIs - """ + """Wraps Kafka Connect APIs.""" def __init__(self, host: str | None): if not host: @@ -40,11 +38,11 @@ def host(self) -> str: def create_connector( self, connector_config: KafkaConnectorConfig ) -> KafkaConnectResponse: - """ - Creates a new connector + """Create a new connector. + API Reference: https://docs.confluent.io/platform/current/connect/references/restapi.html#post--connectors :param connector_config: The config of the connector - :return: The current connector info if successful + :return: The current connector info if successful. """ config_json = connector_config.model_dump() connect_data = {"name": connector_config.name, "config": config_json} @@ -68,7 +66,7 @@ def get_connector(self, connector_name: str | None) -> KafkaConnectResponse: Get information about the connector. API Reference: https://docs.confluent.io/platform/current/connect/references/restapi.html#get--connectors-(string-name) :param connector_name: Nameof the crated connector - :return: Information about the connector + :return: Information about the connector. """ if connector_name is None: msg = "Connector name not set" @@ -82,7 +80,7 @@ def get_connector(self, connector_name: str | None) -> KafkaConnectResponse: return KafkaConnectResponse(**response.json()) elif response.status_code == httpx.codes.NOT_FOUND: log.info(f"The named connector {connector_name} does not exists.") - raise ConnectorNotFoundException() + raise ConnectorNotFoundException elif response.status_code == httpx.codes.CONFLICT: log.warning( "Rebalancing in progress while getting a connector... Retrying..." @@ -94,8 +92,11 @@ def get_connector(self, connector_name: str | None) -> KafkaConnectResponse: def update_connector_config( self, connector_config: KafkaConnectorConfig ) -> KafkaConnectResponse: - """ - Create a new connector using the given configuration, or update the configuration for an existing connector. + """Create or update a connector. + + Create a new connector using the given configuration,or update the + configuration for an existing connector. + :param connector_config: Configuration parameters for the connector. :return: Information about the connector after the change has been made. """ @@ -126,10 +127,11 @@ def update_connector_config( def validate_connector_config( self, connector_config: KafkaConnectorConfig ) -> list[str]: - """ - Validate connector config using the given configuration + """Validate connector config using the given configuration. + :param connector_config: Configuration parameters for the connector. - :return: + :raises KafkaConnectError: Kafka Konnect error + :return: List of all found errors """ response = httpx.put( url=f"{self._host}/connector-plugins/{connector_config.class_name}/config/validate", @@ -142,7 +144,7 @@ def validate_connector_config( **response.json() ) - errors = [] + errors: list[str] = [] if kafka_connect_error_response.error_count > 0: for config in kafka_connect_error_response.configs: if len(config.value.errors) > 0: @@ -154,9 +156,12 @@ def validate_connector_config( raise KafkaConnectError(response) def delete_connector(self, connector_name: str) -> None: - """ - Deletes a connector, halting all tasks and deleting its configuration. - API Reference:https://docs.confluent.io/platform/current/connect/references/restapi.html#delete--connectors-(string-name)- + """Delete a connector, halting all tasks and deleting its configuration. + + API Reference: + https://docs.confluent.io/platform/current/connect/references/restapi.html#delete--connectors-(string-name)-. + :param connector_name: Configuration parameters for the connector. + :raises ConnectorNotFoundException: Connector not found """ response = httpx.delete( url=f"{self._host}/connectors/{connector_name}", headers=HEADERS @@ -166,7 +171,7 @@ def delete_connector(self, connector_name: str) -> None: return elif response.status_code == httpx.codes.NOT_FOUND: log.info(f"The named connector {connector_name} does not exists.") - raise ConnectorNotFoundException() + raise ConnectorNotFoundException elif response.status_code == httpx.codes.CONFLICT: log.warning( "Rebalancing in progress while deleting a connector... Retrying..." diff --git a/kpops/component_handlers/kafka_connect/kafka_connect_handler.py b/kpops/component_handlers/kafka_connect/kafka_connect_handler.py index 4b8ccca47..6bd981045 100644 --- a/kpops/component_handlers/kafka_connect/kafka_connect_handler.py +++ b/kpops/component_handlers/kafka_connect/kafka_connect_handler.py @@ -8,18 +8,18 @@ ConnectorNotFoundException, ConnectorStateException, ) -from kpops.component_handlers.kafka_connect.model import KafkaConnectorConfig from kpops.component_handlers.kafka_connect.timeout import timeout from kpops.utils.colorify import magentaify from kpops.utils.dict_differ import render_diff -try: - from typing import Self -except ImportError: - from typing_extensions import Self - if TYPE_CHECKING: + try: + from typing import Self + except ImportError: + from typing_extensions import Self + from kpops.cli.pipeline_config import PipelineConfig + from kpops.component_handlers.kafka_connect.model import KafkaConnectorConfig log = logging.getLogger("KafkaConnectHandler") @@ -36,8 +36,10 @@ def __init__( def create_connector( self, connector_config: KafkaConnectorConfig, *, dry_run: bool ) -> None: - """ - Creates a connector. If the connector exists the config of that connector gets updated. + """Create a connector. + + If the connector exists the config of that connector gets updated. + :param connector_config: The connector config. :param dry_run: If the connector creation should be run in dry run mode. """ @@ -64,8 +66,8 @@ def create_connector( ) def destroy_connector(self, connector_name: str, *, dry_run: bool) -> None: - """ - Deletes a connector resource from the cluster. + """Delete a connector resource from the cluster. + :param connector_name: The connector name. :param dry_run: If the connector deletion should be run in dry run mode. """ @@ -112,9 +114,8 @@ def __dry_run_connector_creation( errors = self._connect_wrapper.validate_connector_config(connector_config) if len(errors) > 0: formatted_errors = "\n".join(errors) - raise ConnectorStateException( - f"Connector Creation: validating the connector config for connector {connector_name} resulted in the following errors: {formatted_errors}" - ) + msg = f"Connector Creation: validating the connector config for connector {connector_name} resulted in the following errors: {formatted_errors}" + raise ConnectorStateException(msg) else: log.info( f"Connector Creation: connector config for {connector_name} is valid!" diff --git a/kpops/component_handlers/kafka_connect/model.py b/kpops/component_handlers/kafka_connect/model.py index 0ebe6a914..9efeebdeb 100644 --- a/kpops/component_handlers/kafka_connect/model.py +++ b/kpops/component_handlers/kafka_connect/model.py @@ -31,7 +31,8 @@ class KafkaConnectorConfig(DescConfigModel): @field_validator("connector_class") def connector_class_must_contain_dot(cls, connector_class: str) -> str: if "." not in connector_class: - raise ValueError(f"Invalid connector class {connector_class}") + msg = f"Invalid connector class {connector_class}" + raise ValueError(msg) return connector_class @property diff --git a/kpops/component_handlers/kafka_connect/timeout.py b/kpops/component_handlers/kafka_connect/timeout.py index d93d608b7..e75ac7361 100644 --- a/kpops/component_handlers/kafka_connect/timeout.py +++ b/kpops/component_handlers/kafka_connect/timeout.py @@ -1,7 +1,8 @@ import asyncio import logging from asyncio import TimeoutError -from typing import Callable, TypeVar +from collections.abc import Callable +from typing import TypeVar log = logging.getLogger("Timeout") @@ -9,10 +10,10 @@ def timeout(func: Callable[..., T], *, secs: int = 0) -> T | None: - """ - Sets a timeout for a given lambda function + """Set a timeout for a given lambda function. + :param func: The callable function - :param secs: The timeout in seconds + :param secs: The timeout in seconds. """ async def main_supervisor(func: Callable[..., T], secs: int) -> T: @@ -25,9 +26,8 @@ async def main_supervisor(func: Callable[..., T], secs: int) -> T: loop = asyncio.get_event_loop() try: - complete = loop.run_until_complete(main_supervisor(func, secs)) - return complete + return loop.run_until_complete(main_supervisor(func, secs)) except TimeoutError: - log.error( + log.exception( f"Kafka Connect operation {func.__name__} timed out after {secs} seconds. To increase the duration, set the `timeout` option in config.yaml." ) diff --git a/kpops/component_handlers/schema_handler/schema_handler.py b/kpops/component_handlers/schema_handler/schema_handler.py index a053ccc62..63d88b726 100644 --- a/kpops/component_handlers/schema_handler/schema_handler.py +++ b/kpops/component_handlers/schema_handler/schema_handler.py @@ -3,20 +3,23 @@ import json import logging from functools import cached_property +from typing import TYPE_CHECKING from schema_registry.client import SchemaRegistryClient from schema_registry.client.schema import AvroSchema from kpops.cli.exception import ClassNotFoundError -from kpops.cli.pipeline_config import PipelineConfig from kpops.cli.registry import find_class from kpops.component_handlers.schema_handler.schema_provider import ( Schema, SchemaProvider, ) -from kpops.components.base_components.models.to_section import ToSection from kpops.utils.colorify import greenify, magentaify +if TYPE_CHECKING: + from kpops.cli.pipeline_config import PipelineConfig + from kpops.components.base_components.models.to_section import ToSection + log = logging.getLogger("SchemaHandler") @@ -29,16 +32,13 @@ def __init__(self, url: str, components_module: str | None): def schema_provider(self) -> SchemaProvider: try: if not self.components_module: - raise ValueError( - f"The Schema Registry URL is set but you haven't specified the component module path. Please provide a valid component module path where your {SchemaProvider.__name__} implementation exists." - ) + msg = f"The Schema Registry URL is set but you haven't specified the component module path. Please provide a valid component module path where your {SchemaProvider.__name__} implementation exists." + raise ValueError(msg) schema_provider_class = find_class(self.components_module, SchemaProvider) return schema_provider_class() # pyright: ignore[reportGeneralTypeIssues] - except ClassNotFoundError: - raise ValueError( - f"No schema provider found in components module {self.components_module}. " - f"Please implement the abstract method in {SchemaProvider.__module__}.{SchemaProvider.__name__}." - ) + except ClassNotFoundError as e: + msg = f"No schema provider found in components module {self.components_module}. Please implement the abstract method in {SchemaProvider.__module__}.{SchemaProvider.__name__}." + raise ValueError(msg) from e @classmethod def load_schema_handler( @@ -144,9 +144,8 @@ def __check_compatibility( if isinstance(schema, AvroSchema) else str(schema) ) - raise Exception( - f"Schema is not compatible for {subject} and model {schema_class}. \n {json.dumps(schema_str, indent=4)}" - ) + msg = f"Schema is not compatible for {subject} and model {schema_class}. \n {json.dumps(schema_str, indent=4)}" + raise Exception(msg) else: log.debug( f"Schema Submission: schema was already submitted for the subject {subject} as version {registered_version.schema}. Therefore, the specified schema must be compatible." diff --git a/kpops/component_handlers/schema_handler/schema_provider.py b/kpops/component_handlers/schema_handler/schema_provider.py index 2b93bf943..0c0423a40 100644 --- a/kpops/component_handlers/schema_handler/schema_provider.py +++ b/kpops/component_handlers/schema_handler/schema_provider.py @@ -1,11 +1,12 @@ from __future__ import annotations from abc import ABC, abstractmethod -from typing import TypeAlias +from typing import TYPE_CHECKING, TypeAlias from schema_registry.client.schema import AvroSchema, JsonSchema -from kpops.components.base_components.models import ModelName, ModelVersion +if TYPE_CHECKING: + from kpops.components.base_components.models import ModelName, ModelVersion Schema: TypeAlias = AvroSchema | JsonSchema diff --git a/kpops/component_handlers/topic/handler.py b/kpops/component_handlers/topic/handler.py index 386b8b512..2cda5c4a9 100644 --- a/kpops/component_handlers/topic/handler.py +++ b/kpops/component_handlers/topic/handler.py @@ -65,7 +65,7 @@ def create_topics(self, to_section: ToSection, dry_run: bool) -> None: self.proxy_wrapper.create_topic(topic_spec=topic_spec) def delete_topics(self, to_section: ToSection, dry_run: bool) -> None: - for topic_name in to_section.topics.keys(): + for topic_name in to_section.topics: if dry_run: self.__dry_run_topic_deletion(topic_name=topic_name) else: @@ -148,9 +148,8 @@ def __check_partition_count( f"Topic Creation: partition count of topic {topic_name} did not change. Current partitions count {partition_count}. Updating configs." ) else: - raise TopicTransactionError( - f"Topic Creation: partition count of topic {topic_name} changed! Partitions count of topic {topic_name} is {partition_count}. The given partitions count {topic_spec.partitions_count}." - ) + msg = f"Topic Creation: partition count of topic {topic_name} changed! Partitions count of topic {topic_name} is {partition_count}. The given partitions count {topic_spec.partitions_count}." + raise TopicTransactionError(msg) @staticmethod def __check_replication_factor( @@ -168,9 +167,8 @@ def __check_replication_factor( f"Topic Creation: replication factor of topic {topic_name} did not change. Current replication factor {replication_factor}. Updating configs." ) else: - raise TopicTransactionError( - f"Topic Creation: replication factor of topic {topic_name} changed! Replication factor of topic {topic_name} is {replication_factor}. The given replication count {topic_spec.replication_factor}." - ) + msg = f"Topic Creation: replication factor of topic {topic_name} changed! Replication factor of topic {topic_name} is {replication_factor}. The given replication count {topic_spec.replication_factor}." + raise TopicTransactionError(msg) def __dry_run_topic_deletion(self, topic_name: str) -> None: try: @@ -199,11 +197,11 @@ def __dry_run_topic_deletion(self, topic_name: str) -> None: @classmethod def __prepare_body(cls, topic_name: str, topic_config: TopicConfig) -> TopicSpec: - """ - Prepares the POST request body needed for the topic creation + """Prepare the POST request body needed for the topic creation. + :param topic_name: The name of the topic :param topic_config: The topic config - :return: + :return: Topic specification """ topic_spec_json: dict = topic_config.model_dump( include={ diff --git a/kpops/component_handlers/topic/proxy_wrapper.py b/kpops/component_handlers/topic/proxy_wrapper.py index 34ed3011e..1fbdb50b1 100644 --- a/kpops/component_handlers/topic/proxy_wrapper.py +++ b/kpops/component_handlers/topic/proxy_wrapper.py @@ -21,27 +21,26 @@ class ProxyWrapper: - """ - Wraps Kafka REST Proxy APIs - """ + """Wraps Kafka REST Proxy APIs.""" def __init__(self, pipeline_config: PipelineConfig) -> None: if not pipeline_config.kafka_rest_host: - raise ValueError( - "The Kafka REST Proxy host is not set. Please set the host in the config.yaml using the kafka_rest_host property or set the environemt variable KPOPS_REST_PROXY_HOST." - ) + msg = "The Kafka REST Proxy host is not set. Please set the host in the config.yaml using the kafka_rest_host property or set the environemt variable KPOPS_REST_PROXY_HOST." + raise ValueError(msg) self._host = pipeline_config.kafka_rest_host @cached_property def cluster_id(self) -> str: - """ - Gets the Kafka cluster ID by sending a requests to Kafka REST proxy. + """Get the Kafka cluster ID by sending a request to Kafka REST proxy. + More information about the cluster ID can be found here: - https://docs.confluent.io/platform/current/kafka-rest/api.html#cluster-v3 + https://docs.confluent.io/platform/current/kafka-rest/api.html#cluster-v3. Currently both Kafka and Kafka REST Proxy are only aware of the Kafka cluster pointed at by the bootstrap.servers configuration. Therefore, only one Kafka cluster will be returned. + + :raises KafkaRestProxyError: Kafka REST proxy error :return: The Kafka cluster ID. """ response = httpx.get(url=f"{self._host}/v3/clusters") @@ -56,10 +55,13 @@ def host(self) -> str: return self._host def create_topic(self, topic_spec: TopicSpec) -> None: - """ - Creates a topic. - API Reference: https://docs.confluent.io/platform/current/kafka-rest/api.html#post--clusters-cluster_id-topics + """Create a topic. + + API Reference: + https://docs.confluent.io/platform/current/kafka-rest/api.html#post--clusters-cluster_id-topics + :param topic_spec: The topic specification. + :raises KafkaRestProxyError: Kafka REST proxy error """ response = httpx.post( url=f"{self._host}/v3/clusters/{self.cluster_id}/topics", @@ -74,10 +76,13 @@ def create_topic(self, topic_spec: TopicSpec) -> None: raise KafkaRestProxyError(response) def delete_topic(self, topic_name: str) -> None: - """ - Deletes a topic - API Reference: https://docs.confluent.io/platform/current/kafka-rest/api.html#delete--clusters-cluster_id-topics-topic_name - :param topic_name: Name of the topic + """Delete a topic. + + API Reference: + https://docs.confluent.io/platform/current/kafka-rest/api.html#delete--clusters-cluster_id-topics-topic_name + + :param topic_name: Name of the topic. + :raises KafkaRestProxyError: Kafka REST proxy error """ response = httpx.delete( url=f"{self.host}/v3/clusters/{self.cluster_id}/topics/{topic_name}", @@ -90,11 +95,15 @@ def delete_topic(self, topic_name: str) -> None: raise KafkaRestProxyError(response) def get_topic(self, topic_name: str) -> TopicResponse: - """ - Returns the topic with the given topic_name. - API Reference: https://docs.confluent.io/platform/current/kafka-rest/api.html#get--clusters-cluster_id-topics-topic_name + """Return the topic with the given topic_name. + + API Reference: + https://docs.confluent.io/platform/current/kafka-rest/api.html#get--clusters-cluster_id-topics-topic_name + :param topic_name: The topic name. - :return: Response of the get topic API + :raises TopicNotFoundException: Topic not found + :raises KafkaRestProxyError: Kafka REST proxy error + :return: Response of the get topic API. """ response = httpx.get( url=f"{self.host}/v3/clusters/{self.cluster_id}/topics/{topic_name}", @@ -111,15 +120,19 @@ def get_topic(self, topic_name: str) -> TopicResponse: ): log.debug(f"Topic {topic_name} not found.") log.debug(response.json()) - raise TopicNotFoundException() + raise TopicNotFoundException raise KafkaRestProxyError(response) def get_topic_config(self, topic_name: str) -> TopicConfigResponse: - """ - Return the config with the given topic_name. - API Reference: https://docs.confluent.io/platform/current/kafka-rest/api.html#acl-v3 + """Return the config with the given topic_name. + + API Reference: + https://docs.confluent.io/platform/current/kafka-rest/api.html#acl-v3 + :param topic_name: The topic name. + :raises TopicNotFoundException: Topic not found + :raises KafkaRestProxyError: Kafka REST proxy error :return: The topic configuration. """ response = httpx.get( @@ -138,16 +151,19 @@ def get_topic_config(self, topic_name: str) -> TopicConfigResponse: ): log.debug(f"Configs for {topic_name} not found.") log.debug(response.json()) - raise TopicNotFoundException() + raise TopicNotFoundException raise KafkaRestProxyError(response) def batch_alter_topic_config(self, topic_name: str, json_body: list[dict]) -> None: - """ - Reset config of given config_name param to the default value on the kafka server. - API Reference: https://docs.confluent.io/platform/current/kafka-rest/api.html#post--clusters-cluster_id-topics-topic_name-configs-alter + """Reset config of given config_name param to the default value on the kafka server. + + API Reference: + https://docs.confluent.io/platform/current/kafka-rest/api.html#post--clusters-cluster_id-topics-topic_name-configs-alter + :param topic_name: The topic name. :param config_name: The configuration parameter name. + :raises KafkaRestProxyError: Kafka REST proxy error """ response = httpx.post( url=f"{self.host}/v3/clusters/{self.cluster_id}/topics/{topic_name}/configs:alter", @@ -161,9 +177,12 @@ def batch_alter_topic_config(self, topic_name: str, json_body: list[dict]) -> No raise KafkaRestProxyError(response) def get_broker_config(self) -> BrokerConfigResponse: - """ - Return the list of configuration parameters for all the brokers in the given Kafka cluster. - API Reference: https://docs.confluent.io/platform/current/kafka-rest/api.html#get--clusters-cluster_id-brokers---configs + """Return the list of configuration parameters for all the brokers in the given Kafka cluster. + + API Reference: + https://docs.confluent.io/platform/current/kafka-rest/api.html#get--clusters-cluster_id-brokers---configs + + :raises KafkaRestProxyError: Kafka REST proxy error :return: The broker configuration. """ response = httpx.get( diff --git a/kpops/component_handlers/utils/exception.py b/kpops/component_handlers/utils/exception.py index fe906190f..5de7f7717 100644 --- a/kpops/component_handlers/utils/exception.py +++ b/kpops/component_handlers/utils/exception.py @@ -10,11 +10,11 @@ def __init__(self, response: httpx.Response) -> None: self.error_code = response.status_code self.error_msg = "Something went wrong!" try: - log.error( - f"The request responded with the code {self.error_code}. Error body: {response.json()}" + log.exception( + f"The request responded with the code {self.error_code}. Error body: {response.json()}", ) response.raise_for_status() except httpx.HTTPError as e: self.error_msg = str(e) - log.error(f"More information: {self.error_msg}") + log.exception(f"More information: {self.error_msg}") super().__init__() diff --git a/kpops/components/base_components/base_defaults_component.py b/kpops/components/base_components/base_defaults_component.py index 15ca0bb28..17e780e58 100644 --- a/kpops/components/base_components/base_defaults_component.py +++ b/kpops/components/base_components/base_defaults_component.py @@ -76,15 +76,15 @@ def __init__(self, **kwargs) -> None: self._validate_custom(**kwargs) @cached_classproperty - def type(cls: type[Self]) -> str: # pyright: ignore - """Return calling component's type + def type(cls: type[Self]) -> str: # pyright: ignore[reportGeneralTypeIssues] + """Return calling component's type. :returns: Component class name in dash-case """ return to_dash(cls.__name__) def extend_with_defaults(self, **kwargs) -> dict: - """Merge parent components' defaults with own + """Merge parent components' defaults with own. :param kwargs: The init kwargs for pydantic :returns: Enriched kwargs with inheritted defaults @@ -104,15 +104,13 @@ def extend_with_defaults(self, **kwargs) -> dict: defaults = load_defaults( self.__class__, main_default_file_path, environment_default_file_path ) - kwargs = update_nested(kwargs, defaults) - return kwargs + return update_nested(kwargs, defaults) def _validate_custom(self, **kwargs) -> None: """Run custom validation on component. :param kwargs: The init kwargs for the component """ - pass def load_defaults( @@ -120,7 +118,7 @@ def load_defaults( defaults_file_path: Path, environment_defaults_file_path: Path | None = None, ) -> dict: - """Resolve component-specific defaults including environment defaults + """Resolve component-specific defaults including environment defaults. :param component_class: Component class :param defaults_file_path: Path to `defaults.yaml` @@ -152,7 +150,7 @@ def load_defaults( def defaults_from_yaml(path: Path, key: str) -> dict: - """Read component-specific settings from a defaults yaml file and return @default if not found + """Read component-specific settings from a defaults yaml file and return @default if not found. :param path: Path to defaults yaml file :param key: Component type @@ -164,9 +162,10 @@ def defaults_from_yaml(path: Path, key: str) -> dict: """ content = load_yaml_file(path, substitution=ENV) if not isinstance(content, dict): - raise TypeError( + msg = ( "Default files should be structured as map ([app type] -> [default config]" ) + raise TypeError(msg) value = content.get(key) if value is None: return {} @@ -177,7 +176,7 @@ def defaults_from_yaml(path: Path, key: str) -> dict: def get_defaults_file_paths(config: PipelineConfig) -> tuple[Path, Path]: - """Return the paths to the main and the environment defaults-files + """Return the paths to the main and the environment defaults-files. The files need not exist, this function will only check if the dir set in `config.defaults_path` exists and return paths to the defaults files diff --git a/kpops/components/base_components/kafka_app.py b/kpops/components/base_components/kafka_app.py index 28c16c9ac..9d277a4f7 100644 --- a/kpops/components/base_components/kafka_app.py +++ b/kpops/components/base_components/kafka_app.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +from abc import ABC from pydantic import ConfigDict, Field from typing_extensions import override @@ -38,7 +39,7 @@ class KafkaStreamsConfig(CamelCaseConfigModel, DescConfigModel): class KafkaAppConfig(KubernetesAppConfig): - """Settings specific to Kafka Apps + """Settings specific to Kafka Apps. :param streams: Kafka streams config :param name_override: Override name with this value, defaults to None @@ -52,7 +53,7 @@ class KafkaAppConfig(KubernetesAppConfig): ) -class KafkaApp(KubernetesApp): +class KafkaApp(KubernetesApp, ABC): """Base component for Kafka-based components. Producer or streaming apps should inherit from this class. @@ -82,8 +83,8 @@ class KafkaApp(KubernetesApp): @property def clean_up_helm_chart(self) -> str: - """Helm chart used to destroy and clean this component""" - raise NotImplementedError() + """Helm chart used to destroy and clean this component.""" + raise NotImplementedError @override def deploy(self, dry_run: bool) -> None: @@ -104,7 +105,7 @@ def _run_clean_up_job( dry_run: bool, retain_clean_jobs: bool = False, ) -> None: - """Clean an app using the respective cleanup job + """Clean an app using the respective cleanup job. :param values: The value YAML for the chart :param dry_run: Dry run command @@ -133,7 +134,7 @@ def _run_clean_up_job( self.__uninstall_clean_up_job(clean_up_release_name, dry_run) def __uninstall_clean_up_job(self, release_name: str, dry_run: bool) -> None: - """Uninstall clean up job + """Uninstall clean up job. :param release_name: Name of the Helm release :param dry_run: Whether to do a dry run of the command @@ -147,10 +148,10 @@ def __install_clean_up_job( values: dict, dry_run: bool, ) -> str: - """Install clean up job + """Install clean up job. :param release_name: Name of the Helm release - :param suffix: Suffix to add to the realease name, e.g. "-clean" + :param suffix: Suffix to add to the release name, e.g. "-clean" :param values: The Helm values for the chart :param dry_run: Whether to do a dry run of the command :return: Install clean up job with helm, return the output of the installation diff --git a/kpops/components/base_components/kafka_connector.py b/kpops/components/base_components/kafka_connector.py index 56a6d065d..43a7c246d 100644 --- a/kpops/components/base_components/kafka_connector.py +++ b/kpops/components/base_components/kafka_connector.py @@ -3,7 +3,7 @@ import logging from abc import ABC from functools import cached_property -from typing import Any, NoReturn +from typing import TYPE_CHECKING, Any, NoReturn from pydantic import Field, ValidationInfo, field_validator from typing_extensions import override @@ -25,16 +25,18 @@ KafkaConnectResetterValues, ) from kpops.components.base_components.base_defaults_component import deduplicate -from kpops.components.base_components.models.from_section import FromTopic from kpops.components.base_components.pipeline_component import PipelineComponent from kpops.utils.colorify import magentaify from kpops.utils.docstring import describe_attr +if TYPE_CHECKING: + from kpops.components.base_components.models.from_section import FromTopic + log = logging.getLogger("KafkaConnector") class KafkaConnector(PipelineComponent, ABC): - """Base class for all Kafka connectors + """Base class for all Kafka connectors. Should only be used to set defaults @@ -46,6 +48,7 @@ class KafkaConnector(PipelineComponent, ABC): :param version: Helm chart version, defaults to "1.0.4" :param resetter_values: Overriding Kafka Connect Resetter Helm values. E.g. to override the Image Tag etc., defaults to dict + :param _connector_type: Defines the type of the connector (Source or Sink) """ namespace: str = Field( @@ -70,7 +73,7 @@ class KafkaConnector(PipelineComponent, ABC): default_factory=dict, description=describe_attr("resetter_values", __doc__), ) - + _connector_type: KafkaConnectorType = Field() @field_validator("app") @classmethod def connector_config_should_have_component_name( @@ -83,14 +86,15 @@ def connector_config_should_have_component_name( component_name = info.data["prefix"] + info.data["name"] connector_name: str | None = app.get("name") if connector_name is not None and connector_name != component_name: - raise ValueError("Connector name should be the same as component name") + msg = "Connector name should be the same as component name" + raise ValueError(msg) app["name"] = component_name app = KafkaConnectorConfig(**app) return app @cached_property def helm(self) -> Helm: - """Helm object that contains component-specific config such as repo""" + """Helm object that contains component-specific config such as repo.""" helm_repo_config = self.repo_config helm = Helm(self.config.helm_config) helm.add_repo( @@ -100,11 +104,14 @@ def helm(self) -> Helm: ) return helm - def _get_resetter_helm_chart(self) -> str: - """Get reseter Helm chart + @property + def _resetter_release_name(self) -> str: + suffix = "-clean" + clean_up_release_name = self.full_name + suffix + return trim_release_name(clean_up_release_name, suffix) - :return: returns the component resetter's helm chart - """ + @property + def _resetter_helm_chart(self) -> str: return f"{self.repo_config.repository_name}/kafka-connect-resetter" @cached_property @@ -112,14 +119,9 @@ def dry_run_handler(self) -> DryRunHandler: helm_diff = HelmDiff(self.config.helm_diff_config) return DryRunHandler(self.helm, helm_diff, self.namespace) - @property - def kafka_connect_resetter_chart(self) -> str: - """Resetter chart for this component""" - return f"{self.repo_config.repository_name}/kafka-connect-resetter" - @property def helm_flags(self) -> HelmFlags: - """Return shared flags for Helm commands""" + """Return shared flags for Helm commands.""" return HelmFlags( **self.repo_config.repo_auth_flags.model_dump(), version=self.version, @@ -128,7 +130,7 @@ def helm_flags(self) -> HelmFlags: @property def template_flags(self) -> HelmTemplateFlags: - """Return flags for Helm template command""" + """Return flags for Helm template command.""" return HelmTemplateFlags( **self.helm_flags.model_dump(), api_version=self.config.helm_config.api_version, @@ -150,7 +152,9 @@ def deploy(self, dry_run: bool) -> None: @override def destroy(self, dry_run: bool) -> None: - self.handlers.connector_handler.destroy_connector(self.name, dry_run=dry_run) + self.handlers.connector_handler.destroy_connector( + self.full_name, dry_run=dry_run + ) @override def clean(self, dry_run: bool) -> None: @@ -163,81 +167,58 @@ def clean(self, dry_run: bool) -> None: def _run_connect_resetter( self, - connector_name: str, - connector_type: KafkaConnectorType, dry_run: bool, retain_clean_jobs: bool, **kwargs, ) -> None: - """Clean the connector from the cluster + """Clean the connector from the cluster. At first, it deletes the previous cleanup job (connector resetter) to make sure that there is no running clean job in the cluster. Then it releases a cleanup job. If the retain_clean_jobs flag is set to false the cleanup job will be deleted. - :param connector_name: Name of the connector - :param connector_type: Type of the connector (SINK or SOURCE) :param dry_run: If the cleanup should be run in dry run mode or not :param retain_clean_jobs: If the cleanup job should be kept :param kwargs: Other values for the KafkaConnectResetter """ - trimmed_name = self._get_kafka_resetter_release_name(connector_name) - log.info( magentaify( - f"Connector Cleanup: uninstalling cleanup job Helm release from previous runs for {connector_name}" + f"Connector Cleanup: uninstalling cleanup job Helm release from previous runs for {self.full_name}" ) ) - self.__uninstall_connect_resetter(trimmed_name, dry_run) + self.__uninstall_connect_resetter(self._resetter_release_name, dry_run) log.info( magentaify( - f"Connector Cleanup: deploy Connect {connector_type.value} resetter for {connector_name}" + f"Connector Cleanup: deploy Connect {self._connector_type.value} resetter for {self.full_name}" ) ) - stdout = self.__install_connect_resetter( - trimmed_name, connector_name, connector_type, dry_run, **kwargs - ) + stdout = self.__install_connect_resetter(dry_run, **kwargs) if dry_run: - self.dry_run_handler.print_helm_diff(stdout, trimmed_name, log) + self.dry_run_handler.print_helm_diff( + stdout, self._resetter_release_name, log + ) if not retain_clean_jobs: log.info(magentaify("Connector Cleanup: uninstall Kafka Resetter.")) - self.__uninstall_connect_resetter(trimmed_name, dry_run) - - def _get_kafka_resetter_release_name(self, connector_name: str) -> str: - """Get connector resetter's release name - - :param connector_name: Name of the connector to be reset - :return: The name of the resetter to be used - """ - suffix = "-clean" - clean_up_release_name = connector_name + suffix - trimmed_name = trim_release_name(clean_up_release_name, suffix) - return trimmed_name + self.__uninstall_connect_resetter(self._resetter_release_name, dry_run) def __install_connect_resetter( self, - release_name: str, - connector_name: str, - connector_type: KafkaConnectorType, dry_run: bool, **kwargs, ) -> str: - """Install connector resetter + """Install connector resetter. - :param release_name: Release name for the resetter - :param connector_name: Name of the connector-to-be-reset - :param connector_type: Type of the connector :param dry_run: Whether to dry run the command :return: The output of `helm upgrade --install` """ return self.helm.upgrade_install( - release_name=release_name, + release_name=self._resetter_release_name, namespace=self.namespace, - chart=self.kafka_connect_resetter_chart, + chart=self._resetter_helm_chart, dry_run=dry_run, flags=HelmUpgradeInstallFlags( create_namespace=self.config.create_namespace, @@ -246,39 +227,33 @@ def __install_connect_resetter( wait=True, ), values=self._get_kafka_connect_resetter_values( - connector_name, - connector_type, **kwargs, ), ) def _get_kafka_connect_resetter_values( self, - connector_name: str, - connector_type: KafkaConnectorType, **kwargs, ) -> dict: - """Get connector resetter helm chart values + """Get connector resetter helm chart values. - :param connector_name: Name of the connector - :param connector_type: Type of the connector :return: The Helm chart values of the connector resetter """ return { **KafkaConnectResetterValues( config=KafkaConnectResetterConfig( - connector=connector_name, + connector=self.full_name, brokers=self.config.brokers, **kwargs, ), - connector_type=connector_type.value, - name_override=connector_name, + connector_type=self._connector_type.value, + name_override=self.full_name, ).model_dump(), **self.resetter_values, } def __uninstall_connect_resetter(self, release_name: str, dry_run: bool) -> None: - """Uninstall connector resetter + """Uninstall connector resetter. :param release_name: Name of the release to be uninstalled :param dry_run: Whether to do a dry run of the command @@ -291,7 +266,7 @@ def __uninstall_connect_resetter(self, release_name: str, dry_run: bool) -> None class KafkaSourceConnector(KafkaConnector): - """Kafka source connector model + """Kafka source connector model. :param offset_topic: offset.storage.topic, more info: https://kafka.apache.org/documentation/#connect_running, @@ -303,20 +278,21 @@ class KafkaSourceConnector(KafkaConnector): description=describe_attr("offset_topic", __doc__), ) + _connector_type = KafkaConnectorType.SOURCE + @override def apply_from_inputs(self, name: str, topic: FromTopic) -> NoReturn: - raise NotImplementedError("Kafka source connector doesn't support FromSection") + msg = "Kafka source connector doesn't support FromSection" + raise NotImplementedError(msg) @override def template(self) -> None: values = self._get_kafka_connect_resetter_values( - self.name, - KafkaConnectorType.SOURCE, offset_topic=self.offset_topic, ) stdout = self.helm.template( - self._get_kafka_resetter_release_name(self.name), - self._get_resetter_helm_chart(), + self._resetter_release_name, + self._resetter_helm_chart, self.namespace, values, self.template_flags, @@ -333,13 +309,11 @@ def clean(self, dry_run: bool) -> None: self.__run_kafka_connect_resetter(dry_run) def __run_kafka_connect_resetter(self, dry_run: bool) -> None: - """Runs the connector resetter + """Run the connector resetter. :param dry_run: Whether to do a dry run of the command """ self._run_connect_resetter( - connector_name=self.name, - connector_type=KafkaConnectorType.SOURCE, dry_run=dry_run, retain_clean_jobs=self.config.retain_clean_jobs, offset_topic=self.offset_topic, @@ -347,7 +321,9 @@ def __run_kafka_connect_resetter(self, dry_run: bool) -> None: class KafkaSinkConnector(KafkaConnector): - """Kafka sink connector model""" + """Kafka sink connector model.""" + + _connector_type = KafkaConnectorType.SINK @override def add_input_topics(self, topics: list[str]) -> None: @@ -358,12 +334,10 @@ def add_input_topics(self, topics: list[str]) -> None: @override def template(self) -> None: - values = self._get_kafka_connect_resetter_values( - self.name, KafkaConnectorType.SINK - ) + values = self._get_kafka_connect_resetter_values() stdout = self.helm.template( - self._get_kafka_resetter_release_name(self.name), - self._get_resetter_helm_chart(), + self._resetter_release_name, + self._resetter_helm_chart, self.namespace, values, self.template_flags, @@ -390,13 +364,12 @@ def clean(self, dry_run: bool) -> None: def __run_kafka_connect_resetter( self, dry_run: bool, delete_consumer_group: bool ) -> None: - """Runs the connector resetter + """Run the connector resetter. :param dry_run: Whether to do a dry run of the command + :param delete_consumer_group: Whether the consumer group should be deleted or not """ self._run_connect_resetter( - connector_name=self.name, - connector_type=KafkaConnectorType.SINK, dry_run=dry_run, retain_clean_jobs=self.config.retain_clean_jobs, delete_consumer_group=delete_consumer_group, diff --git a/kpops/components/base_components/kubernetes_app.py b/kpops/components/base_components/kubernetes_app.py index 4eca9b21d..a6e1581cb 100644 --- a/kpops/components/base_components/kubernetes_app.py +++ b/kpops/components/base_components/kubernetes_app.py @@ -44,7 +44,8 @@ class KubernetesApp(PipelineComponent): :param app: Application-specific settings :param repo_config: Configuration of the Helm chart repo to be used for - deploying the component, defaults to None + deploying the component, defaults to None this means that the command "helm repo add" is not called and Helm + expects a path to local Helm chart. :param namespace: Namespace in which the component shall be deployed :param version: Helm chart version, defaults to None """ @@ -57,8 +58,8 @@ class KubernetesApp(PipelineComponent): default=..., description=describe_attr("app", __doc__), ) - repo_config: HelmRepoConfig = Field( - default=..., + repo_config: HelmRepoConfig | None = Field( + default=None, description=describe_attr("repo_config", __doc__), ) version: str | None = Field( @@ -68,7 +69,7 @@ class KubernetesApp(PipelineComponent): @cached_property def helm(self) -> Helm: - """Helm object that contains component-specific config such as repo""" + """Helm object that contains component-specific config such as repo.""" helm = Helm(self.config.helm_config) if self.repo_config is not None: helm.add_repo( @@ -80,7 +81,7 @@ def helm(self) -> Helm: @cached_property def helm_diff(self) -> HelmDiff: - """Helm diff object of last and current release of this component""" + """Helm diff object of last and current release of this component.""" return HelmDiff(self.config.helm_diff_config) @cached_property @@ -91,27 +92,29 @@ def dry_run_handler(self) -> DryRunHandler: @property def helm_release_name(self) -> str: """The name for the Helm release. Can be overridden.""" - return self.name + return self.full_name @property def helm_chart(self) -> str: - """Return component's Helm chart""" - raise NotImplementedError( + """Return component's Helm chart.""" + msg = ( f"Please implement the helm_chart property of the {self.__module__} module." ) + raise NotImplementedError(msg) @property def helm_flags(self) -> HelmFlags: - """Return shared flags for Helm commands""" + """Return shared flags for Helm commands.""" + auth_flags = self.repo_config.repo_auth_flags.dict() if self.repo_config else {} return HelmFlags( - **self.repo_config.repo_auth_flags.model_dump(), + **auth_flags, version=self.version, create_namespace=self.config.create_namespace, ) @property def template_flags(self) -> HelmTemplateFlags: - """Return flags for Helm template command""" + """Return flags for Helm template command.""" return HelmTemplateFlags( **self.helm_flags.model_dump(), api_version=self.config.helm_config.api_version, @@ -130,8 +133,8 @@ def template(self) -> None: @property def deploy_flags(self) -> HelmUpgradeInstallFlags: - """Return flags for Helm upgrade install command""" - return HelmUpgradeInstallFlags(**self.helm_flags.model_dump()) + """Return flags for Helm upgrade install command.""" + return HelmUpgradeInstallFlags(**self.helm_flags.dict()) @override def deploy(self, dry_run: bool) -> None: @@ -158,7 +161,7 @@ def destroy(self, dry_run: bool) -> None: log.info(magentaify(stdout)) def to_helm_values(self) -> dict: - """Generate a dictionary of values readable by Helm from `self.app` + """Generate a dictionary of values readable by Helm from `self.app`. :returns: Thte values to be used by Helm """ @@ -167,7 +170,7 @@ def to_helm_values(self) -> dict: ) def print_helm_diff(self, stdout: str) -> None: - """Print the diff of the last and current release of this component + """Print the diff of the last and current release of this component. :param stdout: The output of a Helm command that installs or upgrades the release """ @@ -188,13 +191,14 @@ def _validate_custom(self, **kwargs) -> None: @staticmethod def validate_kubernetes_name(name: str) -> None: - """Check if a name is valid for a Kubernetes resource + """Check if a name is valid for a Kubernetes resource. :param name: Name that is to be used for the resource :raises ValueError: The component name {name} is invalid for Kubernetes. """ if not bool(KUBERNETES_NAME_CHECK_PATTERN.match(name)): - raise ValueError(f"The component name {name} is invalid for Kubernetes.") + msg = f"The component name {name} is invalid for Kubernetes." + raise ValueError(msg) @override def model_dump(self, *, exclude=None, **kwargs) -> dict[str, Any]: diff --git a/kpops/components/base_components/models/from_section.py b/kpops/components/base_components/models/from_section.py index a26870abb..57e933f5d 100644 --- a/kpops/components/base_components/models/from_section.py +++ b/kpops/components/base_components/models/from_section.py @@ -9,7 +9,7 @@ class InputTopicTypes(str, Enum): - """Input topic types + """Input topic types. INPUT (input topic), PATTERN (extra-topic-pattern or input-topic-pattern) """ diff --git a/kpops/components/base_components/models/to_section.py b/kpops/components/base_components/models/to_section.py index 901792c11..5bd0e9e03 100644 --- a/kpops/components/base_components/models/to_section.py +++ b/kpops/components/base_components/models/to_section.py @@ -9,7 +9,7 @@ class OutputTopicTypes(str, Enum): - """Types of output topic + """Types of output topic. OUTPUT (output topic), ERROR (error topic) """ diff --git a/kpops/components/base_components/pipeline_component.py b/kpops/components/base_components/pipeline_component.py index ae163ee3b..8c8a7b292 100644 --- a/kpops/components/base_components/pipeline_component.py +++ b/kpops/components/base_components/pipeline_component.py @@ -1,6 +1,7 @@ from __future__ import annotations from pydantic import AliasChoices, ConfigDict, Field +from abc import ABC from kpops.components.base_components.base_defaults_component import ( BaseDefaultsComponent, @@ -18,8 +19,8 @@ from kpops.utils.docstring import describe_attr -class PipelineComponent(BaseDefaultsComponent): - """Base class for all components +class PipelineComponent(BaseDefaultsComponent, ABC): + """Base class for all components. :param name: Component name :param prefix: Pipeline prefix that will prefix every component name. @@ -57,6 +58,10 @@ def __init__(self, **kwargs) -> None: self.set_input_topics() self.set_output_topics() + @property + def full_name(self) -> str: + return self.prefix + self.name + def add_input_topics(self, topics: list[str]) -> None: """Add given topics to the list of input topics. @@ -71,39 +76,39 @@ def add_extra_input_topics(self, role: str, topics: list[str]) -> None: """ def set_input_pattern(self, name: str) -> None: - """Set input pattern + """Set input pattern. :param name: Input pattern name """ def add_extra_input_pattern(self, role: str, topic: str) -> None: - """Add an input pattern of type extra + """Add an input pattern of type extra. :param role: Custom identifier belonging to one or multiple topics :param topic: Topic name """ def set_output_topic(self, topic_name: str) -> None: - """Set output topic + """Set output topic. :param topic_name: Output topic name """ def set_error_topic(self, topic_name: str) -> None: - """Set error topic + """Set error topic. :param topic_name: Error topic name """ def add_extra_output_topic(self, topic_name: str, role: str) -> None: - """Add an output topic of type extra + """Add an output topic of type extra. :param topic_name: Output topic name :param role: Role that is unique to the extra output topic """ def set_input_topics(self) -> None: - """Put values of config.from into the streams config section of streams bootstrap + """Put values of config.from into the streams config section of streams bootstrap. Supports extra_input_topics (topics by role) or input_topics. """ @@ -112,7 +117,7 @@ def set_input_topics(self) -> None: self.apply_from_inputs(name, topic) def apply_from_inputs(self, name: str, topic: FromTopic) -> None: - """Add a `from` section input to the component config + """Add a `from` section input to the component config. :param name: Name of the field :param topic: Value of the field @@ -128,7 +133,7 @@ def apply_from_inputs(self, name: str, topic: FromTopic) -> None: self.add_input_topics([name]) def set_output_topics(self) -> None: - """Put values of config.to into the producer config section of streams bootstrap + """Put values of config.to into the producer config section of streams bootstrap. Supports extra_output_topics (topics by role) or output_topics. """ @@ -137,7 +142,7 @@ def set_output_topics(self) -> None: self.apply_to_outputs(name, topic) def apply_to_outputs(self, name: str, topic: TopicConfig) -> None: - """Add a `to` section input to the component config + """Add a `to` section input to the component config. :param name: Name of the field :param topic: Value of the field @@ -153,12 +158,14 @@ def apply_to_outputs(self, name: str, topic: TopicConfig) -> None: def weave_from_topics( self, to: ToSection | None, - from_topic: FromTopic = FromTopic(type=InputTopicTypes.INPUT), + from_topic: FromTopic | None = None, ) -> None: - """Weave output topics of upstream component or from component into config + """Weave output topics of upstream component or from component into config. Override this method to apply custom logic """ + if from_topic is None: + from_topic = FromTopic(type=InputTopicTypes.INPUT) if not to: return input_topics = [ @@ -170,7 +177,7 @@ def weave_from_topics( self.apply_from_inputs(input_topic, from_topic) def inflate(self) -> list[PipelineComponent]: - """Inflate a component + """Inflate a component. This is helpful if one component should result in multiple components. To support this, override this method and return a list of components @@ -180,8 +187,7 @@ def inflate(self) -> list[PipelineComponent]: return [self] def template(self) -> None: - """ - Runs `helm template` + """Run `helm template`. From HELM: Render chart templates locally and display the output. Any values that would normally be looked up or retrieved in-cluster will @@ -190,25 +196,25 @@ def template(self) -> None: """ def deploy(self, dry_run: bool) -> None: - """Deploy the component (self) to the k8s cluster + """Deploy the component (self) to the k8s cluster. :param dry_run: Whether to do a dry run of the command """ def destroy(self, dry_run: bool) -> None: - """Uninstall the component (self) from the k8s cluster + """Uninstall the component (self) from the k8s cluster. :param dry_run: Whether to do a dry run of the command """ def reset(self, dry_run: bool) -> None: - """Reset component (self) state + """Reset component (self) state. :param dry_run: Whether to do a dry run of the command """ def clean(self, dry_run: bool) -> None: - """Remove component (self) and any trace of it + """Remove component (self) and any trace of it. :param dry_run: Whether to do a dry run of the command """ diff --git a/kpops/components/streams_bootstrap/producer/model.py b/kpops/components/streams_bootstrap/producer/model.py index 1d39d8874..01bda1dbc 100644 --- a/kpops/components/streams_bootstrap/producer/model.py +++ b/kpops/components/streams_bootstrap/producer/model.py @@ -8,7 +8,7 @@ class ProducerStreamsConfig(KafkaStreamsConfig): - """Kafka Streams settings specific to Producer + """Kafka Streams settings specific to Producer. :param extra_output_topics: Extra output topics :param output_topic: Output topic, defaults to None @@ -23,7 +23,7 @@ class ProducerStreamsConfig(KafkaStreamsConfig): class ProducerValues(KafkaAppConfig): - """Settings specific to producers + """Settings specific to producers. :param streams: Kafka Streams settings """ diff --git a/kpops/components/streams_bootstrap/producer/producer_app.py b/kpops/components/streams_bootstrap/producer/producer_app.py index 022ff3e5e..6091cdd77 100644 --- a/kpops/components/streams_bootstrap/producer/producer_app.py +++ b/kpops/components/streams_bootstrap/producer/producer_app.py @@ -1,4 +1,4 @@ -from __future__ import annotations +# from __future__ import annotations from pydantic import Field from typing_extensions import override @@ -14,7 +14,7 @@ class ProducerApp(KafkaApp): - """Producer component + """Producer component. This producer holds configuration to use as values for the streams bootstrap producer helm chart. @@ -40,7 +40,8 @@ class ProducerApp(KafkaApp): def apply_to_outputs(self, name: str, topic: TopicConfig) -> None: match topic.type: case OutputTopicTypes.ERROR: - raise ValueError("Producer apps do not support error topics") + msg = "Producer apps do not support error topics" + raise ValueError(msg) case _: super().apply_to_outputs(name, topic) diff --git a/kpops/components/streams_bootstrap/streams/model.py b/kpops/components/streams_bootstrap/streams/model.py index 56327aed7..bc5ced4e6 100644 --- a/kpops/components/streams_bootstrap/streams/model.py +++ b/kpops/components/streams_bootstrap/streams/model.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping, Set from typing import Any from pydantic import ConfigDict, Field, model_serializer @@ -13,7 +14,7 @@ class StreamsConfig(KafkaStreamsConfig): - """Streams Bootstrap streams section + """Streams Bootstrap streams section. :param input_topics: Input topics, defaults to [] :param input_pattern: Input pattern, defaults to None diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index 956980ff7..a466b4eba 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -1,5 +1,3 @@ -from __future__ import annotations - from pydantic import Field from typing_extensions import override @@ -10,7 +8,7 @@ class StreamsApp(KafkaApp): - """StreamsApp component that configures a streams bootstrap app + """StreamsApp component that configures a streams bootstrap app. :param app: Application-specific settings """ @@ -67,7 +65,7 @@ def clean(self, dry_run: bool) -> None: self.__run_streams_clean_up_job(dry_run, delete_output=True) def __run_streams_clean_up_job(self, dry_run: bool, delete_output: bool) -> None: - """Run clean job for this Streams app + """Run clean job for this Streams app. :param dry_run: Whether to do a dry run of the command :param delete_output: Whether to delete the output of the app that is being cleaned diff --git a/kpops/pipeline_generator/pipeline.py b/kpops/pipeline_generator/pipeline.py index 06e0ed5fd..b9f7ed299 100644 --- a/kpops/pipeline_generator/pipeline.py +++ b/kpops/pipeline_generator/pipeline.py @@ -3,23 +3,27 @@ import json import logging from collections import Counter -from collections.abc import Iterator from contextlib import suppress -from pathlib import Path +from typing import TYPE_CHECKING import yaml from pydantic import BaseModel, SerializeAsAny from rich.console import Console from rich.syntax import Syntax -from kpops.cli.pipeline_config import PipelineConfig -from kpops.cli.registry import Registry -from kpops.component_handlers import ComponentHandlers from kpops.components.base_components.pipeline_component import PipelineComponent from kpops.utils.dict_ops import generate_substitution, update_nested_pair from kpops.utils.environment import ENV from kpops.utils.yaml_loading import load_yaml_file, substitute, substitute_nested +if TYPE_CHECKING: + from collections.abc import Iterator + from pathlib import Path + + from kpops.cli.pipeline_config import PipelineConfig + from kpops.cli.registry import Registry + from kpops.component_handlers import ComponentHandlers + log = logging.getLogger("PipelineGenerator") @@ -32,7 +36,7 @@ class ValidationError(Exception): class PipelineComponents(BaseModel): - """Stores the pipeline components""" + """Stores the pipeline components.""" components: list[SerializeAsAny[PipelineComponent]] = [] @@ -42,9 +46,10 @@ def last(self) -> PipelineComponent: def find(self, component_name: str) -> PipelineComponent: for component in self.components: - if component_name == component.name.removeprefix(component.prefix): + if component_name == component.name: return component - raise ValueError(f"Component {component_name} not found") + msg = f"Component {component_name} not found" + raise ValueError(msg) def add(self, component: PipelineComponent) -> None: self._populate_component_name(component) @@ -60,27 +65,25 @@ def __len__(self) -> int: return len(self.components) def validate_unique_names(self) -> None: - step_names = [component.name for component in self.components] + step_names = [component.full_name for component in self.components] duplicates = [name for name, count in Counter(step_names).items() if count > 1] if duplicates: - raise ValidationError( - f"step names should be unique. duplicate step names: {', '.join(duplicates)}" - ) + msg = f"step names should be unique. duplicate step names: {', '.join(duplicates)}" + raise ValidationError(msg) @staticmethod - def _populate_component_name(component: PipelineComponent) -> None: - component.name = component.prefix + component.name + def _populate_component_name(component: PipelineComponent) -> None: # TODO: remove with suppress( AttributeError # Some components like Kafka Connect do not have a name_override attribute ): if (app := getattr(component, "app")) and app.name_override is None: - app.name_override = component.name + app.name_override = component.full_name def create_env_components_index( environment_components: list[dict], ) -> dict[str, dict]: - """Create an index for all registered components in the project + """Create an index for all registered components in the project. :param environment_components: List of all components to be included :return: component index @@ -88,9 +91,8 @@ def create_env_components_index( index: dict[str, dict] = {} for component in environment_components: if "type" not in component or "name" not in component: - raise ValueError( - "To override components per environment, every component should at least have a type and a name." - ) + msg = "To override components per environment, every component should at least have a type and a name." + raise ValueError(msg) index[component["name"]] = component return index @@ -121,7 +123,7 @@ def load_from_yaml( config: PipelineConfig, handlers: ComponentHandlers, ) -> Pipeline: - """Load pipeline definition from yaml + """Load pipeline definition from yaml. The file is often named ``pipeline.yaml`` @@ -138,22 +140,19 @@ def load_from_yaml( main_content = load_yaml_file(path, substitution=ENV) if not isinstance(main_content, list): - raise TypeError( - f"The pipeline definition {path} should contain a list of components" - ) + msg = f"The pipeline definition {path} should contain a list of components" + raise TypeError(msg) env_content = [] if (env_file := Pipeline.pipeline_filename_environment(path, config)).exists(): env_content = load_yaml_file(env_file, substitution=ENV) if not isinstance(env_content, list): - raise TypeError( - f"The pipeline definition {env_file} should contain a list of components" - ) + msg = f"The pipeline definition {env_file} should contain a list of components" + raise TypeError(msg) - pipeline = cls(main_content, env_content, registry, config, handlers) - return pipeline + return cls(main_content, env_content, registry, config, handlers) def parse_components(self, component_list: list[dict]) -> None: - """Instantiate, enrich and inflate a list of components + """Instantiate, enrich and inflate a list of components. :param component_list: List of components :raises ValueError: Every component must have a type defined @@ -164,19 +163,17 @@ def parse_components(self, component_list: list[dict]) -> None: try: try: component_type: str = component_data["type"] - except KeyError: - raise ValueError( - "Every component must have a type defined, this component does not have one." - ) + except KeyError as ke: + msg = "Every component must have a type defined, this component does not have one." + raise ValueError(msg) from ke component_class = self.registry[component_type] self.apply_component(component_class, component_data) - except Exception as ex: + except Exception as ex: # noqa: BLE001 if "name" in component_data: - raise ParsingException( - f"Error enriching {component_data['type']} component {component_data['name']}" - ) from ex + msg = f"Error enriching {component_data['type']} component {component_data['name']}" + raise ParsingException(msg) from ex else: - raise ParsingException() from ex + raise ParsingException from ex def apply_component( self, component_class: type[PipelineComponent], component_data: dict @@ -208,12 +205,8 @@ def apply_component( original_from_component_name ) inflated_from_component = original_from_component.inflate()[-1] - if inflated_from_component is not original_from_component: - resolved_from_component_name = inflated_from_component.name - else: - resolved_from_component_name = original_from_component_name resolved_from_component = self.components.find( - resolved_from_component_name + inflated_from_component.name ) enriched_component.weave_from_topics( resolved_from_component.to, from_topic @@ -228,7 +221,7 @@ def enrich_component( self, component: PipelineComponent, ) -> PipelineComponent: - """Enrich a pipeline component with env-specific config and substitute variables + """Enrich a pipeline component with env-specific config and substitute variables. :param component: Component to be enriched :returns: Enriched component @@ -252,7 +245,7 @@ def enrich_component( ) def print_yaml(self, substitution: dict | None = None) -> None: - """Print the generated pipeline definition + """Print the generated pipeline definition. :param substitution: Substitution dictionary, defaults to None """ @@ -278,7 +271,7 @@ def __len__(self) -> int: return len(self.components) def substitute_in_component(self, component_as_dict: dict) -> dict: - """Substitute all $-placeholders in a component in dict representation + """Substitute all $-placeholders in a component in dict representation. :param component_as_dict: Component represented as dict :return: Updated component @@ -312,7 +305,7 @@ def validate(self) -> None: @staticmethod def pipeline_filename_environment(path: Path, config: PipelineConfig) -> Path: - """Add the environment name from the PipelineConfig to the pipeline.yaml path + """Add the environment name from the PipelineConfig to the pipeline.yaml path. :param path: Path to pipeline.yaml file :param config: The PipelineConfig @@ -337,7 +330,8 @@ def set_pipeline_name_env_vars(base_dir: Path, path: Path) -> None: """ path_without_file = path.resolve().relative_to(base_dir.resolve()).parts[:-1] if not path_without_file: - raise ValueError("The pipeline-base-dir should not equal the pipeline-path") + msg = "The pipeline-base-dir should not equal the pipeline-path" + raise ValueError(msg) pipeline_name = "-".join(path_without_file) ENV["pipeline_name"] = pipeline_name for level, parent in enumerate(path_without_file): diff --git a/kpops/utils/dict_differ.py b/kpops/utils/dict_differ.py index 2cdaa95b0..934924e21 100644 --- a/kpops/utils/dict_differ.py +++ b/kpops/utils/dict_differ.py @@ -3,12 +3,15 @@ from dataclasses import dataclass from difflib import Differ from enum import Enum -from typing import Generic, Iterable, Iterator, Sequence, TypeVar +from typing import TYPE_CHECKING, Generic, TypeVar import typer import yaml from dictdiffer import diff, patch +if TYPE_CHECKING: + from collections.abc import Iterable, Iterator, Sequence + differ = Differ() @@ -39,7 +42,8 @@ def factory(type: DiffType, change: T | tuple[T, T]) -> Change: return Change(change, None) case DiffType.CHANGE if isinstance(change, tuple): return Change(*change) - raise ValueError(f"{type} is not part of {DiffType}") + msg = f"{type} is not part of {DiffType}" + raise ValueError(msg) @dataclass @@ -53,9 +57,9 @@ def from_dicts( d1: dict, d2: dict, ignore: set[str] | None = None ) -> Iterator[Diff]: for diff_type, keys, changes in diff(d1, d2, ignore=ignore): - if not isinstance(changes, list): - changes = [("", changes)] - for key, change in changes: + if not isinstance(changes_tmp := changes, list): + changes_tmp = [("", changes)] + for key, change in changes_tmp: yield Diff( DiffType.from_str(diff_type), Diff.__find_changed_key(keys, key), @@ -64,9 +68,7 @@ def from_dicts( @staticmethod def __find_changed_key(key_1: list[str] | str, key_2: str = "") -> str: - """ - Generates a string that points to the changed key in the dictionary. - """ + """Generate a string that points to the changed key in the dictionary.""" if isinstance(key_1, list) and len(key_1) > 1: return f"{key_1[0]}[{key_1[1]}]" if not key_1: diff --git a/kpops/utils/dict_ops.py b/kpops/utils/dict_ops.py index 64e88a89b..14cc849e3 100644 --- a/kpops/utils/dict_ops.py +++ b/kpops/utils/dict_ops.py @@ -1,8 +1,9 @@ -from typing import Any, Mapping +from collections.abc import Mapping +from typing import Any def update_nested_pair(original_dict: dict, other_dict: Mapping) -> dict: - """Nested update for 2 dictionaries + """Nested update for 2 dictionaries. Adds all new fields in ``other_dict`` to ``original_dict``. Does not update existing fields. @@ -19,9 +20,8 @@ def update_nested_pair(original_dict: dict, other_dict: Mapping) -> dict: nested_val = original_dict.get(key, {}) if isinstance(nested_val, dict): original_dict[key] = update_nested_pair(nested_val, value) - else: - if key not in original_dict: - original_dict[key] = value + elif key not in original_dict: + original_dict[key] = value return original_dict @@ -48,7 +48,7 @@ def update_nested(*argv: dict) -> dict: def flatten_mapping( nested_mapping: Mapping[str, Any], prefix: str | None = None, separator: str = "_" ) -> dict[str, Any]: - """Flattens a Mapping + """Flattens a Mapping. :param nested_mapping: Nested mapping that is to be flattened :param prefix: Prefix that will be applied to all top-level keys in the output., defaults to None @@ -56,11 +56,13 @@ def flatten_mapping( :returns: "Flattened" mapping in the form of dict """ if not isinstance(nested_mapping, Mapping): - raise TypeError("Argument nested_mapping is not a Mapping") + msg = "Argument nested_mapping is not a Mapping" + raise TypeError(msg) top: dict[str, Any] = {} for key, value in nested_mapping.items(): if not isinstance(key, str): - raise TypeError(f"Argument nested_mapping contains a non-str key: {key}") + msg = f"Argument nested_mapping contains a non-str key: {key}" + raise TypeError(msg) if prefix: key = prefix + separator + key if isinstance(value, Mapping): @@ -76,7 +78,7 @@ def generate_substitution( prefix: str | None = None, existing_substitution: dict | None = None, ) -> dict: - """Generate a complete substitution dict from a given dict + """Generate a complete substitution dict from a given dict. Finds all attributes that belong to a model and expands them to create a dict containing each variable name and value to substitute with. diff --git a/kpops/utils/docstring.py b/kpops/utils/docstring.py index fc6f4c61d..d5ca287d3 100644 --- a/kpops/utils/docstring.py +++ b/kpops/utils/docstring.py @@ -4,7 +4,7 @@ def describe_attr(name: str, docstr: str | None) -> str: - """Read attribute description from class docstring + """Read attribute description from class docstring. **Works only with reStructuredText docstrings.** @@ -19,7 +19,7 @@ def describe_attr(name: str, docstr: str | None) -> str: def describe_object(docstr: str | None) -> str: - """Return description from an object's docstring + """Return description from an object's docstring. Excludes parameters and return definitions @@ -44,7 +44,7 @@ def describe_object(docstr: str | None) -> str: def _trim_description_end(desc: str) -> str: - """Remove the unwanted text that comes after a description in a docstring + """Remove the unwanted text that comes after a description in a docstring. Also removes all whitespaces and newlines and replaces them with a single space. diff --git a/kpops/utils/environment.py b/kpops/utils/environment.py index c46f83611..0ed7ae920 100644 --- a/kpops/utils/environment.py +++ b/kpops/utils/environment.py @@ -1,7 +1,7 @@ import os import platform from collections import UserDict -from typing import Callable +from collections.abc import Callable class Environment(UserDict): diff --git a/kpops/utils/gen_schema.py b/kpops/utils/gen_schema.py index d392b20f2..6587d76ab 100644 --- a/kpops/utils/gen_schema.py +++ b/kpops/utils/gen_schema.py @@ -1,7 +1,10 @@ import json +import inspect import logging +from abc import ABC +from collections.abc import Sequence from enum import Enum -from typing import Annotated, Literal, Union +from typing import Annotated, Any, Literal, Union from pydantic import Field, TypeAdapter from pydantic.json_schema import model_json_schema, models_json_schema @@ -24,19 +27,20 @@ class SchemaScope(str, Enum): def _is_valid_component( defined_component_types: set[str], component: type[PipelineComponent] ) -> bool: - """ - Check whether a PipelineComponent subclass has a valid definition for the schema generation. + """Check whether a PipelineComponent subclass has a valid definition for the schema generation. :param defined_component_types: types defined so far :param component: component type to be validated :return: Whether component is valid for schema generation """ + if inspect.isabstract(component) or ABC in component.__bases__: + log.warning(f"SKIPPED {component.__name__}, component is abstract.") + return False if component.type in defined_component_types: log.warning(f"SKIPPED {component.__name__}, component type must be unique.") return False - else: - defined_component_types.add(component.type) - return True + defined_component_types.add(component.type) + return True def _add_components( @@ -54,7 +58,7 @@ def _add_components( :return: Extended tuple """ if components is None: - components = tuple() + components = tuple() # noqa: C408 # Set of existing types, against which to check the new ones defined_component_types = {component.type for component in components} custom_components = ( @@ -82,7 +86,7 @@ def gen_pipeline_schema( # Add stock components if enabled components: tuple[type[PipelineComponent], ...] = tuple() if include_stock_components: - components = tuple(_find_classes("kpops.components", PipelineComponent)) + components = _add_components("kpops.components") # Add custom components if provided if components_module: components = _add_components(components_module, components) diff --git a/kpops/utils/yaml_loading.py b/kpops/utils/yaml_loading.py index adafbb884..fb810c193 100644 --- a/kpops/utils/yaml_loading.py +++ b/kpops/utils/yaml_loading.py @@ -70,7 +70,6 @@ def substitute_nested(input: str, **kwargs) -> str: steps.add(new_str) old_str, new_str = new_str, substitute(new_str, kwargs) if new_str != old_str: - raise ValueError( - "An infinite loop condition detected. Check substitution variables." - ) + msg = "An infinite loop condition detected. Check substitution variables." + raise ValueError(msg) return old_str diff --git a/poetry.lock b/poetry.lock index 75bae65c4..a12566d30 100644 --- a/poetry.lock +++ b/poetry.lock @@ -360,22 +360,6 @@ files = [ docs = ["furo (>=2022.9.29)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"] testing = ["covdefaults (>=2.2.2)", "coverage (>=6.5)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-timeout (>=2.1)"] -[[package]] -name = "flake8" -version = "4.0.1" -description = "the modular source code checker: pep8 pyflakes and co" -optional = false -python-versions = ">=3.6" -files = [ - {file = "flake8-4.0.1-py2.py3-none-any.whl", hash = "sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d"}, - {file = "flake8-4.0.1.tar.gz", hash = "sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d"}, -] - -[package.dependencies] -mccabe = ">=0.6.0,<0.7.0" -pycodestyle = ">=2.8.0,<2.9.0" -pyflakes = ">=2.4.0,<2.5.0" - [[package]] name = "ghp-import" version = "2.1.0" @@ -484,23 +468,6 @@ files = [ {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, ] -[[package]] -name = "isort" -version = "5.12.0" -description = "A Python utility / library to sort Python imports." -optional = false -python-versions = ">=3.8.0" -files = [ - {file = "isort-5.12.0-py3-none-any.whl", hash = "sha256:f84c2818376e66cf843d497486ea8fed8700b340f308f076c6fb1229dff318b6"}, - {file = "isort-5.12.0.tar.gz", hash = "sha256:8bef7dde241278824a6d83f44a544709b065191b95b6e50894bdc722fcba0504"}, -] - -[package.extras] -colors = ["colorama (>=0.4.3)"] -pipfile-deprecated-finder = ["pip-shims (>=0.5.2)", "pipreqs", "requirementslib"] -plugins = ["setuptools"] -requirements-deprecated-finder = ["pip-api", "pipreqs"] - [[package]] name = "jinja2" version = "3.1.2" @@ -617,17 +584,6 @@ chardet = ">=3.0.4,<6" [package.extras] test = ["Faker (>=1.0.2)", "pytest (>=6.0.1)", "pytest-md-report (>=0.1)"] -[[package]] -name = "mccabe" -version = "0.6.1" -description = "McCabe checker, plugin for flake8" -optional = false -python-versions = "*" -files = [ - {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, - {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, -] - [[package]] name = "mergedeep" version = "1.3.4" @@ -880,17 +836,6 @@ pyyaml = ">=5.1" toml = "*" virtualenv = ">=20.0.8" -[[package]] -name = "pycodestyle" -version = "2.8.0" -description = "Python style guide checker" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -files = [ - {file = "pycodestyle-2.8.0-py2.py3-none-any.whl", hash = "sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20"}, - {file = "pycodestyle-2.8.0.tar.gz", hash = "sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f"}, -] - [[package]] name = "pydantic" version = "2.4.2" @@ -1043,17 +988,6 @@ files = [ pydantic = ">=2.0.1" python-dotenv = ">=0.21.0" -[[package]] -name = "pyflakes" -version = "2.4.0" -description = "passive checker of Python programs" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -files = [ - {file = "pyflakes-2.4.0-py2.py3-none-any.whl", hash = "sha256:3bb3a3f256f4b7968c9c788781e4ff07dce46bdf12339dcda61053375426ee2e"}, - {file = "pyflakes-2.4.0.tar.gz", hash = "sha256:05a85c2872edf37a4ed30b0cce2f6093e1d0581f8c19d7393122da7e25b2b24c"}, -] - [[package]] name = "pygments" version = "2.14.0" @@ -1559,6 +1493,32 @@ pygments = ">=2.6.0,<3.0.0" [package.extras] jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] +[[package]] +name = "ruff" +version = "0.0.292" +description = "An extremely fast Python linter, written in Rust." +optional = false +python-versions = ">=3.7" +files = [ + {file = "ruff-0.0.292-py3-none-macosx_10_7_x86_64.whl", hash = "sha256:02f29db018c9d474270c704e6c6b13b18ed0ecac82761e4fcf0faa3728430c96"}, + {file = "ruff-0.0.292-py3-none-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:69654e564342f507edfa09ee6897883ca76e331d4bbc3676d8a8403838e9fade"}, + {file = "ruff-0.0.292-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c3c91859a9b845c33778f11902e7b26440d64b9d5110edd4e4fa1726c41e0a4"}, + {file = "ruff-0.0.292-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f4476f1243af2d8c29da5f235c13dca52177117935e1f9393f9d90f9833f69e4"}, + {file = "ruff-0.0.292-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:be8eb50eaf8648070b8e58ece8e69c9322d34afe367eec4210fdee9a555e4ca7"}, + {file = "ruff-0.0.292-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:9889bac18a0c07018aac75ef6c1e6511d8411724d67cb879103b01758e110a81"}, + {file = "ruff-0.0.292-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6bdfabd4334684a4418b99b3118793f2c13bb67bf1540a769d7816410402a205"}, + {file = "ruff-0.0.292-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aa7c77c53bfcd75dbcd4d1f42d6cabf2485d2e1ee0678da850f08e1ab13081a8"}, + {file = "ruff-0.0.292-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e087b24d0d849c5c81516ec740bf4fd48bf363cfb104545464e0fca749b6af9"}, + {file = "ruff-0.0.292-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:f160b5ec26be32362d0774964e218f3fcf0a7da299f7e220ef45ae9e3e67101a"}, + {file = "ruff-0.0.292-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:ac153eee6dd4444501c4bb92bff866491d4bfb01ce26dd2fff7ca472c8df9ad0"}, + {file = "ruff-0.0.292-py3-none-musllinux_1_2_i686.whl", hash = "sha256:87616771e72820800b8faea82edd858324b29bb99a920d6aa3d3949dd3f88fb0"}, + {file = "ruff-0.0.292-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:b76deb3bdbea2ef97db286cf953488745dd6424c122d275f05836c53f62d4016"}, + {file = "ruff-0.0.292-py3-none-win32.whl", hash = "sha256:e854b05408f7a8033a027e4b1c7f9889563dd2aca545d13d06711e5c39c3d003"}, + {file = "ruff-0.0.292-py3-none-win_amd64.whl", hash = "sha256:f27282bedfd04d4c3492e5c3398360c9d86a295be00eccc63914438b4ac8a83c"}, + {file = "ruff-0.0.292-py3-none-win_arm64.whl", hash = "sha256:7f67a69c8f12fbc8daf6ae6d36705037bde315abf8b82b6e1f4c9e74eb750f68"}, + {file = "ruff-0.0.292.tar.gz", hash = "sha256:1093449e37dd1e9b813798f6ad70932b57cf614e5c2b5c51005bf67d55db33ac"}, +] + [[package]] name = "setuptools" version = "65.6.3" diff --git a/pyproject.toml b/pyproject.toml index 0cc035402..13ddd6316 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "kpops" -version = "2.0.2" +version = "2.0.9" description = "KPOps is a tool to deploy Kafka pipelines to Kubernetes" authors = ["bakdata "] license = "MIT" @@ -46,9 +46,8 @@ pytest-mock = "^3.10.0" pytest-timeout = "^2.1.0" snapshottest = "^0.6.0" pre-commit = "^2.19.0" -flake8 = "^4.0.1" +ruff = "^0.0.292" black = "^23.7.0" -isort = "^5.12.0" typer-cli = "^0.0.13" pyright = "^1.1.314" pytest-rerunfailures = "^11.1.2" @@ -68,6 +67,101 @@ mike = "^1.1.2" [tool.poetry_bumpversion.file."kpops/__init__.py"] +[tool.ruff] +ignore = [ + # "E203", # whitespace before ':' -- Not PEP8 compliant, black won't correct it, add when out of nursery + "E501", # Line too long -- Clashes with `black` + "D1", # Missing docstring for {} -- Inconvenient to enforce +# The following "D" rules do not correspond to our coding style. We use the pep257 convention, but +# "D212" should not be ignored. In ruff (0.0.291) we cannot select a rule that is excluded by specifying +# a convention, hence our only option is to manually replicate it. + "D203", # 1 blank line required before class docstring + "D213", # Multi-line docstring summary should start at the second line + "D214", # Section is over-indented ("{name}") + "D215", # Section underline is over-indented ("{name}") + "D404", # First word of the docstring should not be "This" + "D405", # Section name should be properly capitalized ("{name}") + "D406", # Section name should end with a newline ("{name}") + "D407", # Missing dashed underline after section ("{name}") + "D408", # Section underline should be in the line following the section's name ("{name}") + "D409", # Section underline should match the length of its name ("{name}") + "D410", # Missing blank line after section ("{name}") + "D411", # Missing blank line before section ("{name}") + "D413", # Missing blank line after last section ("{name}") + "D415", # First line should end with a period, question mark, or exclamation point + "D416", # Section name should end with a colon ("{name}") + "D417", # Missing argument description in the docstring for {definition}: {name} + "B009", # Do not call getattr with a constant attribute value. -- Not always applicable + "B010", # Do not call setattr with a constant attribute value. -- Not always applicable + "RUF012", # type class attrs with `ClassVar` -- Too strict/trigger-happy + "UP007", # Use X | Y for type annotations -- `typer` doesn't support it + "COM812", # Checks for the absence of trailing commas -- leads to undesirable behavior from formatters + "PIE804", # Unnecessary `dict` kwargs -- Inconvenient to enforce + "RET505", # Unnecessary {branch} after return statement -- Lots of false positives + "RET506", # Unnecessary {branch} after raise statement -- Lots of false positives + "RET507", # Unnecessary {branch} after continue statement -- Lots of false positives + "RET508", # Unnecessary {branch} after break statement -- Lots of false positives + "PLR09", # upper bound on number of arguments, functions, etc. -- Inconvenient to enforce + "PLR2004", # Magic value used in comparison, consider replacing {value} with a constant variable -- Inconvenient to enforce + "PLW2901", # `for` loop variable `{var}` overwritten by assignment target -- Inconvenient to enforce + "TRY002", # Create your own exception -- Inconvenient to enforce + "TRY003", # Avoid specifying long messages outside the exception class -- Inconvenient to enforce +] +select = [ + "F", # Pyflakes + "E", # pycodestyle Errors + "W", # pycodestyle Warnings + "C90", # mccabe + "I", # isort + "D", # pydocstyle + "UP", # pyupgrade + "B", # flake8-bugbear + "INP", # flake8-no-pep420 + "RUF", # Ruff-specific rules + "YTT", # flake8-2020 + "ASYNC", # flake8-async + "BLE", # flake8-blind-except + "COM", # flake8-commas + "C4", # flake8-comprehensions + "T10", # flake8-debugger + "EM", # flake8-errmsg + "FA", # flake8-future-annotations + "ISC", # flake8-implicit-str-concat + "ICN", # flake8-import-conventions + "INP", # flake8-no-pep420 + "PIE", # flake8-pie + "PT", # flake8-pytest-style + "Q", # flake8-quotes + "RSE", # flake8-raise + "RET", # flake8-return + "SLOT", # flake8-slots + "SIM", # flake8-simplify + "TCH", # flake8-type-checking, configure correctly and add + "PTH", # flake8-use-pathlib + "PGH", # pygrep-hooks + "PL", # Pylint + "TRY", # tryceratops + # "FURB", # refurb, add when out of nursery + # "LOG", # flake8-logging, add when out of nursery +] +output-format = "grouped" +show-fixes = true +task-tags = ["TODO", "HACK", "FIXME", "XXX"] +target-version = "py310" +exclude = ["tests/*snapshots/*"] + +[tool.ruff.extend-per-file-ignores] +"tests/*/__init__.py" = ["F401"] + +[tool.ruff.isort] +split-on-trailing-comma = false + +[tool.ruff.flake8-bugbear] +extend-immutable-calls = ["typer.Argument"] + +[tool.ruff.flake8-type-checking] +runtime-evaluated-base-classes = ["pydantic.BaseModel"] + [build-system] requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 89429d3e0..000000000 --- a/setup.cfg +++ /dev/null @@ -1,19 +0,0 @@ -[flake8] -exclude = - .git, - __pycache__ -max-complexity = 10 -# black docs regarding flake8: https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html#flake8 -# black enforces an equal amount of whitespace around slice operators. It is not PEP8 compliant. -# black and flake8 also disagree on line length -extend-ignore = - # E203: Whitespace before ':' - E203, - # E501: Line too long - E501, -per-file-ignores = - # F401: unused imports - tests/*/__init__.py: F401 - -[isort] -profile = black diff --git a/tests/cli/resources/empty_module.py b/tests/cli/resources/empty_module.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/cli/snapshots/snap_test_schema_generation.py b/tests/cli/snapshots/snap_test_schema_generation.py index 2a19e65c1..2dd92b512 100644 --- a/tests/cli/snapshots/snap_test_schema_generation.py +++ b/tests/cli/snapshots/snap_test_schema_generation.py @@ -58,7 +58,7 @@ }, "FromSection": { "additionalProperties": false, - "description": "Holds multiple input topics", + "description": "Holds multiple input topics.", "properties": { "components": { "additionalProperties": { @@ -84,7 +84,7 @@ }, "FromTopic": { "additionalProperties": false, - "description": "Input topic", + "description": "Input topic.", "properties": { "role": { "description": "Custom identifier belonging to a topic; define only if `type` is `pattern` or `None`", @@ -104,7 +104,7 @@ "type": "object" }, "InputTopicTypes": { - "description": "Input topic types\\n\\nINPUT (input topic), PATTERN (extra-topic-pattern or input-topic-pattern)", + "description": "Input topic types.\\n\\nINPUT (input topic), PATTERN (extra-topic-pattern or input-topic-pattern)", "enum": [ "input", "pattern" @@ -113,7 +113,7 @@ "type": "string" }, "OutputTopicTypes": { - "description": "Types of output topic\\n\\nOUTPUT (output topic), ERROR (error topic)", + "description": "Types of output topic.\\n\\nOUTPUT (output topic), ERROR (error topic)", "enum": [ "output", "error" @@ -216,7 +216,7 @@ "type": "object" }, "SubPipelineComponentCorrectDocstr": { - "description": "Newline before title is removed\\nSummarry is correctly imported. All whitespaces are removed and replaced with a single space. The description extraction terminates at the correct place, deletes 1 trailing coma", + "description": "Newline before title is removed.\\nSummarry is correctly imported. All whitespaces are removed and replaced with a single space. The description extraction terminates at the correct place, deletes 1 trailing coma", "properties": { "example_attr": { "description": "Parameter description looks correct and it is not included in the class description, terminates here", @@ -254,7 +254,7 @@ }, "type": { "default": "sub-pipeline-component-correct-docstr", - "description": "Newline before title is removed\\nSummarry is correctly imported. All whitespaces are removed and replaced with a single space. The description extraction terminates at the correct place, deletes 1 trailing coma", + "description": "Newline before title is removed.\\nSummarry is correctly imported. All whitespaces are removed and replaced with a single space. The description extraction terminates at the correct place, deletes 1 trailing coma", "enum": [ "sub-pipeline-component-correct-docstr" ], @@ -317,7 +317,7 @@ "type": "object" }, "ToSection": { - "description": "Holds multiple output topics", + "description": "Holds multiple output topics.", "properties": { "models": { "additionalProperties": { @@ -343,7 +343,7 @@ }, "TopicConfig": { "additionalProperties": false, - "description": "Configure an output topic", + "description": "Configure an output topic.", "properties": { "configs": { "additionalProperties": { diff --git a/tests/cli/test_pipeline_steps.py b/tests/cli/test_pipeline_steps.py index ac310a51a..a09d7b064 100644 --- a/tests/cli/test_pipeline_steps.py +++ b/tests/cli/test_pipeline_steps.py @@ -18,9 +18,9 @@ class TestComponent: prefix: str = PREFIX -test_component_1 = TestComponent(PREFIX + "example1") -test_component_2 = TestComponent(PREFIX + "example2") -test_component_3 = TestComponent(PREFIX + "example3") +test_component_1 = TestComponent("example1") +test_component_2 = TestComponent("example2") +test_component_3 = TestComponent("example3") @pytest.fixture(autouse=True) diff --git a/tests/cli/test_schema_generation.py b/tests/cli/test_schema_generation.py index 493db4c63..cbb855d14 100644 --- a/tests/cli/test_schema_generation.py +++ b/tests/cli/test_schema_generation.py @@ -1,16 +1,21 @@ from __future__ import annotations import logging +from abc import ABC, abstractmethod from pathlib import Path +from typing import TYPE_CHECKING import pytest from pydantic import Field -from snapshottest.module import SnapshotTest from typer.testing import CliRunner from kpops.cli.main import app from kpops.components.base_components import PipelineComponent from kpops.utils.docstring import describe_attr +from tests.cli.resources import empty_module + +if TYPE_CHECKING: + from snapshottest.module import SnapshotTest RESOURCE_PATH = Path(__file__).parent / "resources" @@ -24,6 +29,18 @@ class Config: anystr_strip_whitespace = True +# abstract component inheriting from ABC should be excluded +class AbstractBaseComponent(PipelineComponent, ABC): + ... + + +# abstract component with abstractmethods should be excluded +class AbstractPipelineComponent(AbstractBaseComponent): + @abstractmethod + def not_implemented(self) -> None: + ... + + class SubPipelineComponent(EmptyPipelineComponent): ... @@ -40,8 +57,7 @@ class SubPipelineComponentCorrect(SubPipelineComponent): # Correctly defined, docstr test class SubPipelineComponentCorrectDocstr(SubPipelineComponent): - """ - Newline before title is removed + """Newline before title is removed. Summarry is correctly imported. All @@ -92,6 +108,19 @@ def test_gen_pipeline_schema_no_modules(self, caplog: pytest.LogCaptureFixture): ] assert result.exit_code == 0 + def test_gen_pipeline_schema_no_components(self): + with pytest.raises(RuntimeError, match="^No valid components found.$"): + runner.invoke( + app, + [ + "schema", + "pipeline", + "--no-include-stock-components", + empty_module.__name__, + ], + catch_exceptions=False, + ) + def test_gen_pipeline_schema_only_stock_module(self): result = runner.invoke( app, @@ -121,7 +150,12 @@ def test_gen_pipeline_schema_only_stock_module(self): def test_gen_pipeline_schema_only_custom_module(self, snapshot: SnapshotTest): result = runner.invoke( app, - ["schema", "pipeline", MODULE, "--no-include-stock-components"], + [ + "schema", + "pipeline", + MODULE, + "--no-include-stock-components", + ], catch_exceptions=False, ) diff --git a/tests/compiler/test_pipeline_name.py b/tests/compiler/test_pipeline_name.py index 7a07c1a12..f0a1b1b1e 100644 --- a/tests/compiler/test_pipeline_name.py +++ b/tests/compiler/test_pipeline_name.py @@ -8,49 +8,51 @@ DEFAULTS_PATH = Path(__file__).parent / "resources" PIPELINE_PATH = Path("./some/random/path/for/testing/pipeline.yaml") -DEFAULT_BASE_DIR = Path(".") +DEFAULT_BASE_DIR = Path() def test_should_set_pipeline_name_with_default_base_dir(): Pipeline.set_pipeline_name_env_vars(DEFAULT_BASE_DIR, PIPELINE_PATH) - assert "some-random-path-for-testing" == ENV["pipeline_name"] - assert "some" == ENV["pipeline_name_0"] - assert "random" == ENV["pipeline_name_1"] - assert "path" == ENV["pipeline_name_2"] - assert "for" == ENV["pipeline_name_3"] - assert "testing" == ENV["pipeline_name_4"] + assert ENV["pipeline_name"] == "some-random-path-for-testing" + assert ENV["pipeline_name_0"] == "some" + assert ENV["pipeline_name_1"] == "random" + assert ENV["pipeline_name_2"] == "path" + assert ENV["pipeline_name_3"] == "for" + assert ENV["pipeline_name_4"] == "testing" def test_should_set_pipeline_name_with_specific_relative_base_dir(): Pipeline.set_pipeline_name_env_vars(Path("./some/random/path"), PIPELINE_PATH) - assert "for-testing" == ENV["pipeline_name"] - assert "for" == ENV["pipeline_name_0"] - assert "testing" == ENV["pipeline_name_1"] + assert ENV["pipeline_name"] == "for-testing" + assert ENV["pipeline_name_0"] == "for" + assert ENV["pipeline_name_1"] == "testing" def test_should_set_pipeline_name_with_specific_absolute_base_dir(): Pipeline.set_pipeline_name_env_vars(Path("some/random/path"), PIPELINE_PATH) - assert "for-testing" == ENV["pipeline_name"] - assert "for" == ENV["pipeline_name_0"] - assert "testing" == ENV["pipeline_name_1"] + assert ENV["pipeline_name"] == "for-testing" + assert ENV["pipeline_name_0"] == "for" + assert ENV["pipeline_name_1"] == "testing" def test_should_set_pipeline_name_with_absolute_base_dir(): Pipeline.set_pipeline_name_env_vars(Path.cwd(), PIPELINE_PATH) - assert "some-random-path-for-testing" == ENV["pipeline_name"] - assert "some" == ENV["pipeline_name_0"] - assert "random" == ENV["pipeline_name_1"] - assert "path" == ENV["pipeline_name_2"] - assert "for" == ENV["pipeline_name_3"] - assert "testing" == ENV["pipeline_name_4"] + assert ENV["pipeline_name"] == "some-random-path-for-testing" + assert ENV["pipeline_name_0"] == "some" + assert ENV["pipeline_name_1"] == "random" + assert ENV["pipeline_name_2"] == "path" + assert ENV["pipeline_name_3"] == "for" + assert ENV["pipeline_name_4"] == "testing" def test_should_not_set_pipeline_name_with_the_same_base_dir(): - with pytest.raises(ValueError): + with pytest.raises( + ValueError, match="The pipeline-base-dir should not equal the pipeline-path" + ): Pipeline.set_pipeline_name_env_vars(PIPELINE_PATH, PIPELINE_PATH) diff --git a/tests/component_handlers/helm_wrapper/test_dry_run_handler.py b/tests/component_handlers/helm_wrapper/test_dry_run_handler.py index 20c02f50d..bad4f2aa8 100644 --- a/tests/component_handlers/helm_wrapper/test_dry_run_handler.py +++ b/tests/component_handlers/helm_wrapper/test_dry_run_handler.py @@ -12,13 +12,13 @@ class TestDryRunHandler: - @pytest.fixture + @pytest.fixture() def helm_mock(self, mocker: MockerFixture) -> MagicMock: return mocker.patch( "kpops.component_handlers.helm_wrapper.dry_run_handler.Helm" ).return_value - @pytest.fixture + @pytest.fixture() def helm_diff_mock(self, mocker: MockerFixture) -> MagicMock: return mocker.patch( "kpops.component_handlers.helm_wrapper.dry_run_handler.HelmDiff" diff --git a/tests/component_handlers/helm_wrapper/test_helm_wrapper.py b/tests/component_handlers/helm_wrapper/test_helm_wrapper.py index de23dca8e..ce6fae709 100644 --- a/tests/component_handlers/helm_wrapper/test_helm_wrapper.py +++ b/tests/component_handlers/helm_wrapper/test_helm_wrapper.py @@ -29,15 +29,15 @@ def temp_file_mock(self, mocker: MockerFixture) -> MagicMock: temp_file_mock.return_value.__enter__.return_value.name = "values.yaml" return temp_file_mock - @pytest.fixture + @pytest.fixture() def run_command(self, mocker: MockerFixture) -> MagicMock: return mocker.patch.object(Helm, "_Helm__execute") - @pytest.fixture + @pytest.fixture() def log_warning_mock(self, mocker: MockerFixture) -> MagicMock: return mocker.patch("kpops.component_handlers.helm_wrapper.helm.log.warning") - @pytest.fixture + @pytest.fixture() def mock_get_version(self, mocker: MockerFixture) -> MagicMock: mock_get_version = mocker.patch.object(Helm, "get_version") mock_get_version.return_value = Version(major=3, minor=12, patch=0) @@ -337,8 +337,7 @@ def test_raise_parse_error_when_helm_content_is_invalid(self): """ ) with pytest.raises(ParseError, match="Not a valid Helm template source"): - helm_template = list(Helm.load_manifest(stdout)) - assert len(helm_template) == 0 + list(Helm.load_manifest(stdout)) def test_load_manifest(self): stdout = dedent( @@ -498,7 +497,7 @@ def test_should_call_run_command_method_when_helm_template_without_optional_args ) @pytest.mark.parametrize( - "raw_version, expected_version", + ("raw_version", "expected_version"), [ ("v3.12.0+gc9f554d", Version(3, 12, 0)), ("v3.12.0", Version(3, 12, 0)), diff --git a/tests/component_handlers/kafka_connect/test_connect_handler.py b/tests/component_handlers/kafka_connect/test_connect_handler.py index 907065c5e..db64690e9 100644 --- a/tests/component_handlers/kafka_connect/test_connect_handler.py +++ b/tests/component_handlers/kafka_connect/test_connect_handler.py @@ -22,25 +22,25 @@ class TestConnectorHandler: - @pytest.fixture + @pytest.fixture() def log_info_mock(self, mocker: MockerFixture) -> MagicMock: return mocker.patch( "kpops.component_handlers.kafka_connect.kafka_connect_handler.log.info" ) - @pytest.fixture + @pytest.fixture() def log_warning_mock(self, mocker: MockerFixture) -> MagicMock: return mocker.patch( "kpops.component_handlers.kafka_connect.kafka_connect_handler.log.warning" ) - @pytest.fixture + @pytest.fixture() def log_error_mock(self, mocker: MockerFixture) -> MagicMock: return mocker.patch( "kpops.component_handlers.kafka_connect.kafka_connect_handler.log.error" ) - @pytest.fixture + @pytest.fixture() def renderer_diff_mock(self, mocker: MockerFixture) -> MagicMock: return mocker.patch( "kpops.component_handlers.kafka_connect.kafka_connect_handler.render_diff" @@ -53,7 +53,7 @@ def connector_handler(connect_wrapper: MagicMock) -> KafkaConnectHandler: timeout=0, ) - @pytest.fixture + @pytest.fixture() def connector_config(self) -> KafkaConnectorConfig: return KafkaConnectorConfig( **{ @@ -250,6 +250,7 @@ def test_should_call_delete_connector_when_destroying_existing_connector_not_dry handler = self.connector_handler(connector_wrapper) handler.destroy_connector(CONNECTOR_NAME, dry_run=False) + assert connector_wrapper.mock_calls == [ mock.call.get_connector(CONNECTOR_NAME), mock.call.delete_connector(CONNECTOR_NAME), diff --git a/tests/component_handlers/kafka_connect/test_connect_wrapper.py b/tests/component_handlers/kafka_connect/test_connect_wrapper.py index 3db9c090f..8e60d92a7 100644 --- a/tests/component_handlers/kafka_connect/test_connect_wrapper.py +++ b/tests/component_handlers/kafka_connect/test_connect_wrapper.py @@ -26,7 +26,7 @@ class TestConnectorApiWrapper: @pytest.fixture(autouse=True) - def setup(self): + def _setup(self): config = PipelineConfig( defaults_path=DEFAULTS_PATH, environment="development", @@ -34,7 +34,7 @@ def setup(self): ) self.connect_wrapper = ConnectWrapper(host=config.kafka_connect_host) - @pytest.fixture + @pytest.fixture() def connector_config(self) -> KafkaConnectorConfig: return KafkaConnectorConfig( **{ @@ -495,9 +495,9 @@ def test_should_create_correct_validate_connector_config_and_name_gets_added( ) def test_should_parse_validate_connector_config(self, httpx_mock: HTTPXMock): - with open( + with Path( DEFAULTS_PATH / "connect_validation_response.json", - ) as f: + ).open() as f: actual_response = json.load(f) httpx_mock.add_response( method="PUT", diff --git a/tests/component_handlers/schema_handler/test_schema_handler.py b/tests/component_handlers/schema_handler/test_schema_handler.py index ccf75fb61..65ceaae8f 100644 --- a/tests/component_handlers/schema_handler/test_schema_handler.py +++ b/tests/component_handlers/schema_handler/test_schema_handler.py @@ -116,45 +116,48 @@ def test_should_raise_value_error_if_schema_provider_class_not_found(): url="http://mock:8081", components_module=NON_EXISTING_PROVIDER_MODULE ) - with pytest.raises(ValueError) as value_error: + with pytest.raises( + ValueError, + match="No schema provider found in components module pydantic.main. " + "Please implement the abstract method in " + f"{SchemaProvider.__module__}.{SchemaProvider.__name__}.", + ): schema_handler.schema_provider.provide_schema( "com.bakdata.kpops.test.SchemaHandlerTest", {} ) - assert ( - str(value_error.value) - == "No schema provider found in components module pydantic.main. " - "Please implement the abstract method in " - f"{SchemaProvider.__module__}.{SchemaProvider.__name__}." - ) - -def test_should_raise_value_error_when_schema_provider_is_called_and_components_module_is_empty(): +@pytest.mark.parametrize( + ("components_module"), + [ + pytest.param( + None, + id="components_module = None", + ), + pytest.param( + "", + id="components_module = ''", + ), + ], +) +def test_should_raise_value_error_when_schema_provider_is_called_and_components_module_is_empty( + components_module: str, +): config_enable = PipelineConfig( defaults_path=Path("fake"), environment="development", schema_registry_url="http://localhost:8081", ) - - with pytest.raises(ValueError): - schema_handler = SchemaHandler.load_schema_handler(None, config_enable) - assert schema_handler is not None - schema_handler.schema_provider.provide_schema( - "com.bakdata.kpops.test.SchemaHandlerTest", {} - ) - - with pytest.raises(ValueError) as value_error: - schema_handler = SchemaHandler.load_schema_handler("", config_enable) - assert schema_handler is not None + schema_handler = SchemaHandler.load_schema_handler(components_module, config_enable) + assert schema_handler is not None + with pytest.raises( + ValueError, + match="The Schema Registry URL is set but you haven't specified the component module path. Please provide a valid component module path where your SchemaProvider implementation exists.", + ): schema_handler.schema_provider.provide_schema( "com.bakdata.kpops.test.SchemaHandlerTest", {} ) - assert ( - str(value_error.value) - == "The Schema Registry URL is set but you haven't specified the component module path. Please provide a valid component module path where your SchemaProvider implementation exists." - ) - def test_should_log_info_when_submit_schemas_that_not_exists_and_dry_run_true( to_section: ToSection, log_info_mock: MagicMock, schema_registry_mock: MagicMock @@ -210,10 +213,9 @@ def test_should_raise_exception_when_submit_schema_that_exists_and_not_compatibl schema_registry_mock.check_version.return_value = None schema_registry_mock.test_compatibility.return_value = False - with pytest.raises(Exception) as exception: + with pytest.raises(Exception, match="Schema is not compatible for") as exception: schema_handler.submit_schemas(to_section, True) - assert "Schema is not compatible for" in str(exception.value) EXPECTED_SCHEMA = { "type": "record", "name": "KPOps.Employee", diff --git a/tests/component_handlers/topic/test_proxy_wrapper.py b/tests/component_handlers/topic/test_proxy_wrapper.py index 7b587ecb3..e26fb0e5a 100644 --- a/tests/component_handlers/topic/test_proxy_wrapper.py +++ b/tests/component_handlers/topic/test_proxy_wrapper.py @@ -30,15 +30,15 @@ def log_debug_mock(self, mocker: MockerFixture) -> MagicMock: return mocker.patch("kpops.component_handlers.topic.proxy_wrapper.log.debug") @pytest.fixture(autouse=True) - def setup(self, httpx_mock: HTTPXMock): + def _setup(self, httpx_mock: HTTPXMock): config = PipelineConfig( defaults_path=DEFAULTS_PATH, environment="development", kafka_rest_host=HOST ) self.proxy_wrapper = ProxyWrapper(pipeline_config=config) - with open( - DEFAULTS_PATH / "kafka_rest_proxy_responses" / "cluster-info.json" - ) as f: + with Path( + DEFAULTS_PATH / "kafka_rest_proxy_responses" / "cluster-info.json", + ).open() as f: cluster_response = json.load(f) httpx_mock.add_response( @@ -53,12 +53,11 @@ def setup(self, httpx_mock: HTTPXMock): def test_should_raise_exception_when_host_is_not_set(self): config = PipelineConfig(defaults_path=DEFAULTS_PATH, environment="development") config.kafka_rest_host = None - with pytest.raises(ValueError) as exception: + with pytest.raises( + ValueError, + match="The Kafka REST Proxy host is not set. Please set the host in the config.yaml using the kafka_rest_host property or set the environemt variable KPOPS_REST_PROXY_HOST.", + ): ProxyWrapper(pipeline_config=config) - assert ( - str(exception.value) - == "The Kafka REST Proxy host is not set. Please set the host in the config.yaml using the kafka_rest_host property or set the environemt variable KPOPS_REST_PROXY_HOST." - ) @patch("httpx.post") def test_should_create_topic_with_all_topic_configuration( diff --git a/tests/component_handlers/topic/test_topic_handler.py b/tests/component_handlers/topic/test_topic_handler.py index c53a7a60d..6b1b017fc 100644 --- a/tests/component_handlers/topic/test_topic_handler.py +++ b/tests/component_handlers/topic/test_topic_handler.py @@ -51,19 +51,19 @@ def log_error_mock(self, mocker: MockerFixture) -> MagicMock: @pytest.fixture(autouse=True) def get_topic_response_mock(self) -> MagicMock: - with open( - DEFAULTS_PATH / "kafka_rest_proxy_responses/get_topic_response.json" - ) as f: + with Path( + DEFAULTS_PATH / "kafka_rest_proxy_responses/get_topic_response.json", + ).open() as f: response = json.load(f) - with open( - DEFAULTS_PATH / "kafka_rest_proxy_responses/broker_response.json" - ) as f: + with Path( + DEFAULTS_PATH / "kafka_rest_proxy_responses/broker_response.json", + ).open() as f: broker_response = json.load(f) - with open( - DEFAULTS_PATH / "kafka_rest_proxy_responses/topic_config_response.json" - ) as f: + with Path( + DEFAULTS_PATH / "kafka_rest_proxy_responses/topic_config_response.json", + ).open() as f: response_topic_config = json.load(f) wrapper = MagicMock() @@ -76,14 +76,15 @@ def get_topic_response_mock(self) -> MagicMock: @pytest.fixture(autouse=True) def get_default_topic_response_mock(self) -> MagicMock: - with open( - DEFAULTS_PATH / "kafka_rest_proxy_responses/get_default_topic_response.json" - ) as f: + with Path( + DEFAULTS_PATH + / "kafka_rest_proxy_responses/get_default_topic_response.json", + ).open() as f: response = json.load(f) - with open( - DEFAULTS_PATH / "kafka_rest_proxy_responses/broker_response.json" - ) as f: + with Path( + DEFAULTS_PATH / "kafka_rest_proxy_responses/broker_response.json", + ).open() as f: broker_response = json.load(f) wrapper = MagicMock() @@ -369,7 +370,7 @@ def test_should_exit_if_dry_run_and_topic_exists_different_partition_count( match="Topic Creation: partition count of topic topic-X changed! Partitions count of topic topic-X is 10. The given partitions count 200.", ): topic_handler.create_topics(to_section=to_section, dry_run=True) - wrapper.get_topic_config.assert_called_once() # dry run requests the config to create the diff + wrapper.get_topic_config.assert_called_once() # dry run requests the config to create the diff def test_should_exit_if_dry_run_and_topic_exists_different_replication_factor( self, get_topic_response_mock: MagicMock @@ -391,7 +392,7 @@ def test_should_exit_if_dry_run_and_topic_exists_different_replication_factor( match="Topic Creation: replication factor of topic topic-X changed! Replication factor of topic topic-X is 3. The given replication count 300.", ): topic_handler.create_topics(to_section=to_section, dry_run=True) - wrapper.get_topic_config.assert_called_once() # dry run requests the config to create the diff + wrapper.get_topic_config.assert_called_once() # dry run requests the config to create the diff def test_should_log_correct_message_when_delete_existing_topic_dry_run( self, log_info_mock: MagicMock, get_topic_response_mock: MagicMock diff --git a/tests/components/test_base_defaults_component.py b/tests/components/test_base_defaults_component.py index 7b25e5f74..d066d431b 100644 --- a/tests/components/test_base_defaults_component.py +++ b/tests/components/test_base_defaults_component.py @@ -37,7 +37,7 @@ class EnvVarTest(BaseDefaultsComponent): name: str | None = None -@pytest.fixture +@pytest.fixture() def config() -> PipelineConfig: return PipelineConfig( defaults_path=DEFAULTS_PATH, @@ -45,7 +45,7 @@ def config() -> PipelineConfig: ) -@pytest.fixture +@pytest.fixture() def handlers() -> ComponentHandlers: return ComponentHandlers( schema_handler=MagicMock(), diff --git a/tests/components/test_kafka_app.py b/tests/components/test_kafka_app.py index e9b94fe9c..8fd0d98ec 100644 --- a/tests/components/test_kafka_app.py +++ b/tests/components/test_kafka_app.py @@ -17,7 +17,7 @@ class TestKafkaApp: - @pytest.fixture + @pytest.fixture() def config(self) -> PipelineConfig: return PipelineConfig( defaults_path=DEFAULTS_PATH, @@ -25,7 +25,7 @@ def config(self) -> PipelineConfig: helm_diff_config=HelmDiffConfig(), ) - @pytest.fixture + @pytest.fixture() def handlers(self) -> ComponentHandlers: return ComponentHandlers( schema_handler=MagicMock(), @@ -93,7 +93,7 @@ def test_should_deploy_kafka_app( print_helm_diff.assert_called_once() helm_upgrade_install.assert_called_once_with( - "example-name", + "${pipeline_name}-example-name", "test/test-chart", True, "test-namespace", diff --git a/tests/components/test_kafka_connector.py b/tests/components/test_kafka_connector.py index 4e8424e5c..2adf867da 100644 --- a/tests/components/test_kafka_connector.py +++ b/tests/components/test_kafka_connector.py @@ -12,15 +12,13 @@ DEFAULTS_PATH = Path(__file__).parent / "resources" CONNECTOR_NAME = "test-connector-with-long-name-0123456789abcdefghijklmnop" -CONNECTOR_NAME_PREFIXED = ( - "${pipeline_name}-test-connector-with-long-name-0123456789abcdefghijklmnop" -) -CONNECTOR_CLEAN_NAME = "test-connector-with-long-name-0123456789abcdef-clean" +CONNECTOR_FULL_NAME = "${pipeline_name}-" + CONNECTOR_NAME +CONNECTOR_CLEAN_FULL_NAME = "${pipeline_name}-test-connector-with-long-name-clean" CONNECTOR_CLASS = "com.bakdata.connect.TestConnector" class TestKafkaConnector: - @pytest.fixture + @pytest.fixture() def config(self) -> PipelineConfig: return PipelineConfig( defaults_path=DEFAULTS_PATH, @@ -33,7 +31,7 @@ def config(self) -> PipelineConfig: helm_diff_config=HelmDiffConfig(), ) - @pytest.fixture + @pytest.fixture() def handlers(self) -> ComponentHandlers: return ComponentHandlers( schema_handler=MagicMock(), @@ -47,18 +45,18 @@ def helm_mock(self, mocker: MockerFixture) -> MagicMock: "kpops.components.base_components.kafka_connector.Helm" ).return_value - @pytest.fixture + @pytest.fixture() def dry_run_handler(self, mocker: MockerFixture) -> MagicMock: return mocker.patch( "kpops.components.base_components.kafka_connector.DryRunHandler" ).return_value - @pytest.fixture + @pytest.fixture() def connector_config(self) -> KafkaConnectorConfig: return KafkaConnectorConfig( **{ "connector.class": CONNECTOR_CLASS, - "name": CONNECTOR_NAME_PREFIXED, + "name": CONNECTOR_FULL_NAME, } ) @@ -75,16 +73,16 @@ def test_connector_config_name_override( app=connector_config, namespace="test-namespace", ) - assert connector.app.name == CONNECTOR_NAME_PREFIXED + assert connector.app.name == CONNECTOR_FULL_NAME connector = KafkaConnector( name=CONNECTOR_NAME, config=config, handlers=handlers, - app={"connector.class": CONNECTOR_CLASS}, # type: ignore + app={"connector.class": CONNECTOR_CLASS}, # type: ignore[reportGeneralTypeIssues] namespace="test-namespace", ) - assert connector.app.name == CONNECTOR_NAME_PREFIXED + assert connector.app.name == CONNECTOR_FULL_NAME with pytest.raises( ValueError, match="Connector name should be the same as component name" @@ -93,7 +91,7 @@ def test_connector_config_name_override( name=CONNECTOR_NAME, config=config, handlers=handlers, - app={"connector.class": CONNECTOR_CLASS, "name": "different-name"}, # type: ignore + app={"connector.class": CONNECTOR_CLASS, "name": "different-name"}, # type: ignore[reportGeneralTypeIssues] namespace="test-namespace", ) @@ -104,6 +102,6 @@ def test_connector_config_name_override( name=CONNECTOR_NAME, config=config, handlers=handlers, - app={"connector.class": CONNECTOR_CLASS, "name": ""}, # type: ignore + app={"connector.class": CONNECTOR_CLASS, "name": ""}, # type: ignore[reportGeneralTypeIssues] namespace="test-namespace", ) diff --git a/tests/components/test_kafka_sink_connector.py b/tests/components/test_kafka_sink_connector.py index 5de354739..e8ed7aa22 100644 --- a/tests/components/test_kafka_sink_connector.py +++ b/tests/components/test_kafka_sink_connector.py @@ -27,18 +27,19 @@ ) from kpops.utils.colorify import magentaify from tests.components.test_kafka_connector import ( - CONNECTOR_CLEAN_NAME, + CONNECTOR_CLEAN_FULL_NAME, + CONNECTOR_FULL_NAME, CONNECTOR_NAME, TestKafkaConnector, ) class TestKafkaSinkConnector(TestKafkaConnector): - @pytest.fixture + @pytest.fixture() def log_info_mock(self, mocker: MockerFixture) -> MagicMock: return mocker.patch("kpops.components.base_components.kafka_connector.log.info") - @pytest.fixture + @pytest.fixture() def connector( self, config: PipelineConfig, @@ -168,7 +169,9 @@ def test_destroy( connector.destroy(dry_run=True) - mock_destroy_connector.assert_called_once_with(CONNECTOR_NAME, dry_run=True) + mock_destroy_connector.assert_called_once_with( + CONNECTOR_FULL_NAME, dry_run=True + ) def test_reset_when_dry_run_is_true( self, @@ -208,11 +211,11 @@ def test_reset_when_dry_run_is_false( ), mocker.call.helm.uninstall( namespace="test-namespace", - release_name=CONNECTOR_CLEAN_NAME, + release_name=CONNECTOR_CLEAN_FULL_NAME, dry_run=dry_run, ), mocker.call.helm.upgrade_install( - release_name=CONNECTOR_CLEAN_NAME, + release_name=CONNECTOR_CLEAN_FULL_NAME, namespace="test-namespace", chart="bakdata-kafka-connect-resetter/kafka-connect-resetter", dry_run=dry_run, @@ -225,15 +228,15 @@ def test_reset_when_dry_run_is_false( "connectorType": "sink", "config": { "brokers": "broker:9092", - "connector": CONNECTOR_NAME, + "connector": CONNECTOR_FULL_NAME, "deleteConsumerGroup": False, }, - "nameOverride": CONNECTOR_NAME, + "nameOverride": CONNECTOR_FULL_NAME, }, ), mocker.call.helm.uninstall( namespace="test-namespace", - release_name=CONNECTOR_CLEAN_NAME, + release_name=CONNECTOR_CLEAN_FULL_NAME, dry_run=dry_run, ), ] @@ -259,23 +262,7 @@ def test_clean_when_dry_run_is_false( log_info_mock: MagicMock, dry_run_handler: MagicMock, mocker: MockerFixture, - connector_config: KafkaConnectorConfig, ): - connector = KafkaSinkConnector( - name=CONNECTOR_NAME, - config=config, - handlers=handlers, - app=connector_config, - namespace="test-namespace", - to=ToSection( - topics={ - TopicName("${output_topic_name}"): TopicConfig( - type=OutputTopicTypes.OUTPUT, partitions_count=10 - ), - } - ), - ) - mock_delete_topics = mocker.patch.object( connector.handlers.topic_handler, "delete_topics" ) @@ -294,12 +281,12 @@ def test_clean_when_dry_run_is_false( assert log_info_mock.mock_calls == [ call.log_info( magentaify( - f"Connector Cleanup: uninstalling cleanup job Helm release from previous runs for {CONNECTOR_NAME}" + f"Connector Cleanup: uninstalling cleanup job Helm release from previous runs for {CONNECTOR_FULL_NAME}" ) ), call.log_info( magentaify( - f"Connector Cleanup: deploy Connect {KafkaConnectorType.SINK.value} resetter for {CONNECTOR_NAME}" + f"Connector Cleanup: deploy Connect {KafkaConnectorType.SINK.value} resetter for {CONNECTOR_FULL_NAME}" ) ), call.log_info(magentaify("Connector Cleanup: uninstall Kafka Resetter.")), @@ -314,11 +301,11 @@ def test_clean_when_dry_run_is_false( ), mocker.call.helm.uninstall( namespace="test-namespace", - release_name=CONNECTOR_CLEAN_NAME, + release_name=CONNECTOR_CLEAN_FULL_NAME, dry_run=dry_run, ), mocker.call.helm.upgrade_install( - release_name=CONNECTOR_CLEAN_NAME, + release_name=CONNECTOR_CLEAN_FULL_NAME, namespace="test-namespace", chart="bakdata-kafka-connect-resetter/kafka-connect-resetter", dry_run=dry_run, @@ -331,15 +318,15 @@ def test_clean_when_dry_run_is_false( "connectorType": "sink", "config": { "brokers": "broker:9092", - "connector": CONNECTOR_NAME, + "connector": CONNECTOR_FULL_NAME, "deleteConsumerGroup": True, }, - "nameOverride": CONNECTOR_NAME, + "nameOverride": CONNECTOR_FULL_NAME, }, ), mocker.call.helm.uninstall( namespace="test-namespace", - release_name=CONNECTOR_CLEAN_NAME, + release_name=CONNECTOR_CLEAN_FULL_NAME, dry_run=dry_run, ), ] @@ -408,11 +395,11 @@ def test_clean_without_to_when_dry_run_is_false( ), mocker.call.helm.uninstall( namespace="test-namespace", - release_name=CONNECTOR_CLEAN_NAME, + release_name=CONNECTOR_CLEAN_FULL_NAME, dry_run=dry_run, ), mocker.call.helm.upgrade_install( - release_name=CONNECTOR_CLEAN_NAME, + release_name=CONNECTOR_CLEAN_FULL_NAME, namespace="test-namespace", chart="bakdata-kafka-connect-resetter/kafka-connect-resetter", dry_run=dry_run, @@ -425,15 +412,15 @@ def test_clean_without_to_when_dry_run_is_false( "connectorType": "sink", "config": { "brokers": "broker:9092", - "connector": CONNECTOR_NAME, + "connector": CONNECTOR_FULL_NAME, "deleteConsumerGroup": True, }, - "nameOverride": CONNECTOR_NAME, + "nameOverride": CONNECTOR_FULL_NAME, }, ), mocker.call.helm.uninstall( namespace="test-namespace", - release_name=CONNECTOR_CLEAN_NAME, + release_name=CONNECTOR_CLEAN_FULL_NAME, dry_run=dry_run, ), ] diff --git a/tests/components/test_kafka_source_connector.py b/tests/components/test_kafka_source_connector.py index 72c487e74..169111ed3 100644 --- a/tests/components/test_kafka_source_connector.py +++ b/tests/components/test_kafka_source_connector.py @@ -24,14 +24,15 @@ ) from kpops.utils.environment import ENV from tests.components.test_kafka_connector import ( - CONNECTOR_CLEAN_NAME, + CONNECTOR_CLEAN_FULL_NAME, + CONNECTOR_FULL_NAME, CONNECTOR_NAME, TestKafkaConnector, ) class TestKafkaSourceConnector(TestKafkaConnector): - @pytest.fixture + @pytest.fixture() def connector( self, config: PipelineConfig, @@ -112,7 +113,9 @@ def test_destroy( connector.destroy(dry_run=True) - mock_destroy_connector.assert_called_once_with(CONNECTOR_NAME, dry_run=True) + mock_destroy_connector.assert_called_once_with( + CONNECTOR_FULL_NAME, dry_run=True + ) def test_reset_when_dry_run_is_true( self, @@ -154,11 +157,11 @@ def test_reset_when_dry_run_is_false( ), mocker.call.helm.uninstall( namespace="test-namespace", - release_name=CONNECTOR_CLEAN_NAME, + release_name=CONNECTOR_CLEAN_FULL_NAME, dry_run=False, ), mocker.call.helm.upgrade_install( - release_name=CONNECTOR_CLEAN_NAME, + release_name=CONNECTOR_CLEAN_FULL_NAME, namespace="test-namespace", chart="bakdata-kafka-connect-resetter/kafka-connect-resetter", dry_run=False, @@ -171,15 +174,15 @@ def test_reset_when_dry_run_is_false( "connectorType": "source", "config": { "brokers": "broker:9092", - "connector": CONNECTOR_NAME, + "connector": CONNECTOR_FULL_NAME, "offsetTopic": "kafka-connect-offsets", }, - "nameOverride": CONNECTOR_NAME, + "nameOverride": CONNECTOR_FULL_NAME, }, ), mocker.call.helm.uninstall( namespace="test-namespace", - release_name="test-connector-with-long-name-0123456789abcdef-clean", + release_name=CONNECTOR_CLEAN_FULL_NAME, dry_run=False, ), ] @@ -229,11 +232,11 @@ def test_clean_when_dry_run_is_false( ), mocker.call.helm.uninstall( namespace="test-namespace", - release_name=CONNECTOR_CLEAN_NAME, + release_name=CONNECTOR_CLEAN_FULL_NAME, dry_run=False, ), mocker.call.helm.upgrade_install( - release_name=CONNECTOR_CLEAN_NAME, + release_name=CONNECTOR_CLEAN_FULL_NAME, namespace="test-namespace", chart="bakdata-kafka-connect-resetter/kafka-connect-resetter", dry_run=False, @@ -246,15 +249,15 @@ def test_clean_when_dry_run_is_false( "connectorType": "source", "config": { "brokers": "broker:9092", - "connector": CONNECTOR_NAME, + "connector": CONNECTOR_FULL_NAME, "offsetTopic": "kafka-connect-offsets", }, - "nameOverride": CONNECTOR_NAME, + "nameOverride": CONNECTOR_FULL_NAME, }, ), mocker.call.helm.uninstall( namespace="test-namespace", - release_name=CONNECTOR_CLEAN_NAME, + release_name=CONNECTOR_CLEAN_FULL_NAME, dry_run=False, ), ] @@ -304,11 +307,11 @@ def test_clean_without_to_when_dry_run_is_false( ), mocker.call.helm.uninstall( namespace="test-namespace", - release_name=CONNECTOR_CLEAN_NAME, + release_name=CONNECTOR_CLEAN_FULL_NAME, dry_run=False, ), mocker.call.helm.upgrade_install( - release_name=CONNECTOR_CLEAN_NAME, + release_name=CONNECTOR_CLEAN_FULL_NAME, namespace="test-namespace", chart="bakdata-kafka-connect-resetter/kafka-connect-resetter", dry_run=False, @@ -321,15 +324,15 @@ def test_clean_without_to_when_dry_run_is_false( "connectorType": "source", "config": { "brokers": "broker:9092", - "connector": CONNECTOR_NAME, + "connector": CONNECTOR_FULL_NAME, "offsetTopic": "kafka-connect-offsets", }, - "nameOverride": CONNECTOR_NAME, + "nameOverride": CONNECTOR_FULL_NAME, }, ), mocker.call.helm.uninstall( namespace="test-namespace", - release_name=CONNECTOR_CLEAN_NAME, + release_name=CONNECTOR_CLEAN_FULL_NAME, dry_run=False, ), ] diff --git a/tests/components/test_kubernetes_app.py b/tests/components/test_kubernetes_app.py index 331444706..6583ac4bf 100644 --- a/tests/components/test_kubernetes_app.py +++ b/tests/components/test_kubernetes_app.py @@ -3,6 +3,7 @@ import pytest from pytest_mock import MockerFixture +from typing_extensions import override from kpops.cli.pipeline_config import PipelineConfig from kpops.component_handlers import ComponentHandlers @@ -26,7 +27,7 @@ class KubernetesTestValue(KubernetesAppConfig): class TestKubernetesApp: - @pytest.fixture + @pytest.fixture() def config(self) -> PipelineConfig: return PipelineConfig( defaults_path=DEFAULTS_PATH, @@ -34,7 +35,7 @@ def config(self) -> PipelineConfig: helm_diff_config=HelmDiffConfig(), ) - @pytest.fixture + @pytest.fixture() def handlers(self) -> ComponentHandlers: return ComponentHandlers( schema_handler=MagicMock(), @@ -42,25 +43,25 @@ def handlers(self) -> ComponentHandlers: topic_handler=MagicMock(), ) - @pytest.fixture + @pytest.fixture() def helm_mock(self, mocker: MockerFixture) -> MagicMock: return mocker.patch( "kpops.components.base_components.kubernetes_app.Helm" ).return_value - @pytest.fixture + @pytest.fixture() def log_info_mock(self, mocker: MockerFixture) -> MagicMock: return mocker.patch("kpops.components.base_components.kubernetes_app.log.info") - @pytest.fixture + @pytest.fixture() def app_value(self) -> KubernetesTestValue: return KubernetesTestValue(**{"name_override": "test-value"}) - @pytest.fixture + @pytest.fixture() def repo_config(self) -> HelmRepoConfig: return HelmRepoConfig(repository_name="test", url="https://bakdata.com") - @pytest.fixture + @pytest.fixture() def kubernetes_app( self, config: PipelineConfig, @@ -95,7 +96,7 @@ def test_should_lazy_load_helm_wrapper_and_not_repo_add( kubernetes_app.deploy(False) helm_mock.upgrade_install.assert_called_once_with( - "test-kubernetes-app", + "${pipeline_name}-test-kubernetes-app", "test/test-chart", False, "test-namespace", @@ -105,14 +106,15 @@ def test_should_lazy_load_helm_wrapper_and_not_repo_add( def test_should_lazy_load_helm_wrapper_and_call_repo_add_when_implemented( self, - kubernetes_app: KubernetesApp, config: PipelineConfig, handlers: ComponentHandlers, helm_mock: MagicMock, mocker: MockerFixture, app_value: KubernetesTestValue, ): - repo_config = HelmRepoConfig(repository_name="test-repo", url="mock://test") + repo_config = HelmRepoConfig( + repository_name="test-repo", url="https://test.com/charts/" + ) kubernetes_app = KubernetesApp( name="test-kubernetes-app", config=config, @@ -135,11 +137,11 @@ def test_should_lazy_load_helm_wrapper_and_call_repo_add_when_implemented( assert helm_mock.mock_calls == [ mocker.call.add_repo( "test-repo", - "mock://test", + "https://test.com/charts/", RepoAuthFlags(), ), mocker.call.upgrade_install( - "test-kubernetes-app", + "${pipeline_name}-test-kubernetes-app", "test/test-chart", False, "test-namespace", @@ -148,6 +150,42 @@ def test_should_lazy_load_helm_wrapper_and_call_repo_add_when_implemented( ), ] + def test_should_deploy_app_with_local_helm_chart( + self, + config: PipelineConfig, + handlers: ComponentHandlers, + helm_mock: MagicMock, + app_value: KubernetesTestValue, + ): + class AppWithLocalChart(KubernetesApp): + repo_config: None = None + + @property + @override + def helm_chart(self) -> str: + return "path/to/helm/charts/" + + app_with_local_chart = AppWithLocalChart( + name="test-app-with-local-chart", + config=config, + handlers=handlers, + app=app_value, + namespace="test-namespace", + ) + + app_with_local_chart.deploy(dry_run=False) + + helm_mock.add_repo.assert_not_called() + + helm_mock.upgrade_install.assert_called_once_with( + "${pipeline_name}-test-app-with-local-chart", + "path/to/helm/charts/", + False, + "test-namespace", + {"nameOverride": "test-value"}, + HelmUpgradeInstallFlags(), + ) + def test_should_raise_not_implemented_error_when_helm_chart_is_not_set( self, kubernetes_app: KubernetesApp, @@ -157,8 +195,8 @@ def test_should_raise_not_implemented_error_when_helm_chart_is_not_set( kubernetes_app.deploy(True) helm_mock.add_repo.assert_called() assert ( - "Please implement the helm_chart property of the kpops.components.base_components.kubernetes_app module." - == str(error.value) + str(error.value) + == "Please implement the helm_chart property of the kpops.components.base_components.kubernetes_app module." ) def test_should_call_helm_uninstall_when_destroying_kubernetes_app( @@ -173,7 +211,7 @@ def test_should_call_helm_uninstall_when_destroying_kubernetes_app( kubernetes_app.destroy(True) helm_mock.uninstall.assert_called_once_with( - "test-namespace", "test-kubernetes-app", True + "test-namespace", "${pipeline_name}-test-kubernetes-app", True ) log_info_mock.assert_called_once_with(magentaify(stdout)) diff --git a/tests/components/test_producer_app.py b/tests/components/test_producer_app.py index 75df0b5a8..84f9f86c6 100644 --- a/tests/components/test_producer_app.py +++ b/tests/components/test_producer_app.py @@ -19,9 +19,9 @@ class TestProducerApp: PRODUCER_APP_NAME = "test-producer-app-with-long-name-0123456789abcdefghijklmnop" - PRODUCER_APP_CLEAN_NAME = "test-producer-app-with-long-name-0123456789abc-clean" + PRODUCER_APP_CLEAN_NAME = "test-producer-app-with-long-n-clean" - @pytest.fixture + @pytest.fixture() def handlers(self) -> ComponentHandlers: return ComponentHandlers( schema_handler=MagicMock(), @@ -29,7 +29,7 @@ def handlers(self) -> ComponentHandlers: topic_handler=MagicMock(), ) - @pytest.fixture + @pytest.fixture() def config(self) -> PipelineConfig: return PipelineConfig( defaults_path=DEFAULTS_PATH, @@ -40,7 +40,7 @@ def config(self) -> PipelineConfig: ), ) - @pytest.fixture + @pytest.fixture() def producer_app( self, config: PipelineConfig, handlers: ComponentHandlers ) -> ProducerApp: @@ -116,7 +116,7 @@ def test_deploy_order_when_dry_run_is_false( assert mock.mock_calls == [ mocker.call.mock_create_topics(to_section=producer_app.to, dry_run=False), mocker.call.mock_helm_upgrade_install( - self.PRODUCER_APP_NAME, + "${pipeline_name}-" + self.PRODUCER_APP_NAME, "bakdata-streams-bootstrap/producer-app", False, "test-namespace", @@ -150,7 +150,7 @@ def test_destroy( producer_app.destroy(dry_run=True) mock_helm_uninstall.assert_called_once_with( - "test-namespace", self.PRODUCER_APP_NAME, True + "test-namespace", "${pipeline_name}-" + self.PRODUCER_APP_NAME, True ) def test_should_not_reset_producer_app( @@ -175,10 +175,12 @@ def test_should_not_reset_producer_app( assert mock.mock_calls == [ mocker.call.helm_uninstall( - "test-namespace", self.PRODUCER_APP_CLEAN_NAME, True + "test-namespace", + "${pipeline_name}-" + self.PRODUCER_APP_CLEAN_NAME, + True, ), mocker.call.helm_upgrade_install( - self.PRODUCER_APP_CLEAN_NAME, + "${pipeline_name}-" + self.PRODUCER_APP_CLEAN_NAME, "bakdata-streams-bootstrap/producer-app-cleanup-job", True, "test-namespace", @@ -192,11 +194,13 @@ def test_should_not_reset_producer_app( ), mocker.call.print_helm_diff( ANY, - "test-producer-app-with-long-name-0123456789abc-clean", + "${pipeline_name}-" + self.PRODUCER_APP_CLEAN_NAME, logging.getLogger("KafkaApp"), ), mocker.call.helm_uninstall( - "test-namespace", self.PRODUCER_APP_CLEAN_NAME, True + "test-namespace", + "${pipeline_name}-" + self.PRODUCER_APP_CLEAN_NAME, + True, ), ] @@ -216,10 +220,12 @@ def test_should_clean_producer_app_and_deploy_clean_up_job_and_delete_clean_up_w assert mock.mock_calls == [ mocker.call.helm_uninstall( - "test-namespace", self.PRODUCER_APP_CLEAN_NAME, False + "test-namespace", + "${pipeline_name}-" + self.PRODUCER_APP_CLEAN_NAME, + False, ), mocker.call.helm_upgrade_install( - self.PRODUCER_APP_CLEAN_NAME, + "${pipeline_name}-" + self.PRODUCER_APP_CLEAN_NAME, "bakdata-streams-bootstrap/producer-app-cleanup-job", False, "test-namespace", @@ -232,6 +238,8 @@ def test_should_clean_producer_app_and_deploy_clean_up_job_and_delete_clean_up_w HelmUpgradeInstallFlags(version="2.4.2", wait=True, wait_for_jobs=True), ), mocker.call.helm_uninstall( - "test-namespace", self.PRODUCER_APP_CLEAN_NAME, False + "test-namespace", + "${pipeline_name}-" + self.PRODUCER_APP_CLEAN_NAME, + False, ), ] diff --git a/tests/components/test_streams_app.py b/tests/components/test_streams_app.py index 8cac51dfd..0d9135b54 100644 --- a/tests/components/test_streams_app.py +++ b/tests/components/test_streams_app.py @@ -23,9 +23,9 @@ class TestStreamsApp: STREAMS_APP_NAME = "test-streams-app-with-long-name-0123456789abcdefghijklmnop" - STREAMS_APP_CLEAN_NAME = "test-streams-app-with-long-name-0123456789abcd-clean" + STREAMS_APP_CLEAN_NAME = "test-streams-app-with-long-na-clean" - @pytest.fixture + @pytest.fixture() def handlers(self) -> ComponentHandlers: return ComponentHandlers( schema_handler=MagicMock(), @@ -33,7 +33,7 @@ def handlers(self) -> ComponentHandlers: topic_handler=MagicMock(), ) - @pytest.fixture + @pytest.fixture() def config(self) -> PipelineConfig: return PipelineConfig( defaults_path=DEFAULTS_PATH, @@ -45,7 +45,7 @@ def config(self) -> PipelineConfig: helm_diff_config=HelmDiffConfig(), ) - @pytest.fixture + @pytest.fixture() def streams_app( self, config: PipelineConfig, handlers: ComponentHandlers ) -> StreamsApp: @@ -145,7 +145,9 @@ def test_no_empty_input_topic( def test_should_validate(self, config: PipelineConfig, handlers: ComponentHandlers): # An exception should be raised when both role and type are defined and type is input - with pytest.raises(ValueError): + with pytest.raises( + ValueError, match="Define role only if `type` is `pattern` or `None`" + ): StreamsApp( name=self.STREAMS_APP_NAME, config=config, @@ -167,7 +169,9 @@ def test_should_validate(self, config: PipelineConfig, handlers: ComponentHandle ) # An exception should be raised when both role and type are defined and type is error - with pytest.raises(ValueError): + with pytest.raises( + ValueError, match="Define `role` only if `type` is undefined" + ): StreamsApp( name=self.STREAMS_APP_NAME, config=config, @@ -315,7 +319,7 @@ def test_deploy_order_when_dry_run_is_false( assert mock.mock_calls == [ mocker.call.mock_create_topics(to_section=streams_app.to, dry_run=dry_run), mocker.call.mock_helm_upgrade_install( - self.STREAMS_APP_NAME, + "${pipeline_name}-" + self.STREAMS_APP_NAME, "bakdata-streams-bootstrap/streams-app", dry_run, "test-namespace", @@ -351,7 +355,7 @@ def test_destroy(self, streams_app: StreamsApp, mocker: MockerFixture): streams_app.destroy(dry_run=True) mock_helm_uninstall.assert_called_once_with( - "test-namespace", self.STREAMS_APP_NAME, True + "test-namespace", "${pipeline_name}-" + self.STREAMS_APP_NAME, True ) def test_reset_when_dry_run_is_false( @@ -371,10 +375,12 @@ def test_reset_when_dry_run_is_false( assert mock.mock_calls == [ mocker.call.helm_uninstall( - "test-namespace", self.STREAMS_APP_CLEAN_NAME, dry_run + "test-namespace", + "${pipeline_name}-" + self.STREAMS_APP_CLEAN_NAME, + dry_run, ), mocker.call.helm_upgrade_install( - self.STREAMS_APP_CLEAN_NAME, + "${pipeline_name}-" + self.STREAMS_APP_CLEAN_NAME, "bakdata-streams-bootstrap/streams-app-cleanup-job", dry_run, "test-namespace", @@ -388,7 +394,9 @@ def test_reset_when_dry_run_is_false( HelmUpgradeInstallFlags(version="2.9.0", wait=True, wait_for_jobs=True), ), mocker.call.helm_uninstall( - "test-namespace", self.STREAMS_APP_CLEAN_NAME, dry_run + "test-namespace", + "${pipeline_name}-" + self.STREAMS_APP_CLEAN_NAME, + dry_run, ), ] @@ -411,10 +419,12 @@ def test_should_clean_streams_app_and_deploy_clean_up_job_and_delete_clean_up( assert mock.mock_calls == [ mocker.call.helm_uninstall( - "test-namespace", self.STREAMS_APP_CLEAN_NAME, dry_run + "test-namespace", + "${pipeline_name}-" + self.STREAMS_APP_CLEAN_NAME, + dry_run, ), mocker.call.helm_upgrade_install( - self.STREAMS_APP_CLEAN_NAME, + "${pipeline_name}-" + self.STREAMS_APP_CLEAN_NAME, "bakdata-streams-bootstrap/streams-app-cleanup-job", dry_run, "test-namespace", @@ -428,6 +438,8 @@ def test_should_clean_streams_app_and_deploy_clean_up_job_and_delete_clean_up( HelmUpgradeInstallFlags(version="2.9.0", wait=True, wait_for_jobs=True), ), mocker.call.helm_uninstall( - "test-namespace", self.STREAMS_APP_CLEAN_NAME, dry_run + "test-namespace", + "${pipeline_name}-" + self.STREAMS_APP_CLEAN_NAME, + dry_run, ), ] diff --git a/tests/pipeline/snapshots/snap_test_pipeline.py b/tests/pipeline/snapshots/snap_test_pipeline.py index 2efc656ab..c2e339fbc 100644 --- a/tests/pipeline/snapshots/snap_test_pipeline.py +++ b/tests/pipeline/snapshots/snap_test_pipeline.py @@ -28,7 +28,7 @@ 'schemaRegistryUrl': 'http://localhost:8081' } }, - 'name': 'resources-custom-config-app1', + 'name': 'app1', 'namespace': 'development-namespace', 'prefix': 'resources-custom-config-', 'repo_config': { @@ -70,7 +70,7 @@ 'schemaRegistryUrl': 'http://localhost:8081' } }, - 'name': 'resources-custom-config-app2', + 'name': 'app2', 'namespace': 'development-namespace', 'prefix': 'resources-custom-config-', 'repo_config': { @@ -123,7 +123,7 @@ 'schemaRegistryUrl': 'http://localhost:8081' } }, - 'name': 'resources-pipeline-with-inflate-scheduled-producer', + 'name': 'scheduled-producer', 'namespace': 'example-namespace', 'prefix': 'resources-pipeline-with-inflate-', 'repo_config': { @@ -190,7 +190,7 @@ 'schemaRegistryUrl': 'http://localhost:8081' } }, - 'name': 'resources-pipeline-with-inflate-converter', + 'name': 'converter', 'namespace': 'example-namespace', 'prefix': 'resources-pipeline-with-inflate-', 'repo_config': { @@ -265,7 +265,7 @@ 'schemaRegistryUrl': 'http://localhost:8081' } }, - 'name': 'resources-pipeline-with-inflate-should-inflate', + 'name': 'should-inflate', 'namespace': 'example-namespace', 'prefix': 'resources-pipeline-with-inflate-', 'repo_config': { @@ -315,7 +315,7 @@ 'topics': 'resources-pipeline-with-inflate-should-inflate', 'transforms.changeTopic.replacement': 'resources-pipeline-with-inflate-should-inflate-index-v1' }, - 'name': 'resources-pipeline-with-inflate-should-inflate-inflated-sink-connector', + 'name': 'should-inflate-inflated-sink-connector', 'namespace': 'example-namespace', 'prefix': 'resources-pipeline-with-inflate-', 'repo_config': { @@ -358,11 +358,11 @@ 'inputTopics': [ 'kafka-sink-connector' ], - 'outputTopic': 'should-inflate-should-inflate-inflated-streams-app', + 'outputTopic': 'resources-pipeline-with-inflate-should-inflate-should-inflate-inflated-streams-app', 'schemaRegistryUrl': 'http://localhost:8081' } }, - 'name': 'resources-pipeline-with-inflate-should-inflate-inflated-streams-app', + 'name': 'should-inflate-inflated-streams-app', 'namespace': 'example-namespace', 'prefix': 'resources-pipeline-with-inflate-', 'repo_config': { @@ -384,7 +384,7 @@ 'type': 'error', 'value_schema': 'com.bakdata.kafka.DeadLetter' }, - 'should-inflate-should-inflate-inflated-streams-app': { + 'resources-pipeline-with-inflate-should-inflate-should-inflate-inflated-streams-app': { 'configs': { }, 'type': 'output' @@ -425,7 +425,7 @@ } } }, - 'name': 'resources-kafka-connect-sink-streams-app', + 'name': 'streams-app', 'namespace': 'example-namespace', 'prefix': 'resources-kafka-connect-sink-', 'repo_config': { @@ -472,7 +472,7 @@ 'tasks.max': '1', 'topics': 'example-output' }, - 'name': 'resources-kafka-connect-sink-es-sink-connector', + 'name': 'es-sink-connector', 'namespace': 'example-namespace', 'prefix': 'resources-kafka-connect-sink-', 'repo_config': { @@ -509,7 +509,7 @@ 'schemaRegistryUrl': 'http://localhost:8081' } }, - 'name': 'resources-first-pipeline-scheduled-producer', + 'name': 'scheduled-producer', 'namespace': 'example-namespace', 'prefix': 'resources-first-pipeline-', 'repo_config': { @@ -576,7 +576,7 @@ 'schemaRegistryUrl': 'http://localhost:8081' } }, - 'name': 'resources-first-pipeline-converter', + 'name': 'converter', 'namespace': 'example-namespace', 'prefix': 'resources-first-pipeline-', 'repo_config': { @@ -651,7 +651,7 @@ 'schemaRegistryUrl': 'http://localhost:8081' } }, - 'name': 'resources-first-pipeline-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name', + 'name': 'a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name', 'namespace': 'example-namespace', 'prefix': 'resources-first-pipeline-', 'repo_config': { @@ -701,7 +701,7 @@ 'schemaRegistryUrl': 'http://localhost:8081' } }, - 'name': 'resources-pipeline-with-paths-account-producer', + 'name': 'account-producer', 'namespace': 'test', 'prefix': 'resources-pipeline-with-paths-', 'repo_config': { @@ -756,7 +756,7 @@ } } }, - 'name': 'resources-no-input-topic-pipeline-app1', + 'name': 'app1', 'namespace': 'example-namespace', 'prefix': 'resources-no-input-topic-pipeline-', 'repo_config': { @@ -807,7 +807,7 @@ 'schemaRegistryUrl': 'http://localhost:8081' } }, - 'name': 'resources-no-input-topic-pipeline-app2', + 'name': 'app2', 'namespace': 'example-namespace', 'prefix': 'resources-no-input-topic-pipeline-', 'repo_config': { @@ -875,7 +875,7 @@ } } }, - 'name': 'resources-no-user-defined-components-streams-app', + 'name': 'streams-app', 'namespace': 'example-namespace', 'prefix': 'resources-no-user-defined-components-', 'repo_config': { @@ -929,7 +929,7 @@ 'schemaRegistryUrl': 'http://localhost:8081' } }, - 'name': 'resources-pipeline-with-envs-input-producer', + 'name': 'input-producer', 'namespace': 'example-namespace', 'prefix': 'resources-pipeline-with-envs-', 'repo_config': { @@ -996,7 +996,7 @@ 'schemaRegistryUrl': 'http://localhost:8081' } }, - 'name': 'resources-pipeline-with-envs-converter', + 'name': 'converter', 'namespace': 'example-namespace', 'prefix': 'resources-pipeline-with-envs-', 'repo_config': { @@ -1071,7 +1071,7 @@ 'schemaRegistryUrl': 'http://localhost:8081' } }, - 'name': 'resources-pipeline-with-envs-filter', + 'name': 'filter', 'namespace': 'example-namespace', 'prefix': 'resources-pipeline-with-envs-', 'repo_config': { @@ -1131,7 +1131,7 @@ }, 'suspend': True }, - 'name': 'from-pipeline-component-account-producer', + 'name': 'account-producer', 'namespace': '${NAMESPACE}', 'prefix': 'from-pipeline-component-', 'repo_config': { @@ -1160,7 +1160,7 @@ 'schemaRegistryUrl': 'http://localhost:8081' } }, - 'name': 'resources-read-from-component-producer1', + 'name': 'producer1', 'namespace': 'example-namespace', 'prefix': 'resources-read-from-component-', 'repo_config': { @@ -1250,7 +1250,7 @@ 'schemaRegistryUrl': 'http://localhost:8081' } }, - 'name': 'resources-read-from-component-inflate-step', + 'name': 'inflate-step', 'namespace': 'example-namespace', 'prefix': 'resources-read-from-component-', 'repo_config': { @@ -1300,7 +1300,7 @@ 'topics': 'resources-read-from-component-inflate-step', 'transforms.changeTopic.replacement': 'resources-read-from-component-inflate-step-index-v1' }, - 'name': 'resources-read-from-component-inflate-step-inflated-sink-connector', + 'name': 'inflate-step-inflated-sink-connector', 'namespace': 'example-namespace', 'prefix': 'resources-read-from-component-', 'repo_config': { @@ -1343,11 +1343,11 @@ 'inputTopics': [ 'kafka-sink-connector' ], - 'outputTopic': 'inflate-step-inflate-step-inflated-streams-app', + 'outputTopic': 'resources-read-from-component-inflate-step-inflate-step-inflated-streams-app', 'schemaRegistryUrl': 'http://localhost:8081' } }, - 'name': 'resources-read-from-component-inflate-step-inflated-streams-app', + 'name': 'inflate-step-inflated-streams-app', 'namespace': 'example-namespace', 'prefix': 'resources-read-from-component-', 'repo_config': { @@ -1361,7 +1361,7 @@ 'models': { }, 'topics': { - 'inflate-step-inflate-step-inflated-streams-app': { + 'resources-read-from-component-inflate-step-inflate-step-inflated-streams-app': { 'configs': { }, 'type': 'output' @@ -1404,7 +1404,7 @@ }, 'errorTopic': 'resources-read-from-component-inflate-step-without-prefix-error', 'inputTopics': [ - 'inflate-step-inflate-step-inflated-streams-app' + 'resources-read-from-component-inflate-step-inflate-step-inflated-streams-app' ], 'outputTopic': 'resources-read-from-component-inflate-step-without-prefix', 'schemaRegistryUrl': 'http://localhost:8081' @@ -1460,7 +1460,7 @@ 'topics': 'resources-read-from-component-inflate-step-without-prefix', 'transforms.changeTopic.replacement': 'resources-read-from-component-inflate-step-without-prefix-index-v1' }, - 'name': 'resources-read-from-component-inflate-step-without-prefix-inflated-sink-connector', + 'name': 'inflate-step-without-prefix-inflated-sink-connector', 'namespace': 'example-namespace', 'prefix': 'resources-read-from-component-', 'repo_config': { @@ -1507,7 +1507,7 @@ 'schemaRegistryUrl': 'http://localhost:8081' } }, - 'name': 'resources-read-from-component-inflate-step-without-prefix-inflated-streams-app', + 'name': 'inflate-step-without-prefix-inflated-streams-app', 'namespace': 'example-namespace', 'prefix': 'resources-read-from-component-', 'repo_config': { @@ -1564,7 +1564,7 @@ 'topics': { } }, - 'name': 'resources-read-from-component-consumer1', + 'name': 'consumer1', 'namespace': 'example-namespace', 'prefix': 'resources-read-from-component-', 'repo_config': { @@ -1624,7 +1624,7 @@ 'topics': { } }, - 'name': 'resources-read-from-component-consumer2', + 'name': 'consumer2', 'namespace': 'example-namespace', 'prefix': 'resources-read-from-component-', 'repo_config': { @@ -1679,7 +1679,7 @@ } } }, - 'name': 'resources-read-from-component-consumer3', + 'name': 'consumer3', 'namespace': 'example-namespace', 'prefix': 'resources-read-from-component-', 'repo_config': { @@ -1716,7 +1716,7 @@ }, 'errorTopic': 'resources-read-from-component-consumer4-error', 'inputTopics': [ - 'inflate-step-inflate-step-inflated-streams-app' + 'resources-read-from-component-inflate-step-inflate-step-inflated-streams-app' ], 'schemaRegistryUrl': 'http://localhost:8081' } @@ -1730,7 +1730,7 @@ 'topics': { } }, - 'name': 'resources-read-from-component-consumer4', + 'name': 'consumer4', 'namespace': 'example-namespace', 'prefix': 'resources-read-from-component-', 'repo_config': { @@ -1781,7 +1781,7 @@ 'topics': { } }, - 'name': 'resources-read-from-component-consumer5', + 'name': 'consumer5', 'namespace': 'example-namespace', 'prefix': 'resources-read-from-component-', 'repo_config': { @@ -1835,7 +1835,7 @@ 'schemaRegistryUrl': 'http://localhost:8081' } }, - 'name': 'resources-component-type-substitution-scheduled-producer', + 'name': 'scheduled-producer', 'namespace': 'example-namespace', 'prefix': 'resources-component-type-substitution-', 'repo_config': { @@ -1902,7 +1902,7 @@ 'schemaRegistryUrl': 'http://localhost:8081' } }, - 'name': 'resources-component-type-substitution-converter', + 'name': 'converter', 'namespace': 'example-namespace', 'prefix': 'resources-component-type-substitution-', 'repo_config': { @@ -1984,7 +1984,7 @@ 'schemaRegistryUrl': 'http://localhost:8081' } }, - 'name': 'resources-component-type-substitution-filter-app', + 'name': 'filter-app', 'namespace': 'example-namespace', 'prefix': 'resources-component-type-substitution-', 'repo_config': { @@ -2042,7 +2042,7 @@ 'schemaRegistryUrl': 'http://localhost:8081' } }, - 'name': 'resources-custom-config-app1', + 'name': 'app1', 'namespace': 'development-namespace', 'prefix': 'resources-custom-config-', 'repo_config': { @@ -2084,7 +2084,7 @@ 'schemaRegistryUrl': 'http://localhost:8081' } }, - 'name': 'resources-custom-config-app2', + 'name': 'app2', 'namespace': 'development-namespace', 'prefix': 'resources-custom-config-', 'repo_config': { @@ -2139,7 +2139,7 @@ 'schemaRegistryUrl': 'http://localhost:8081' } }, - 'name': 'resources-custom-config-app1', + 'name': 'app1', 'namespace': 'development-namespace', 'prefix': 'resources-custom-config-', 'repo_config': { @@ -2181,7 +2181,7 @@ 'schemaRegistryUrl': 'http://localhost:8081' } }, - 'name': 'resources-custom-config-app2', + 'name': 'app2', 'namespace': 'development-namespace', 'prefix': 'resources-custom-config-', 'repo_config': { @@ -2243,7 +2243,7 @@ } } }, - 'name': 'resources-kafka-connect-sink-streams-app-development', + 'name': 'streams-app-development', 'namespace': 'development-namespace', 'prefix': 'resources-kafka-connect-sink-', 'repo_config': { @@ -2290,7 +2290,7 @@ 'tasks.max': '1', 'topics': 'example-output' }, - 'name': 'resources-kafka-connect-sink-es-sink-connector', + 'name': 'es-sink-connector', 'namespace': 'example-namespace', 'prefix': 'resources-kafka-connect-sink-', 'repo_config': { diff --git a/tests/pipeline/test_components/components.py b/tests/pipeline/test_components/components.py index 9a2d25cb4..86e2c8b8e 100644 --- a/tests/pipeline/test_components/components.py +++ b/tests/pipeline/test_components/components.py @@ -37,15 +37,15 @@ class ShouldInflate(StreamsApp): def inflate(self) -> list[PipelineComponent]: inflate_steps = super().inflate() if self.to: - name = self.name.removeprefix(self.prefix) for topic_name, topic_config in self.to.topics.items(): if topic_config.type == OutputTopicTypes.OUTPUT: kafka_connector = KafkaSinkConnector( - name=f"{name}-inflated-sink-connector", + name=f"{self.name}-inflated-sink-connector", config=self.config, handlers=self.handlers, namespace="example-namespace", - app={ # type: ignore # FIXME + # FIXME + app={ # type: ignore[reportGeneralTypeIssues] "topics": topic_name, "transforms.changeTopic.replacement": f"{topic_name}-index-v1", }, @@ -62,13 +62,13 @@ def inflate(self) -> list[PipelineComponent]: ) inflate_steps.append(kafka_connector) streams_app = StreamsApp( - name=f"{name}-inflated-streams-app", + name=f"{self.name}-inflated-streams-app", config=self.config, handlers=self.handlers, - to=ToSection( # type: ignore + to=ToSection( # type: ignore[reportGeneralTypeIssues] topics={ TopicName( - f"{self.name}-" + "${component_name}" + f"{self.full_name}-" + "${component_name}" ): TopicConfig(type=OutputTopicTypes.OUTPUT) } ).dict(), diff --git a/tests/pipeline/test_pipeline.py b/tests/pipeline/test_pipeline.py index 095d0eded..433960e74 100644 --- a/tests/pipeline/test_pipeline.py +++ b/tests/pipeline/test_pipeline.py @@ -1,3 +1,4 @@ +import logging from pathlib import Path import pytest @@ -46,6 +47,36 @@ def test_load_pipeline(self, snapshot: SnapshotTest): snapshot.assert_match(enriched_pipeline, "test-pipeline") + def test_generate_with_steps_flag_should_write_log_warning( + self, caplog: pytest.LogCaptureFixture + ): + result = runner.invoke( + app, + [ + "generate", + "--pipeline-base-dir", + str(PIPELINE_BASE_DIR_PATH), + str(RESOURCE_PATH / "first-pipeline/pipeline.yaml"), + "tests.pipeline.test_components", + "--defaults", + str(RESOURCE_PATH), + "--steps", + "a", + ], + catch_exceptions=False, + ) + + assert caplog.record_tuples == [ + ( + "root", + logging.WARNING, + "The following flags are considered only when `--template` is set: \n \ + '--steps'", + ) + ] + + assert result.exit_code == 0 + def test_name_equal_prefix_name_concatenation(self): result = runner.invoke( app, @@ -65,10 +96,8 @@ def test_name_equal_prefix_name_concatenation(self): enriched_pipeline: dict = yaml.safe_load(result.stdout) - assert ( - enriched_pipeline["components"][0]["name"] - == "my-fake-prefix-my-streams-app" - ) + assert enriched_pipeline["components"][0]["prefix"] == "my-fake-prefix-" + assert enriched_pipeline["components"][0]["name"] == "my-streams-app" def test_pipelines_with_env_values(self, snapshot: SnapshotTest): result = runner.invoke( @@ -129,9 +158,10 @@ def test_substitute_in_component(self, snapshot: SnapshotTest): enriched_pipeline: dict = yaml.safe_load(result.stdout) assert ( - enriched_pipeline["components"][0]["name"] - == "resources-component-type-substitution-scheduled-producer" + enriched_pipeline["components"][0]["prefix"] + == "resources-component-type-substitution-" ) + assert enriched_pipeline["components"][0]["name"] == "scheduled-producer" labels = enriched_pipeline["components"][0]["app"]["labels"] assert labels["app_name"] == "scheduled-producer" @@ -428,8 +458,34 @@ def test_default_config(self, snapshot: SnapshotTest): snapshot.assert_match(enriched_pipeline, "test-pipeline") + def test_env_vars_precedence_over_config( + self, + monkeypatch: pytest.MonkeyPatch, + snapshot: SnapshotTest, + ): + monkeypatch.setenv(name="KPOPS_KAFKA_BROKERS", value="env_broker") + + result = runner.invoke( + app, + [ + "generate", + "--pipeline-base-dir", + str(PIPELINE_BASE_DIR_PATH), + str(RESOURCE_PATH / "custom-config/pipeline.yaml"), + "--config", + str(RESOURCE_PATH / "custom-config/config.yaml"), + ], + catch_exceptions=False, + ) + assert result.exit_code == 0 + enriched_pipeline: dict = yaml.safe_load(result.stdout) + assert ( + enriched_pipeline["components"][0]["app"]["streams"]["brokers"] + == "env_broker" + ) + def test_model_serialization(self, snapshot: SnapshotTest): - """Test model serialization of component containing pathlib.Path attribute""" + """Test model serialization of component containing pathlib.Path attribute.""" result = runner.invoke( app, [ diff --git a/tests/pipeline/test_template.py b/tests/pipeline/test_template.py index cd4436b7a..a43fbec5b 100644 --- a/tests/pipeline/test_template.py +++ b/tests/pipeline/test_template.py @@ -15,7 +15,7 @@ class TestTemplate: - @pytest.fixture + @pytest.fixture() def run_command(self, mocker: MockerFixture) -> MagicMock: return mocker.patch.object(Helm, "_Helm__execute") diff --git a/tests/utils/resources/__init__.py b/tests/utils/resources/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/utils/resources/nested_base_settings.py b/tests/utils/resources/nested_base_settings.py new file mode 100644 index 000000000..f7f92358a --- /dev/null +++ b/tests/utils/resources/nested_base_settings.py @@ -0,0 +1,14 @@ +from pydantic import BaseSettings, Field + + +class NestedSettings(BaseSettings): + attr: str = Field("attr") + + +class ParentSettings(BaseSettings): + not_nested_field: str = Field("not_nested_field") + nested_field: NestedSettings = Field(...) + field_with_env_defined: str = Field( + default=..., + env="FIELD_WITH_ENV_DEFINED", + ) diff --git a/tests/utils/test_doc_gen.py b/tests/utils/test_doc_gen.py new file mode 100644 index 000000000..d234bd79d --- /dev/null +++ b/tests/utils/test_doc_gen.py @@ -0,0 +1,252 @@ +from pathlib import Path +from typing import Any + +import pytest + +from hooks.gen_docs.gen_docs_env_vars import ( + EnvVarAttrs, + append_csv_to_dotenv_file, + collect_fields, + csv_append_env_var, + write_csv_to_md_file, + write_title_to_dotenv_file, +) +from tests.utils.resources.nested_base_settings import ParentSettings + + +class TestEnvDocGen: + def test_collect_fields(self): + expected: list[Any] = [ + "not_nested_field", + "attr", + Ellipsis, + Ellipsis, + ] + actual = [field.field_info.default for field in collect_fields(ParentSettings)] + assert actual == expected + + @pytest.mark.parametrize( + ("var_name", "default_value", "description", "extra_args", "expected_outcome"), + [ + pytest.param( + "var_name", + "default_value", + "description", + (), + "var_name,default_value,False,description", + id="String desc", + ), + pytest.param( + "var_name", + "default_value", + ["description", " description"], + (), + "var_name,default_value,False,description description", + id="List desc", + ), + pytest.param( + "var_name", + "default_value", + "description", + ("extra arg 1", "extra arg 2"), + "var_name,default_value,False,description,extra arg 1,extra arg 2", + id="Extra args", + ), + pytest.param( + "var_name", + "default_value", + None, + (), + "var_name,default_value,False,", + id="No desc", + ), + ], + ) + def test_csv_append_env_var( + self, + tmp_path: Path, + var_name: str, + default_value: Any, + description: str | list[str], + extra_args: tuple, + expected_outcome: str, + ): + target = tmp_path / "target.csv" + csv_append_env_var(target, var_name, default_value, description, *extra_args) + with target.open() as t: + assert ( + t.read().replace("\r\n", "\n").replace("\r", "\n") + == expected_outcome + "\n" + ) + + def test_write_title_to_dotenv_file(self, tmp_path: Path): + target = tmp_path / "target.ENV" + write_title_to_dotenv_file(target, "title", "description of length 72" * 3) + with target.open() as t: + assert t.read().replace("\r\n", "\n").replace("\r", "\n") == ( + "# title\n" + "#\n" + "# " + "description of length 72description of length 72description of" + "\n" + "# length 72\n#\n" + ) + + @pytest.mark.parametrize( + ("var_name", "default", "required", "description", "setting_name", "expected"), + [ + pytest.param( + "NAME", + "default", + "True", + "description", + "setting_name", + "# setting_name\n# description\nNAME=default\n", + id="default exists, required", + ), + pytest.param( + "NAME", + "", + "True", + "description", + "setting_name", + "# setting_name\n" + "# description\n" + "NAME # No default value, required\n", + id="default not exists, required", + ), + pytest.param( + "NAME", + "default", + "False", + "description", + "setting_name", + "# setting_name\n# description\nNAME=default\n", + id="default exists, not required", + ), + pytest.param( + "NAME", + "", + "False", + "description", + "setting_name", + "# setting_name\n" + "# description\n" + "NAME # No default value, not required\n", + id="default not exists, not required", + ), + pytest.param( + "NAME", + "default", + "True", + "description", + "", + "# description\nNAME=default\n", + id="no setting name", + ), + ], + ) + def test_append_csv_to_dotenv_file( + self, + tmp_path: Path, + var_name: str, + default: Any, + required: str, + description: str | list[str], + setting_name: str, + expected: str, + ): + source = tmp_path / "source.csv" + target = tmp_path / "target.env" + csv_record = [var_name, default, required, description] + csv_column_names = [ + EnvVarAttrs.NAME, + EnvVarAttrs.DEFAULT_VALUE, + EnvVarAttrs.REQUIRED, + EnvVarAttrs.DESCRIPTION, + ] + with source.open("w+", newline="") as f: + if setting_name is not None: + csv_record.append(setting_name) + csv_column_names.append(EnvVarAttrs.CORRESPONDING_SETTING_NAME) + f.write(",".join(csv_column_names) + "\n") + f.write(",".join(csv_record)) + append_csv_to_dotenv_file(source, target) + with target.open("r", newline="") as f: + assert f.read().replace("\r\n", "\n").replace("\r", "\n") == expected + + @pytest.mark.parametrize( + ("title", "description", "heading", "expected"), + [ + pytest.param( + "title", + "description", + "###", + "### title\n\ndescription\n\n", + id="all provided, default heading", + ), + pytest.param( + "title", + "description", + "##", + "## title\n\ndescription\n\n", + id="all provided, different heading", + ), + pytest.param( + "title", + "description", + "", + "title\n\ndescription\n\n", + id="all provided, heading empty str", + ), + pytest.param( + "title", + "description", + None, + "title\n\ndescription\n\n", + id="all provided, heading is None", + ), + pytest.param( + None, + "description", + "###", + "description\n\n", + id="no title", + ), + pytest.param( + "title", + None, + "###", + "### title\n\n", + id="no description", + ), + ], + ) + def test_write_csv_to_md_file( + self, + tmp_path: Path, + title: str, + description: str, + heading: str, + expected: str, + ): + source = tmp_path / "source.csv" + target = tmp_path / "target.env" + csv_record = ["NAME", "default", "True", "description", "setting_name"] + csv_column_names = [ + EnvVarAttrs.NAME, + EnvVarAttrs.DEFAULT_VALUE, + EnvVarAttrs.REQUIRED, + EnvVarAttrs.DESCRIPTION, + EnvVarAttrs.CORRESPONDING_SETTING_NAME, + ] + with source.open("w+", newline="") as f: + f.write(",".join(csv_column_names) + "\n") + f.write(",".join(csv_record)) + write_csv_to_md_file(source, target, title, description, heading) + with target.open("r", newline="") as f: + assert f.read().replace("\r\n", "\n").replace("\r", "\n") == expected + str( + "|Name|Default Value|Required|Description|Setting name|\n" + "|----|-------------|--------|-----------|------------|\n" + "|NAME|default |True |description|setting_name|\n", + ) diff --git a/tests/utils/test_environment.py b/tests/utils/test_environment.py index 09bbb75de..8fc02c826 100644 --- a/tests/utils/test_environment.py +++ b/tests/utils/test_environment.py @@ -5,12 +5,12 @@ from kpops.utils.environment import Environment -@pytest.fixture +@pytest.fixture() def fake_environment_windows(): return {"MY": "fake", "ENVIRONMENT": "here"} -@pytest.fixture +@pytest.fixture() def fake_environment_linux(): return {"my": "fake", "environment": "here"}