Replace black with ruff

.github/workflows/ci.yaml

@@ -43,11 +43,11 @@ jobs:
             echo "::add-matcher::.github/ruff-matcher.json"
             poetry run ruff check . --config pyproject.toml --output-format text --no-fix
           else
-            poetry run pre-commit run ruff --all-files --show-diff-on-failure
+            poetry run pre-commit run ruff-lint --all-files --show-diff-on-failure
           fi;
 
-      - name: Formatting (black)
-        run: poetry run pre-commit run black --all-files --show-diff-on-failure
+      - name: Formatting (ruff)
+        run: poetry run pre-commit run ruff-format --all-files --show-diff-on-failure
 
       - name: Typing (pyright)
         run: poetry run pre-commit run pyright --all-files
@@ -64,6 +64,9 @@ jobs:
       - name: Generate pipeline definitions
         run: poetry run pre-commit run gen-docs-components --all-files --show-diff-on-failure
 
+      - name: Copy contribution guide into docs
+        run: poetry run pre-commit run gen-docs-contributing --all-files --show-diff-on-failure
+
       - name: Test
         run: poetry run pytest tests
 

.pre-commit-config.yaml

@@ -1,21 +1,22 @@
 repos:
   - repo: local
     hooks:
-      - id: ruff
-        name: ruff
+      - id: ruff-lint
+        name: ruff-lint
         entry: ruff check .
         args: [ --config, pyproject.toml, --fix, --show-fixes, --exit-non-zero-on-fix ]
         language: system
         types_or: [python]
         require_serial: true # run once for all files
         pass_filenames: false
-      - id: black
-        name: black
-        entry: black
+      - id: ruff-format
+        name: ruff-format
+        entry: ruff format .
+        args: [ --config, pyproject.toml ]
         language: system
-        types_or: [python, pyi]
+        types_or: [python]
         require_serial: true # run once for all files
-        exclude: ^tests/.*snapshots/
+        pass_filenames: false
       - id: pyright
         name: pyright
         entry: pyright
@@ -38,13 +39,15 @@ repos:
         types: [python]
         require_serial: true
         exclude: ^tests/.*snapshots/
+        pass_filenames: false
       - id: gen-docs-env-vars
         name: gen-docs-env-vars
         entry: python hooks/gen_docs/gen_docs_env_vars.py
         language: system
         types: [python]
         require_serial: true
         exclude: ^tests/.*snapshots/
+        pass_filenames: false
       - id: gen-docs-components
         name: gen-docs-components
         entry: python hooks/gen_docs/gen_docs_components.py
@@ -66,6 +69,13 @@ repos:
             | kpops/components/.*\.py
           )$
         require_serial: true
+      - id: gen-docs-contributing
+        name: gen-docs-contributing
+        entry: python hooks/gen_docs/gen_docs_contributing.py
+        language: system
+        types: [python]
+        require_serial: true
+        pass_filenames: false
       - id: dprint
         name: dprint 
         entry: dprint fmt

CONTRIBUTING.md

@@ -0,0 +1,41 @@
+# Contributions
+
+**Welcome!** We are glad to have you visit our contributibg guide!
+
+If you find any bugs or have suggestions for improvements, please open an [issue](https://github.com/bakdata/kpops/issues/new) and optionally a [pull request (PR)](https://github.com/bakdata/kpops/compare). In the case of a PR, we would appreciate it if you preface it with an issue outlining your goal and means of achieving it.
+
+## Style
+
+We advise that you stick to our `pre-commit` hooks, which you can run with `poetry run pre-commit run -a`. In order for `dprint` to work, you have to manually [install](#markdown) it locally. It will work in the CI, so it is also possible to manually carry out formatting changes flagged by `dprint` in the CI and skip installing it locally.
+
+### Python
+
+To ensure a consistent Python code style, we use [Ruff](https://docs.astral.sh/ruff/) for both linting and formatting.
+
+**Configuration**
+
+Our configuration can be found in [KPOps](https://github.com/bakdata/kpops)' top-level `pyproject.toml`.
+
+**Editor integration**
+
+Below are listed existing Ruff plugins/extensions for some of the most popular python IDEs.
+
+- [VSCode](https://marketplace.visualstudio.com/items?itemName=charliermarsh.ruff)
+- [JetBrains](https://plugins.jetbrains.com/plugin/20574-ruff)
+
+If you cannot find your Editor of choices or you want something more custom, [`ruff-lsp`](https://github.com/astral-sh/ruff-lsp) enables Ruff to be used in any editor that supports the LSP
+
+### Markdown
+
+To ensure a consistent markdown style, we use [dprint](https://dprint.dev)'s [Markdown code formatter](https://dprint.dev/plugins/markdown/).
+
+**Configuration**
+
+Our configuration can be found [here](https://github.com/bakdata/kpops/blob/main/dprint.json).
+
+**Editor integration**
+
+Below are listed existing `dprint` plugins/extensions for some of the most popular python IDEs.
+
+- [VSCode](https://marketplace.visualstudio.com/items?itemName=dprint.dprint)
+- [JetBrains](https://plugins.jetbrains.com/plugin/18192-dprint)

docs/docs/developer/auto-generation.md

@@ -37,6 +37,10 @@ Generated
 - `pipeline-components/*\.yaml` -- All single-component pipeline definitions and one big (complete) `pipeline.yaml` that contains all of them.
 - `pipeline-defaults/*\.yaml` -- All single-component defaults definitions and one big (complete) `defaults.yaml` that contains all of them.
 
+#### [Contributing](../developer/contributing.md)
+
+Copied from [`CONTRIBUTING.md`](https://github.com/bakdata/kpops/blob/main/CONTRIBUTING.md)
+
 ### [Editor integration](https://github.com/bakdata/kpops/blob/main/hooks/gen_schema.py)
 
 #### [Schemas](https://github.com/bakdata/kpops/tree/main/docs/docs/schema)

docs/docs/developer/contributing.md

@@ -0,0 +1,43 @@
+<!-- This file is autogenerated by hooks/gen_docs/gen_docs_contributing.py. -->
+
+# Contributions
+
+**Welcome!** We are glad to have you visit our contributibg guide!
+
+If you find any bugs or have suggestions for improvements, please open an [issue](https://github.com/bakdata/kpops/issues/new) and optionally a [pull request (PR)](https://github.com/bakdata/kpops/compare). In the case of a PR, we would appreciate it if you preface it with an issue outlining your goal and means of achieving it.
+
+## Style
+
+We advise that you stick to our `pre-commit` hooks, which you can run with `poetry run pre-commit run -a`. In order for `dprint` to work, you have to manually [install](#markdown) it locally. It will work in the CI, so it is also possible to manually carry out formatting changes flagged by `dprint` in the CI and skip installing it locally.
+
+### Python
+
+To ensure a consistent Python code style, we use [Ruff](https://docs.astral.sh/ruff/) for both linting and formatting.
+
+**Configuration**
+
+Our configuration can be found in [KPOps](https://github.com/bakdata/kpops)' top-level `pyproject.toml`.
+
+**Editor integration**
+
+Below are listed existing Ruff plugins/extensions for some of the most popular python IDEs.
+
+- [VSCode](https://marketplace.visualstudio.com/items?itemName=charliermarsh.ruff)
+- [JetBrains](https://plugins.jetbrains.com/plugin/20574-ruff)
+
+If you cannot find your Editor of choices or you want something more custom, [`ruff-lsp`](https://github.com/astral-sh/ruff-lsp) enables Ruff to be used in any editor that supports the LSP
+
+### Markdown
+
+To ensure a consistent markdown style, we use [dprint](https://dprint.dev)'s [Markdown code formatter](https://dprint.dev/plugins/markdown/).
+
+**Configuration**
+
+Our configuration can be found [here](https://github.com/bakdata/kpops/blob/main/dprint.json).
+
+**Editor integration**
+
+Below are listed existing `dprint` plugins/extensions for some of the most popular python IDEs.
+
+- [VSCode](https://marketplace.visualstudio.com/items?itemName=dprint.dprint)
+- [JetBrains](https://plugins.jetbrains.com/plugin/18192-dprint)

docs/docs/developer/getting-started.md

@@ -0,0 +1,12 @@
+# Getting started
+
+**Welcome!** We are glad to have you visit our developer guide! If you find any bugs or have suggestions for improvements, please open an [issue](https://github.com/bakdata/kpops/issues/new) and optionally a [pull request (PR)](https://github.com/bakdata/kpops/compare). In the case of a PR, we would appreciate it if you preface it with an issue outlining your goal and means of achieving it.
+
+Find more about our code-style or insights into KPOps' code base here in our developer guide.
+
+<!-- dprint-ignore-start -->
+
+!!! wip "Work in progress"
+    The developer guide is still under construction. If you have a question left unanswered here, feel free to ask it by opening an issue.
+
+<!-- dprint-ignore-end -->

docs/docs/stylesheets/extra.css

@@ -13,6 +13,8 @@
 
     /* Color the links with the primary color */
     --md-typeset-a-color: var(--md-primary-fg-color);
+
+    --md-admonition-icon--wip: url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="m13.78 15.3 6 6 2.11-2.16-6-6-2.11 2.16m3.72-5.2c-.39 0-.81-.05-1.14-.19L4.97 21.25l-2.11-2.11 7.41-7.4L8.5 9.96l-.72.7-1.45-1.41v2.86l-.7.7-3.52-3.56.7-.7h2.81l-1.4-1.41 3.56-3.56a2.976 2.976 0 0 1 4.22 0L9.89 5.74l1.41 1.4-.71.71 1.79 1.78 1.82-1.88c-.14-.33-.2-.75-.2-1.12a3.49 3.49 0 0 1 3.5-3.52c.59 0 1.11.14 1.58.42L16.41 6.2l1.5 1.5 2.67-2.67c.28.47.42.97.42 1.6 0 1.92-1.55 3.47-3.5 3.47Z"/></svg>')
 }
 
 .md-typeset .md-button, .md-button--primary {
@@ -22,3 +24,18 @@
 .md-typeset .md-button:hover, .md-button--primary:hover {
     border-color: white;
 }
+
+.md-typeset .admonition.wip,
+.md-typeset details.wip {
+border-color: rgb(160, 45, 0);
+}
+.md-typeset .wip > .admonition-title,
+.md-typeset .wip > summary {
+background-color: rgb(255, 195, 172);
+}
+.md-typeset .wip > .admonition-title::before,
+.md-typeset .wip > summary::before {
+background-color: rgb(255, 77, 6);
+-webkit-mask-image: var(--md-admonition-icon--wip);
+        mask-image: var(--md-admonition-icon--wip);
+}

docs/mkdocs.yml

@@ -1,6 +1,7 @@
 site_name: KPOps
-repo_url: https://github.com/bakdata/kpops/
 site_url: https://bakdata.github.io/kpops/
+repo_name: KPOps
+repo_url: https://github.com/bakdata/kpops/
 remote_branch: gh-pages
 copyright: Copyright © 2023 bakdata
 
@@ -111,5 +112,7 @@ nav:
       - CI integration:
         - GitHub Actions: user/references/ci-integration/github-actions.md
   - Developer Guide:
+    - Getting Started: developer/getting-started.md
+    - Contributing: developer/contributing.md
+    - Code base:
       - Auto generation: developer/auto-generation.md
-      - Formatting: developer/formatting.md

hooks/gen_docs/gen_docs_components.py

@@ -212,9 +212,7 @@ def get_sections(component_name: str, *, exist_changes: bool) -> KpopsComponent:
         ]
         component_sections_not_inherited: list[
             str
-        ] = DEFAULTS_PIPELINE_COMPONENT_DEPENDENCIES[
-            component_file_name
-        ]  # type: ignore [reportGeneralTypeIssues]
+        ] = DEFAULTS_PIPELINE_COMPONENT_DEPENDENCIES[component_file_name]  # type: ignore [reportGeneralTypeIssues]
     return KpopsComponent(component_sections, component_sections_not_inherited)
 
 

hooks/gen_docs/gen_docs_contributing.py

@@ -0,0 +1,21 @@
+"""Copies ``CONTROBUTING.md`` from root to the docs."""
+import shutil
+from pathlib import Path
+from tempfile import TemporaryFile
+
+KPOPS_ROOT = Path(__file__).parents[2].resolve()
+SOURCE = KPOPS_ROOT / "CONTRIBUTING.md"
+DEST = KPOPS_ROOT / "docs/docs/developer/contributing.md"
+HEADER = f"<!-- This file is autogenerated by {Path(__file__).relative_to(KPOPS_ROOT)}. -->\n\n".encode()
+
+# Copy `CONTRIBUTING.md` into the equivalent file in the docs,
+# adding a comment at the top which notes that the file is generated
+with (
+    TemporaryFile() as file_header,
+    DEST.open("w+b") as file_dest,
+    SOURCE.open("rb") as file_source,
+):
+    file_header.write(HEADER)
+    file_header.seek(0)
+    shutil.copyfileobj(file_header, file_dest)
+    shutil.copyfileobj(file_source, file_dest)

kpops/cli/main.py

@@ -271,9 +271,7 @@ def generate(
     return pipeline
 
 
-@app.command(
-    help="Deploy pipeline steps"
-)  # pyright: ignore[reportGeneralTypeIssues] https://github.com/rec/dtyper/issues/8
+@app.command(help="Deploy pipeline steps")  # pyright: ignore[reportGeneralTypeIssues] https://github.com/rec/dtyper/issues/8
 def deploy(
     pipeline_path: Path = PIPELINE_PATH_ARG,
     components_module: Optional[str] = COMPONENTS_MODULES,
@@ -296,9 +294,7 @@ def deploy(
         component.deploy(dry_run)
 
 
-@app.command(
-    help="Destroy pipeline steps"
-)  # pyright: ignore[reportGeneralTypeIssues] https://github.com/rec/dtyper/issues/8
+@app.command(help="Destroy pipeline steps")  # pyright: ignore[reportGeneralTypeIssues] https://github.com/rec/dtyper/issues/8
 def destroy(
     pipeline_path: Path = PIPELINE_PATH_ARG,
     components_module: Optional[str] = COMPONENTS_MODULES,
@@ -320,9 +316,7 @@ def destroy(
         component.destroy(dry_run)
 
 
-@app.command(
-    help="Reset pipeline steps"
-)  # pyright: ignore[reportGeneralTypeIssues] https://github.com/rec/dtyper/issues/8
+@app.command(help="Reset pipeline steps")  # pyright: ignore[reportGeneralTypeIssues] https://github.com/rec/dtyper/issues/8
 def reset(
     pipeline_path: Path = PIPELINE_PATH_ARG,
     components_module: Optional[str] = COMPONENTS_MODULES,
@@ -345,9 +339,7 @@ def reset(
         component.reset(dry_run)
 
 
-@app.command(
-    help="Clean pipeline steps"
-)  # pyright: ignore[reportGeneralTypeIssues] https://github.com/rec/dtyper/issues/8
+@app.command(help="Clean pipeline steps")  # pyright: ignore[reportGeneralTypeIssues] https://github.com/rec/dtyper/issues/8
 def clean(
     pipeline_path: Path = PIPELINE_PATH_ARG,
     components_module: Optional[str] = COMPONENTS_MODULES,