Merge branch 'master' into jvandooren/getting-started-documentation

.github/workflows/release.yml

@@ -27,4 +27,5 @@ jobs:
           pdm sync -G docs
       - name: Deploy docs
         run: |
+          pdm run dbt-score list -f markdown -n dbt_score.rules.generic --title Generic > docs/rules/generic.md
           pdm run mkdocs gh-deploy --force

CHANGELOG.md

@@ -0,0 +1,19 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to
+[Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+
+- CLI based on Click.
+- Ability to parse dbt's `manifest.json` into internal structures.
+- Rule registry and rule discovery.
+- Rule API, decorator-based or class-based.
+- Linting and scoring functionality for dbt models.
+- Configuration through `pyproject.toml`.
+- Default rules in `dbt_score.rules.generic`.

docs/rules/generic.md

@@ -0,0 +1 @@
+(content generated in CI)

mkdocs.yml

@@ -1,12 +1,25 @@
 site_name: dbt-score
 theme:
   name: material
+  features:
+    - content.code.copy # Copy button for code blocks
 plugins:
   - search
   - mkdocstrings
+markdown_extensions:
+  - pymdownx.highlight:
+      anchor_linenums: true
+      line_spans: __span
+      pygments_lang_class: true
+  - pymdownx.details
+  - pymdownx.inlinehilite
+  - pymdownx.snippets
+  - pymdownx.superfences
 nav:
   - Home: index.md
   - Get started: get_started.md
+  - Rules:
+      - rules/generic.md
   - Reference:
       - reference/cli.md
       - reference/config.md
@@ -19,5 +32,4 @@ nav:
       - Formatters:
           - reference/formatters/index.md
           - reference/formatters/human_readable_formatter.md
-      - Rules:
-          - reference/rules/generic.md
+  - Changelog: https://github.com/PicnicSupermarket/dbt-score/blob/master/CHANGELOG.md

pyproject.toml

@@ -27,7 +27,7 @@ readme = "README.md"
 license = {text = "MIT"}
 
 [project.scripts]
-dbt-score = "dbt_score.cli:cli"
+dbt-score = "dbt_score.__main__:main"
 
 [tool.pdm]
 [tool.pdm.dev-dependencies]
@@ -52,7 +52,7 @@ docs = [
 source = "scm"
 
 [tool.pdm.scripts]
-dbt-score = {call = "dbt_score.cli:cli"}
+dbt-score = {call = "dbt_score.__main__:main"}
 
 ### Mypy ###
 

src/dbt_score/__main__.py

@@ -18,6 +18,11 @@ def set_logging() -> None:
         handler.setLevel(logging.WARNING)
 
 
-if __name__ == "__main__":
+def main() -> None:
+    """Main entrypoint."""
     set_logging()
     cli()
+
+
+if __name__ == "__main__":
+    main()

src/dbt_score/cli.py

@@ -1,7 +1,7 @@
 """CLI interface."""
 
 from pathlib import Path
-from typing import Final
+from typing import Final, Literal
 
 import click
 from click.core import ParameterSource
@@ -10,6 +10,7 @@
 from dbt_score.config import Config
 from dbt_score.lint import lint_dbt_project
 from dbt_score.parse import dbt_parse, get_default_manifest_path
+from dbt_score.rule_catalog import display_catalog
 
 BANNER: Final[str] = r"""
           __ __     __
@@ -31,6 +32,14 @@ def cli() -> None:
 
 
 @cli.command()
+@click.option(
+    "--format",
+    "-f",
+    help="Output format. Plain is suitable for terminals, manifest for rich "
+    "documentation.",
+    type=click.Choice(["plain", "manifest"]),
+    default="plain",
+)
 @click.option(
     "--select",
     "-s",
@@ -39,6 +48,19 @@ def cli() -> None:
     type=tuple,
     multiple=True,
 )
+@click.option(
+    "--namespace",
+    "-n",
+    help="Namespace.",
+    default=None,
+    multiple=True,
+)
+@click.option(
+    "--disabled-rule",
+    help="Rule to disable.",
+    default=None,
+    multiple=True,
+)
 @click.option(
     "--manifest",
     "-m",
@@ -53,7 +75,14 @@ def cli() -> None:
     is_flag=True,
     default=False,
 )
-def lint(select: tuple[str], manifest: Path, run_dbt_parse: bool) -> None:
+def lint(
+    format: Literal["plain", "manifest"],
+    select: tuple[str],
+    namespace: list[str],
+    disabled_rule: list[str],
+    manifest: Path,
+    run_dbt_parse: bool,
+) -> None:
     """Lint dbt models metadata."""
     manifest_provided = (
         click.get_current_context().get_parameter_source("manifest")
@@ -64,8 +93,52 @@ def lint(select: tuple[str], manifest: Path, run_dbt_parse: bool) -> None:
 
     config = Config()
     config.load()
+    if namespace:
+        config.overload({"rule_namespaces": namespace})
+    if disabled_rule:
+        config.overload({"disabled_rules": disabled_rule})
 
     if run_dbt_parse:
         dbt_parse()
 
-    lint_dbt_project(manifest, config)
+    lint_dbt_project(manifest_path=manifest, config=config, format=format)
+
+
+@cli.command(name="list")
+@click.option(
+    "--namespace",
+    "-n",
+    help="Namespace.",
+    default=None,
+    multiple=True,
+)
+@click.option(
+    "--disabled-rule",
+    help="Rule to disable.",
+    default=None,
+    multiple=True,
+)
+@click.option(
+    "--title",
+    help="Page title (Markdown only).",
+    default=None,
+)
+@click.option(
+    "--format",
+    "-f",
+    help="Output format.",
+    type=click.Choice(["terminal", "markdown"]),
+    default="terminal",
+)
+def list_command(
+    namespace: list[str], disabled_rule: list[str], title: str, format: str
+) -> None:
+    """Display rules list."""
+    config = Config()
+    config.load()
+    if namespace:
+        config.overload({"rule_namespaces": namespace})
+    if disabled_rule:
+        config.overload({"disabled_rules": disabled_rule})
+
+    display_catalog(config, title, format)

src/dbt_score/config.py

@@ -21,7 +21,7 @@ class Config:
 
     def __init__(self) -> None:
         """Initialize the Config object."""
-        self.rule_namespaces: list[str] = ["dbt_score_rules"]
+        self.rule_namespaces: list[str] = ["dbt_score.rules", "dbt_score_rules"]
         self.disabled_rules: list[str] = []
         self.rules_config: dict[str, RuleConfig] = {}
         self.config_file: Path | None = None
@@ -70,3 +70,8 @@ def load(self) -> None:
         config_file = self.get_config_file(Path.cwd())
         if config_file:
             self._load_toml_file(str(config_file))
+
+    def overload(self, values: dict[str, Any]) -> None:
+        """Overload config with additional values."""
+        for key, value in values.items():
+            self.set_option(key, value)

src/dbt_score/formatters/__init__.py

@@ -7,12 +7,16 @@
 
 if typing.TYPE_CHECKING:
     from dbt_score.evaluation import ModelResultsType
-from dbt_score.models import Model
+from dbt_score.models import ManifestLoader, Model
 
 
 class Formatter(ABC):
     """Abstract class to define a formatter."""
 
+    def __init__(self, manifest_loader: ManifestLoader):
+        """Instantiate a formatter."""
+        self._manifest_loader = manifest_loader
+
     @abstractmethod
     def model_evaluated(
         self, model: Model, results: ModelResultsType, score: float

src/dbt_score/formatters/manifest_formatter.py

@@ -0,0 +1,31 @@
+"""Formatter for a manifest.json."""
+
+import copy
+import json
+from typing import Any
+
+from dbt_score.evaluation import ModelResultsType
+from dbt_score.formatters import Formatter
+from dbt_score.models import Model
+
+
+class ManifestFormatter(Formatter):
+    """Formatter to generate manifest.json with score metadata."""
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        """Instantiate a manifest formatter."""
+        self._model_scores: dict[str, float] = {}
+        super().__init__(*args, **kwargs)
+
+    def model_evaluated(
+        self, model: Model, results: ModelResultsType, score: float
+    ) -> None:
+        """Callback when a model has been evaluated."""
+        self._model_scores[model.unique_id] = score
+
+    def project_evaluated(self, score: float) -> None:
+        """Callback when a project has been evaluated."""
+        manifest = copy.copy(self._manifest_loader.raw_manifest)
+        for model_id, score in self._model_scores.items():
+            manifest["nodes"][model_id]["meta"]["score"] = round(score, 1)
+        print(json.dumps(manifest, indent=2))

src/dbt_score/lint.py

@@ -1,16 +1,20 @@
 """Lint dbt models metadata."""
 
 from pathlib import Path
+from typing import Literal
 
 from dbt_score.config import Config
 from dbt_score.evaluation import Evaluation
 from dbt_score.formatters.human_readable_formatter import HumanReadableFormatter
+from dbt_score.formatters.manifest_formatter import ManifestFormatter
 from dbt_score.models import ManifestLoader
 from dbt_score.rule_registry import RuleRegistry
 from dbt_score.scoring import Scorer
 
 
-def lint_dbt_project(manifest_path: Path, config: Config) -> None:
+def lint_dbt_project(
+    manifest_path: Path, config: Config, format: Literal["plain", "manifest"]
+) -> None:
     """Lint dbt manifest."""
     if not manifest_path.exists():
         raise FileNotFoundError(f"Manifest not found at {manifest_path}.")
@@ -20,7 +24,8 @@ def lint_dbt_project(manifest_path: Path, config: Config) -> None:
 
     manifest_loader = ManifestLoader(manifest_path)
 
-    formatter = HumanReadableFormatter()
+    formatters = {"plain": HumanReadableFormatter, "manifest": ManifestFormatter}
+    formatter = formatters[format](manifest_loader=manifest_loader)
 
     scorer = Scorer()
 

src/dbt_score/models.py

@@ -128,6 +128,8 @@ class Model:
         database: The database name of the model.
         schema: The schema name of the model.
         raw_code: The raw code of the model.
+        language: The language of the model, e.g. sql.
+        access: The access level of the model, e.g. public.
         alias: The alias of the model.
         patch_path: The yml path of the model, e.g. `package://model_dir/dir/file.yml`.
         tags: The list of tags attached to the model.
@@ -149,6 +151,8 @@ class Model:
     database: str
     schema: str
     raw_code: str
+    language: str
+    access: str
     alias: str | None = None
     patch_path: str | None = None
     tags: list[str] = field(default_factory=list)
@@ -200,6 +204,8 @@ def from_node(
             database=node_values["database"],
             schema=node_values["schema"],
             raw_code=node_values["raw_code"],
+            language=node_values["language"],
+            access=node_values["access"],
             alias=node_values["alias"],
             patch_path=node_values["patch_path"],
             tags=node_values["tags"],

src/dbt_score/rule.py

@@ -174,6 +174,8 @@ def wrapped_func(self: Rule, *args: Any, **kwargs: Any) -> RuleViolation | None:
                 "severity": severity,
                 "default_config": default_config,
                 "evaluate": wrapped_func,
+                # Save provided evaluate function
+                "_orig_evaluate": func,
                 # Forward origin of the decorated function
                 "__qualname__": func.__qualname__,  # https://peps.python.org/pep-3155/
                 "__module__": func.__module__,