From 326fb11b36fcb4689afdfa7e7c9d36271cbbffea Mon Sep 17 00:00:00 2001 From: Matthieu Caneill Date: Wed, 7 Aug 2024 16:30:51 +0200 Subject: [PATCH 1/7] Add JSON formatter for machine-readable output --- src/dbt_score/cli.py | 4 +- src/dbt_score/formatters/json_formatter.py | 99 ++++++++++++++++++++++ src/dbt_score/lint.py | 2 + tests/formatters/test_json_formatter.py | 60 +++++++++++++ 4 files changed, 163 insertions(+), 2 deletions(-) create mode 100644 src/dbt_score/formatters/json_formatter.py create mode 100644 tests/formatters/test_json_formatter.py diff --git a/src/dbt_score/cli.py b/src/dbt_score/cli.py index 9694b5e..8b439ae 100644 --- a/src/dbt_score/cli.py +++ b/src/dbt_score/cli.py @@ -39,8 +39,8 @@ def cli() -> None: "--format", "-f", help="Output format. Plain is suitable for terminals, manifest for rich " - "documentation.", - type=click.Choice(["plain", "manifest", "ascii"]), + "documentation, json for machine-readable output.", + type=click.Choice(["plain", "manifest", "ascii", "json"]), default="plain", ) @click.option( diff --git a/src/dbt_score/formatters/json_formatter.py b/src/dbt_score/formatters/json_formatter.py new file mode 100644 index 0000000..38e1637 --- /dev/null +++ b/src/dbt_score/formatters/json_formatter.py @@ -0,0 +1,99 @@ +"""JSON formatter. + +Shape of the JSON output: + +{ + "models": { + "model_foo": { + "score": 0.5, + "badge": "🥈", + "results": { + "rule1": { + "result": "OK", + "severity": null + "message": null + }, + "rule2": { + "result": "WARN", + "severity": "medium", + "message": "Model lacks a description." + } + ] + }, + "model_bar": { + "score": 0.0, + "badge": "🥉", + "results": { + "rule1": { + "result": "ERR", + "message": "Exception message" + } + } + } + }, + "project": { + "score": 0.25, + "badge": "🥉" + } +} +""" + + +import json +from typing import Any + +from dbt_score.evaluation import ModelResultsType +from dbt_score.formatters import Formatter +from dbt_score.models import Model +from dbt_score.rule import RuleViolation +from dbt_score.scoring import Score + + +class JSONFormatter(Formatter): + """Formatter for JSON output.""" + + def __init__(self, *args: Any, **kwargs: Any): + """Instantiate formatter.""" + super().__init__(*args, **kwargs) + self._model_results: dict[str, dict[str, Any]] = {} + self._project_results: dict[str, Any] + + def model_evaluated( + self, model: Model, results: ModelResultsType, score: Score + ) -> None: + """Callback when a model has been evaluated.""" + self._model_results[model.name] = { + "score": score.value, + "badge": score.badge, + "results": {}, + } + for rule, result in results.items(): + if result is None: + self._model_results[model.name]["results"][rule.source()] = { + "result": "OK", + "severity": None, + "message": None, + } + elif isinstance(result, RuleViolation): + self._model_results[model.name]["results"][rule.source()] = { + "result": "WARN", + "severity": rule.severity.name.lower(), + "message": result.message, + } + else: + self._model_results[model.name]["results"][rule.source()] = { + "result": "ERR", + "message": str(result), + } + + def project_evaluated(self, score: Score) -> None: + """Callback when a project has been evaluated.""" + self._project_results = { + "score": score.value, + "badge": score.badge, + } + document = { + "models": self._model_results, + "project": self._project_results, + } + print(json.dumps(document, indent=2, ensure_ascii=False)) diff --git a/src/dbt_score/lint.py b/src/dbt_score/lint.py index 5b6d95f..53d09f0 100644 --- a/src/dbt_score/lint.py +++ b/src/dbt_score/lint.py @@ -7,6 +7,7 @@ from dbt_score.evaluation import Evaluation from dbt_score.formatters.ascii_formatter import ASCIIFormatter from dbt_score.formatters.human_readable_formatter import HumanReadableFormatter +from dbt_score.formatters.json_formatter import JSONFormatter from dbt_score.formatters.manifest_formatter import ManifestFormatter from dbt_score.models import ManifestLoader from dbt_score.rule_registry import RuleRegistry @@ -32,6 +33,7 @@ def lint_dbt_project( "plain": HumanReadableFormatter, "manifest": ManifestFormatter, "ascii": ASCIIFormatter, + "json": JSONFormatter, } formatter = formatters[format](manifest_loader=manifest_loader, config=config) diff --git a/tests/formatters/test_json_formatter.py b/tests/formatters/test_json_formatter.py new file mode 100644 index 0000000..b4d6fbd --- /dev/null +++ b/tests/formatters/test_json_formatter.py @@ -0,0 +1,60 @@ +"""Unit tests for the JSON formatter.""" + +from typing import Type + +from dbt_score.formatters.json_formatter import JSONFormatter +from dbt_score.rule import Rule, RuleViolation +from dbt_score.scoring import Score + + +def test_json_formatter( + capsys, + default_config, + manifest_loader, + model1, + rule_severity_low, + rule_severity_medium, + rule_severity_critical, +): + """Ensure the formatter has the correct output after model evaluation.""" + formatter = JSONFormatter(manifest_loader=manifest_loader, config=default_config) + results: dict[Type[Rule], RuleViolation | Exception | None] = { + rule_severity_low: None, + rule_severity_medium: Exception("Oh noes"), + rule_severity_critical: RuleViolation("Error"), + } + formatter.model_evaluated(model1, results, Score(10.0, "🥇")) + formatter.project_evaluated(Score(10.0, "🥇")) + stdout = capsys.readouterr().out + assert ( + stdout + == """{ + "models": { + "model1": { + "score": 10.0, + "badge": "🥇", + "results": { + "tests.conftest.rule_severity_low": { + "result": "OK", + "severity": null, + "message": null + }, + "tests.conftest.rule_severity_medium": { + "result": "ERR", + "message": "Oh noes" + }, + "tests.conftest.rule_severity_critical": { + "result": "WARN", + "severity": "critical", + "message": "Error" + } + } + } + }, + "project": { + "score": 10.0, + "badge": "🥇" + } +} +""" + ) From 36f27e5f9376e0ecce42ff60106a41dfe6d3a08d Mon Sep 17 00:00:00 2001 From: Matthieu Caneill Date: Wed, 7 Aug 2024 16:41:00 +0200 Subject: [PATCH 2/7] Always display severity --- src/dbt_score/formatters/json_formatter.py | 3 ++- tests/formatters/test_json_formatter.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/dbt_score/formatters/json_formatter.py b/src/dbt_score/formatters/json_formatter.py index 38e1637..526593d 100644 --- a/src/dbt_score/formatters/json_formatter.py +++ b/src/dbt_score/formatters/json_formatter.py @@ -71,7 +71,7 @@ def model_evaluated( if result is None: self._model_results[model.name]["results"][rule.source()] = { "result": "OK", - "severity": None, + "severity": rule.severity.name.lower(), "message": None, } elif isinstance(result, RuleViolation): @@ -83,6 +83,7 @@ def model_evaluated( else: self._model_results[model.name]["results"][rule.source()] = { "result": "ERR", + "severity": rule.severity.name.lower(), "message": str(result), } diff --git a/tests/formatters/test_json_formatter.py b/tests/formatters/test_json_formatter.py index b4d6fbd..2ec055b 100644 --- a/tests/formatters/test_json_formatter.py +++ b/tests/formatters/test_json_formatter.py @@ -36,11 +36,12 @@ def test_json_formatter( "results": { "tests.conftest.rule_severity_low": { "result": "OK", - "severity": null, + "severity": "low", "message": null }, "tests.conftest.rule_severity_medium": { "result": "ERR", + "severity": "medium", "message": "Oh noes" }, "tests.conftest.rule_severity_critical": { From 81a5a86bb6cafdc48c519ae22e97a23bcfd653c4 Mon Sep 17 00:00:00 2001 From: Matthieu Caneill Date: Thu, 8 Aug 2024 11:12:47 +0200 Subject: [PATCH 3/7] Apply suggestions from code review Co-authored-by: Jochem van Dooren --- src/dbt_score/formatters/json_formatter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dbt_score/formatters/json_formatter.py b/src/dbt_score/formatters/json_formatter.py index 526593d..6ed6cb3 100644 --- a/src/dbt_score/formatters/json_formatter.py +++ b/src/dbt_score/formatters/json_formatter.py @@ -5,7 +5,7 @@ { "models": { "model_foo": { - "score": 0.5, + "score": 5.0, "badge": "🥈", "results": { "rule1": { @@ -32,7 +32,7 @@ } }, "project": { - "score": 0.25, + "score": 2.5, "badge": "🥉" } } From c887978c57735b6be4c281fb76e909e816e1c4bf Mon Sep 17 00:00:00 2001 From: Matthieu Caneill Date: Thu, 8 Aug 2024 11:15:39 +0200 Subject: [PATCH 4/7] Factor out severity --- src/dbt_score/formatters/json_formatter.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/dbt_score/formatters/json_formatter.py b/src/dbt_score/formatters/json_formatter.py index 6ed6cb3..53f41ec 100644 --- a/src/dbt_score/formatters/json_formatter.py +++ b/src/dbt_score/formatters/json_formatter.py @@ -68,22 +68,23 @@ def model_evaluated( "results": {}, } for rule, result in results.items(): + severity = rule.severity.name.lower() if result is None: self._model_results[model.name]["results"][rule.source()] = { "result": "OK", - "severity": rule.severity.name.lower(), + "severity": severity, "message": None, } elif isinstance(result, RuleViolation): self._model_results[model.name]["results"][rule.source()] = { "result": "WARN", - "severity": rule.severity.name.lower(), + "severity": severity, "message": result.message, } else: self._model_results[model.name]["results"][rule.source()] = { "result": "ERR", - "severity": rule.severity.name.lower(), + "severity": severity, "message": str(result), } From ad8d0abdac8312b1e074aa76befbaac78f9e6471 Mon Sep 17 00:00:00 2001 From: Matthieu Caneill Date: Thu, 8 Aug 2024 11:15:51 +0200 Subject: [PATCH 5/7] Add docs page --- docs/reference/formatters/json_formatter.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 docs/reference/formatters/json_formatter.md diff --git a/docs/reference/formatters/json_formatter.md b/docs/reference/formatters/json_formatter.md new file mode 100644 index 0000000..5b7cdc2 --- /dev/null +++ b/docs/reference/formatters/json_formatter.md @@ -0,0 +1,3 @@ +# JSON formatter + +::: dbt_score.formatters.json_formatter From 61d6d4505a8b62e805943ded178db972f710c80c Mon Sep 17 00:00:00 2001 From: Matthieu Caneill Date: Thu, 8 Aug 2024 11:29:45 +0200 Subject: [PATCH 6/7] Add "pass" property for models and project --- src/dbt_score/formatters/json_formatter.py | 7 ++++++- tests/formatters/test_json_formatter.py | 5 ++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/dbt_score/formatters/json_formatter.py b/src/dbt_score/formatters/json_formatter.py index 53f41ec..9e6f8b3 100644 --- a/src/dbt_score/formatters/json_formatter.py +++ b/src/dbt_score/formatters/json_formatter.py @@ -7,6 +7,7 @@ "model_foo": { "score": 5.0, "badge": "🥈", + "pass": true, "results": { "rule1": { "result": "OK", @@ -23,6 +24,7 @@ "model_bar": { "score": 0.0, "badge": "🥉", + "pass": false, "results": { "rule1": { "result": "ERR", @@ -33,7 +35,8 @@ }, "project": { "score": 2.5, - "badge": "🥉" + "badge": "🥉", + "pass": false } } """ @@ -65,6 +68,7 @@ def model_evaluated( self._model_results[model.name] = { "score": score.value, "badge": score.badge, + "pass": score.value >= self._config.fail_any_model_under, "results": {}, } for rule, result in results.items(): @@ -93,6 +97,7 @@ def project_evaluated(self, score: Score) -> None: self._project_results = { "score": score.value, "badge": score.badge, + "pass": score.value >= self._config.fail_project_under, } document = { "models": self._model_results, diff --git a/tests/formatters/test_json_formatter.py b/tests/formatters/test_json_formatter.py index 2ec055b..696d746 100644 --- a/tests/formatters/test_json_formatter.py +++ b/tests/formatters/test_json_formatter.py @@ -26,6 +26,7 @@ def test_json_formatter( formatter.model_evaluated(model1, results, Score(10.0, "🥇")) formatter.project_evaluated(Score(10.0, "🥇")) stdout = capsys.readouterr().out + print() assert ( stdout == """{ @@ -33,6 +34,7 @@ def test_json_formatter( "model1": { "score": 10.0, "badge": "🥇", + "pass": true, "results": { "tests.conftest.rule_severity_low": { "result": "OK", @@ -54,7 +56,8 @@ def test_json_formatter( }, "project": { "score": 10.0, - "badge": "🥇" + "badge": "🥇", + "pass": true } } """ From f6cf74ad8b4d3630683920349cd8a5d9e1c73ec6 Mon Sep 17 00:00:00 2001 From: Matthieu Caneill Date: Thu, 8 Aug 2024 12:29:40 +0200 Subject: [PATCH 7/7] Update changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 323d145..9f4cc1f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,8 @@ and this project adheres to ## [Unreleased] - Add null check before calling `project_evaluated` in the `evaluate` method to - prevent errors when no models are found. See PR #64. + prevent errors when no models are found. (#64) +- Add JSON formatter for machine-readable output. (#68) ## [0.3.0] - 2024-06-20