Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add primary_key to manifest #10096

Merged
merged 9 commits into from
May 10, 2024
Merged
6 changes: 6 additions & 0 deletions .changes/unreleased/Features-20240506-175642.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Features
body: serialize inferred primary key
time: 2024-05-06T17:56:42.757673-05:00
custom:
Author: dave-connors-3
Issue: "9824"
1 change: 1 addition & 0 deletions core/dbt/artifacts/resources/v1/model.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ class Model(CompiledResource):
latest_version: Optional[NodeVersion] = None
deprecation_date: Optional[datetime] = None
defer_relation: Optional[DeferRelation] = None
primary_key: List[str] = field(default_factory=list)
dave-connors-3 marked this conversation as resolved.
Show resolved Hide resolved

def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None):
dct = super().__post_serialize__(dct, context)
Expand Down
29 changes: 29 additions & 0 deletions core/dbt/parser/manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
)
from dbt.contracts.graph.nodes import (
Exposure,
GenericTestNode,
Macro,
ManifestNode,
Metric,
Expand Down Expand Up @@ -466,6 +467,7 @@
self.process_docs(self.root_project)
self.process_metrics(self.root_project)
self.process_saved_queries(self.root_project)
self.process_model_inferred_primary_keys()
self.check_valid_group_config()
self.check_valid_access_property()

Expand Down Expand Up @@ -1149,6 +1151,15 @@
# 2. process `group_by` of SavedQuery for `depends_on``
_process_metrics_for_node(self.manifest, current_project, saved_query)

def process_model_inferred_primary_keys(self):
"""Processes Model nodes to populate their `primary_key`."""
for node in self.manifest.nodes.values():
if not isinstance(node, ModelNode):
continue

Check warning on line 1158 in core/dbt/parser/manifest.py

View check run for this annotation

Codecov / codecov/patch

core/dbt/parser/manifest.py#L1158

Added line #L1158 was not covered by tests
generic_tests = self._get_generic_tests_for_model(node)
primary_key = node.infer_primary_key(generic_tests)
node.primary_key = sorted(primary_key)

def update_semantic_model(self, semantic_model) -> None:
# This has to be done at the end of parsing because the referenced model
# might have alias/schema/database fields that are updated by yaml config.
Expand Down Expand Up @@ -1344,6 +1355,24 @@
write_file(path, json.dumps(self._perf_info, cls=dbt.utils.JSONEncoder, indent=4))
fire_event(ParsePerfInfoPath(path=path))

def _get_generic_tests_for_model(
self,
model: ModelNode,
) -> List[GenericTestNode]:
"""Return a list of generic tests that are attached to the given model, including disabled tests"""
tests = []
for _, node in self.manifest.nodes.items():
if isinstance(node, GenericTestNode) and node.attached_node == model.unique_id:
tests.append(node)

Check warning on line 1366 in core/dbt/parser/manifest.py

View check run for this annotation

Codecov / codecov/patch

core/dbt/parser/manifest.py#L1366

Added line #L1366 was not covered by tests
for _, nodes in self.manifest.disabled.items():
for disabled_node in nodes:
if (

Check warning on line 1369 in core/dbt/parser/manifest.py

View check run for this annotation

Codecov / codecov/patch

core/dbt/parser/manifest.py#L1368-L1369

Added lines #L1368 - L1369 were not covered by tests
isinstance(disabled_node, GenericTestNode)
and disabled_node.attached_node == model.unique_id
):
tests.append(disabled_node)

Check warning on line 1373 in core/dbt/parser/manifest.py

View check run for this annotation

Codecov / codecov/patch

core/dbt/parser/manifest.py#L1373

Added line #L1373 was not covered by tests
return tests


def invalid_target_fail_unless_test(
node,
Expand Down
Loading
Loading