Skip to content

Commit

Permalink
Switch to using metrics_from_measures in SchemaSourceFile
Browse files Browse the repository at this point in the history
  • Loading branch information
gshank committed Jul 16, 2024
1 parent be06cbd commit af8a7bc
Show file tree
Hide file tree
Showing 4 changed files with 49 additions and 15 deletions.
23 changes: 23 additions & 0 deletions core/dbt/contracts/files.py
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,7 @@ class SchemaSourceFile(BaseSourceFile):
metrics: List[str] = field(default_factory=list)
# metrics generated from semantic_model measures
generated_metrics: List[str] = field(default_factory=list)
metrics_from_measures: Dict[str, Any] = field(default_factory=dict)
groups: List[str] = field(default_factory=list)
# node patches contain models, seeds, snapshots, analyses
ndp: List[str] = field(default_factory=list)
Expand Down Expand Up @@ -259,6 +260,28 @@ def get_tests(self, yaml_key, name):
return self.data_tests[yaml_key][name]
return []

def add_metrics_from_measures(self, semantic_model_name: str, metric_unique_id: str):
if self.generated_metrics:
self.fix_metrics_from_measures()
if semantic_model_name not in self.metrics_from_measures:
self.metrics_from_measures[semantic_model_name] = []
self.metrics_from_measures[semantic_model_name].append(metric_unique_id)

def fix_metrics_from_measures(self):
# Loop through yaml dictionary looking for matching generated metrics
generated_metrics = self.generated_metrics
self.generated_metrics = []
for metric_unique_id in generated_metrics:
parts = metric_unique_id.split(".")
metric_name = parts[-1]
if "semantic_models" in self.dict_from_yaml:
for sem_model in self.dict_from_yaml["semantic_model"]:
if "measures" in sem_model:
for measure in sem_model["measures"]:
if measure["name"] == metric_name:
self.add_metrics_from_measures(sem_model["name"], metric_unique_id)
break

def get_key_and_name_for_test(self, test_unique_id):
yaml_key = None
block_name = None
Expand Down
8 changes: 5 additions & 3 deletions core/dbt/contracts/graph/manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -1565,13 +1565,15 @@ def add_exposure(self, source_file: SchemaSourceFile, exposure: Exposure):
self.exposures[exposure.unique_id] = exposure
source_file.exposures.append(exposure.unique_id)

def add_metric(self, source_file: SchemaSourceFile, metric: Metric, generated: bool = False):
def add_metric(
self, source_file: SchemaSourceFile, metric: Metric, generated_from: Optional[str] = None
):
_check_duplicates(metric, self.metrics)
self.metrics[metric.unique_id] = metric
if not generated:
if not generated_from:
source_file.metrics.append(metric.unique_id)
else:
source_file.generated_metrics.append(metric.unique_id)
source_file.add_metrics_from_measures(generated_from, metric.unique_id)

def add_group(self, source_file: SchemaSourceFile, group: Group):
_check_duplicates(group, self.groups)
Expand Down
16 changes: 9 additions & 7 deletions core/dbt/parser/partial.py
Original file line number Diff line number Diff line change
Expand Up @@ -968,13 +968,15 @@ def delete_schema_semantic_model(self, schema_file, semantic_model_dict):
elif unique_id in self.saved_manifest.disabled:
self.delete_disabled(unique_id, schema_file.file_id)

metrics = schema_file.generated_metrics.copy()
for unique_id in metrics:
if unique_id in self.saved_manifest.metrics:
self.saved_manifest.metrics.pop(unique_id)
schema_file.generated_metrics.remove(unique_id)
elif unique_id in self.saved_manifest.disabled:
self.delete_disabled(unique_id, schema_file.file_id)
if schema_file.generated_metrics:
schema_file.fix_metrics_from_measures()
if semantic_model_name in schema_file.metrics_from_measures:
for unique_id in schema_file.metrics_from_measures[semantic_model_name]:
if unique_id in self.saved_manifest.metrics:
self.saved_manifest.metrics.pop(unique_id)
elif unique_id in self.saved_manifest.disabled:
self.delete_disabled(unique_id, schema_file.file_id)
del schema_file.metrics_from_measures[semantic_model_name]

def delete_schema_unit_test(self, schema_file, unit_test_dict):
unit_test_name = unit_test_dict["name"]
Expand Down
17 changes: 12 additions & 5 deletions core/dbt/parser/schema_yaml_readers.py
Original file line number Diff line number Diff line change
Expand Up @@ -389,7 +389,7 @@ def _get_metric_type_params(self, unparsed_metric: UnparsedMetric) -> MetricType
# input_measures=?,
)

def parse_metric(self, unparsed: UnparsedMetric, generated: bool = False) -> None:
def parse_metric(self, unparsed: UnparsedMetric, generated_from: Optional[str] = None) -> None:
package_name = self.project.project_name
unique_id = f"{NodeType.Metric}.{package_name}.{unparsed.name}"
path = self.yaml.path.relative_path
Expand Down Expand Up @@ -446,7 +446,7 @@ def parse_metric(self, unparsed: UnparsedMetric, generated: bool = False) -> Non
# if the metric is disabled we do not want it included in the manifest, only in the disabled dict
assert isinstance(self.yaml.file, SchemaSourceFile)
if parsed.config.enabled:
self.manifest.add_metric(self.yaml.file, parsed, generated)
self.manifest.add_metric(self.yaml.file, parsed, generated_from)
else:
self.manifest.add_disabled(self.yaml.file, parsed)

Expand Down Expand Up @@ -601,7 +601,12 @@ def _get_measures(self, unparsed_measures: List[UnparsedMeasure]) -> List[Measur
)
return measures

def _create_metric(self, measure: UnparsedMeasure, enabled: bool) -> None:
def _create_metric(
self,
measure: UnparsedMeasure,
enabled: bool,
semantic_model_name: str,
) -> None:
unparsed_metric = UnparsedMetric(
name=measure.name,
label=measure.name,
Expand All @@ -612,7 +617,7 @@ def _create_metric(self, measure: UnparsedMeasure, enabled: bool) -> None:
)

parser = MetricParser(self.schema_parser, yaml=self.yaml)
parser.parse_metric(unparsed=unparsed_metric, generated=True)
parser.parse_metric(unparsed=unparsed_metric, generated_from=semantic_model_name)

def _generate_semantic_model_config(
self, target: UnparsedSemanticModel, fqn: List[str], package_name: str, rendered: bool
Expand Down Expand Up @@ -708,7 +713,9 @@ def parse_semantic_model(self, unparsed: UnparsedSemanticModel) -> None:
# Create a metric for each measure with `create_metric = True`
for measure in unparsed.measures:
if measure.create_metric is True:
self._create_metric(measure=measure, enabled=parsed.config.enabled)
self._create_metric(
measure=measure, enabled=parsed.config.enabled, semantic_model_name=parsed.name
)

def parse(self) -> None:
for data in self.get_key_dicts():
Expand Down

0 comments on commit af8a7bc

Please sign in to comment.