From a86b87544d44335c36bceb1ace6113419a68ea6b Mon Sep 17 00:00:00 2001 From: Quigley Malcolm Date: Fri, 4 Aug 2023 13:56:15 -0700 Subject: [PATCH] Separate generated metrics from metrics for partial parsing I was doing a demo earlier today of this branch (minus this commit) and noticed something odd. When I changes a semantic model, metrics that should have been technically uneffected would get dropped. Basically if I made a change to a semantic model which had metrics in the same file, and then ran parse, those metrics defined in the same file would get dropped. Then with no other changes, if I ran parse again they would come back. What was happening was that parsed metrics and generated metrics were getting tracked the same way on the file objects for partial parsing. In 0787a7c7b67b10a55b0d7727eeb744df831d503c we began dropping all metrics tracked in a file objects when changes to semantic models were detected. Since parsed metrics and generated metrics were being tracked together on the file object, the parsed metrics were getting dropped as well. In this commit we begin separating out the tracking of generated metrics and parsed metrics on the file object, and now only drop the generated metrics when semantic models have a detected change. --- core/dbt/contracts/files.py | 2 ++ core/dbt/contracts/graph/manifest.py | 7 +++++-- core/dbt/parser/partial.py | 4 ++-- core/dbt/parser/schema_yaml_readers.py | 6 +++--- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/core/dbt/contracts/files.py b/core/dbt/contracts/files.py index f54533c38c1..955757a02f7 100644 --- a/core/dbt/contracts/files.py +++ b/core/dbt/contracts/files.py @@ -225,6 +225,8 @@ class SchemaSourceFile(BaseSourceFile): sources: List[str] = field(default_factory=list) exposures: List[str] = field(default_factory=list) metrics: List[str] = field(default_factory=list) + # metrics generated from semantic_model measures + generated_metrics: List[str] = field(default_factory=list) groups: List[str] = field(default_factory=list) # node patches contain models, seeds, snapshots, analyses ndp: List[str] = field(default_factory=list) diff --git a/core/dbt/contracts/graph/manifest.py b/core/dbt/contracts/graph/manifest.py index 20d2dc5f394..12742e725c9 100644 --- a/core/dbt/contracts/graph/manifest.py +++ b/core/dbt/contracts/graph/manifest.py @@ -1331,10 +1331,13 @@ def add_exposure(self, source_file: SchemaSourceFile, exposure: Exposure): self.exposures[exposure.unique_id] = exposure source_file.exposures.append(exposure.unique_id) - def add_metric(self, source_file: SchemaSourceFile, metric: Metric): + def add_metric(self, source_file: SchemaSourceFile, metric: Metric, generated: bool = False): _check_duplicates(metric, self.metrics) self.metrics[metric.unique_id] = metric - source_file.metrics.append(metric.unique_id) + if not generated: + source_file.metrics.append(metric.unique_id) + else: + source_file.generated_metrics.append(metric.unique_id) def add_group(self, source_file: SchemaSourceFile, group: Group): _check_duplicates(group, self.groups) diff --git a/core/dbt/parser/partial.py b/core/dbt/parser/partial.py index b99d876f8ae..3c6bc46bd74 100644 --- a/core/dbt/parser/partial.py +++ b/core/dbt/parser/partial.py @@ -895,11 +895,11 @@ def delete_schema_semantic_model(self, schema_file, semantic_model_dict): elif unique_id in self.saved_manifest.disabled: self.delete_disabled(unique_id, schema_file.file_id) - metrics = schema_file.metrics.copy() + metrics = schema_file.generated_metrics.copy() for unique_id in metrics: if unique_id in self.saved_manifest.metrics: self.saved_manifest.metrics.pop(unique_id) - schema_file.metrics.remove(unique_id) + schema_file.generated_metrics.remove(unique_id) elif unique_id in self.saved_manifest.disabled: self.delete_disabled(unique_id, schema_file.file_id) diff --git a/core/dbt/parser/schema_yaml_readers.py b/core/dbt/parser/schema_yaml_readers.py index 3791900fdd8..927bfdb9a9f 100644 --- a/core/dbt/parser/schema_yaml_readers.py +++ b/core/dbt/parser/schema_yaml_readers.py @@ -303,7 +303,7 @@ def _get_metric_type_params(self, type_params: UnparsedMetricTypeParams) -> Metr # input_measures=?, ) - def parse_metric(self, unparsed: UnparsedMetric): + def parse_metric(self, unparsed: UnparsedMetric, generated: bool = False): package_name = self.project.project_name unique_id = f"{NodeType.Metric}.{package_name}.{unparsed.name}" path = self.yaml.path.relative_path @@ -358,7 +358,7 @@ def parse_metric(self, unparsed: UnparsedMetric): # if the metric is disabled we do not want it included in the manifest, only in the disabled dict if parsed.config.enabled: - self.manifest.add_metric(self.yaml.file, parsed) + self.manifest.add_metric(self.yaml.file, parsed, generated) else: self.manifest.add_disabled(self.yaml.file, parsed) @@ -520,7 +520,7 @@ def _create_metric(self, measure: UnparsedMeasure, enabled: bool) -> None: ) parser = MetricParser(self.schema_parser, yaml=self.yaml) - parser.parse_metric(unparsed=unparsed_metric) + parser.parse_metric(unparsed=unparsed_metric, generated=True) def parse_semantic_model(self, unparsed: UnparsedSemanticModel): package_name = self.project.project_name