From 90f92fe50e29026f28f1b288e19405a46a9f5ce3 Mon Sep 17 00:00:00 2001 From: Courtney Holcomb Date: Fri, 28 Jun 2024 11:56:30 -0700 Subject: [PATCH 1/5] Include SetDefaultGranularityRule in all_rules --- dbt_semantic_interfaces/transformations/pydantic_rule_set.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dbt_semantic_interfaces/transformations/pydantic_rule_set.py b/dbt_semantic_interfaces/transformations/pydantic_rule_set.py index 7c2f4568..9b9f508c 100644 --- a/dbt_semantic_interfaces/transformations/pydantic_rule_set.py +++ b/dbt_semantic_interfaces/transformations/pydantic_rule_set.py @@ -20,6 +20,7 @@ from dbt_semantic_interfaces.transformations.cumulative_type_params import ( SetCumulativeTypeParamsRule, ) +from dbt_semantic_interfaces.transformations.default_granularity import SetDefaultGranularityRule from dbt_semantic_interfaces.transformations.names import LowerCaseNamesRule from dbt_semantic_interfaces.transformations.proxy_measure import CreateProxyMeasureRule from dbt_semantic_interfaces.transformations.rule_set import ( @@ -54,6 +55,7 @@ def secondary_rules(self) -> Sequence[SemanticManifestTransformRule[PydanticSema ConvertMedianToPercentileRule(), AddInputMetricMeasuresRule(), SetCumulativeTypeParamsRule(), + SetDefaultGranularityRule(), ) @property From fdef12680edcec23020421b54cc4faf3aeecda3e Mon Sep 17 00:00:00 2001 From: Courtney Holcomb Date: Fri, 28 Jun 2024 11:58:44 -0700 Subject: [PATCH 2/5] Bump dev version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 76903e1d..95af9f72 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "dbt-semantic-interfaces" -version = "0.6.2.dev2" +version = "0.6.2.dev3" description = 'The shared semantic layer definitions that dbt-core and MetricFlow use' readme = "README.md" requires-python = ">=3.8" From d242236c3bf769dd42c85cbfeab0ba304749f74c Mon Sep 17 00:00:00 2001 From: Courtney Holcomb Date: Fri, 28 Jun 2024 16:54:52 -0700 Subject: [PATCH 3/5] Lint --- dbt_semantic_interfaces/transformations/pydantic_rule_set.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dbt_semantic_interfaces/transformations/pydantic_rule_set.py b/dbt_semantic_interfaces/transformations/pydantic_rule_set.py index 9b9f508c..cc4fb501 100644 --- a/dbt_semantic_interfaces/transformations/pydantic_rule_set.py +++ b/dbt_semantic_interfaces/transformations/pydantic_rule_set.py @@ -20,7 +20,9 @@ from dbt_semantic_interfaces.transformations.cumulative_type_params import ( SetCumulativeTypeParamsRule, ) -from dbt_semantic_interfaces.transformations.default_granularity import SetDefaultGranularityRule +from dbt_semantic_interfaces.transformations.default_granularity import ( + SetDefaultGranularityRule, +) from dbt_semantic_interfaces.transformations.names import LowerCaseNamesRule from dbt_semantic_interfaces.transformations.proxy_measure import CreateProxyMeasureRule from dbt_semantic_interfaces.transformations.rule_set import ( From f20ab6d5a7c7604cae312e96167bcebb267f75af Mon Sep 17 00:00:00 2001 From: Courtney Holcomb Date: Fri, 28 Jun 2024 17:32:05 -0700 Subject: [PATCH 4/5] Better handling for errors during validation Fixes a test that broke CI --- .../transformations/default_granularity.py | 7 ++- .../validations/metrics.py | 47 +++++++++++++------ 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/dbt_semantic_interfaces/transformations/default_granularity.py b/dbt_semantic_interfaces/transformations/default_granularity.py index 0e6b75fd..92a4de9c 100644 --- a/dbt_semantic_interfaces/transformations/default_granularity.py +++ b/dbt_semantic_interfaces/transformations/default_granularity.py @@ -34,7 +34,12 @@ def transform_model(semantic_manifest: PydanticSemanticManifest) -> PydanticSema seen_agg_time_dimensions: Set[TimeDimensionReference] = set() for semantic_model in semantic_manifest.semantic_models: for measure_ref in set(metric.measure_references).intersection(semantic_model.measure_references): - agg_time_dimension_ref = semantic_model.checked_agg_time_dimension_for_measure(measure_ref) + try: + agg_time_dimension_ref = semantic_model.checked_agg_time_dimension_for_measure(measure_ref) + except AssertionError: + # This indicates the agg_time_dimension is misconfigured, which will fail elsewhere. + # Do nothing here to avoid disrupting the validation process. + continue if agg_time_dimension_ref in seen_agg_time_dimensions: continue seen_agg_time_dimensions.add(agg_time_dimension_ref) diff --git a/dbt_semantic_interfaces/validations/metrics.py b/dbt_semantic_interfaces/validations/metrics.py index 4e68afaa..275446eb 100644 --- a/dbt_semantic_interfaces/validations/metrics.py +++ b/dbt_semantic_interfaces/validations/metrics.py @@ -584,8 +584,8 @@ class DefaultGranularityRule(SemanticManifestValidationRule[SemanticManifestT], def _min_queryable_granularity_for_metric( metric: Metric, metric_index: Dict[MetricReference, Metric], - measure_to_agg_time_dimension: Dict[MeasureReference, Dimension], - ) -> TimeGranularity: + measure_to_agg_time_dimension: Dict[MeasureReference, Optional[Dimension]], + ) -> Optional[TimeGranularity]: """Get the minimum time granularity this metric is allowed to be queried with. This should be the largest granularity that any of the metric's agg_time_dimensions is defined at. @@ -594,14 +594,18 @@ def _min_queryable_granularity_for_metric( min_queryable_granularity: Optional[TimeGranularity] = None for measure_reference in PydanticMetric.all_input_measures_for_metric(metric=metric, metric_index=metric_index): agg_time_dimension = measure_to_agg_time_dimension.get(measure_reference) - assert agg_time_dimension, f"Measure '{measure_reference.element_name}' not found in semantic manifest." - if not agg_time_dimension.type_params: - continue - defined_time_granularity = agg_time_dimension.type_params.time_granularity + if not agg_time_dimension: + # This indicates the measure or agg_time_dimension were invalid, so we can't determine granularity. + return None + defined_time_granularity = ( + agg_time_dimension.type_params.time_granularity + if agg_time_dimension.type_params + else TimeGranularity.DAY + ) if not min_queryable_granularity or defined_time_granularity.to_int() > min_queryable_granularity.to_int(): min_queryable_granularity = defined_time_granularity - return min_queryable_granularity or TimeGranularity.DAY + return min_queryable_granularity @staticmethod @validate_safely( @@ -610,7 +614,7 @@ def _min_queryable_granularity_for_metric( def _validate_metric( metric: Metric, metric_index: Dict[MetricReference, Metric], - measure_to_agg_time_dimension: Dict[MeasureReference, Dimension], + measure_to_agg_time_dimension: Dict[MeasureReference, Optional[Dimension]], ) -> Sequence[ValidationIssue]: # noqa: D issues: List[ValidationIssue] = [] context = MetricContext( @@ -622,6 +626,17 @@ def _validate_metric( min_queryable_granularity = DefaultGranularityRule._min_queryable_granularity_for_metric( metric=metric, metric_index=metric_index, measure_to_agg_time_dimension=measure_to_agg_time_dimension ) + if not min_queryable_granularity: + issues.append( + ValidationError( + context=context, + message=( + f"Unable to validate `default_granularity` for metric '{metric.name}' due to " + "misconfiguration with measures or related agg_time_dimensions." + ), + ) + ) + return issues valid_granularities = [ granularity.name for granularity in TimeGranularity @@ -653,15 +668,19 @@ def validate_manifest(semantic_manifest: SemanticManifestT) -> Sequence[Validati """ issues: List[ValidationIssue] = [] - measure_to_agg_time_dimension: Dict[MeasureReference, Dimension] = {} + measure_to_agg_time_dimension: Dict[MeasureReference, Optional[Dimension]] = {} for semantic_model in semantic_manifest.semantic_models: dimension_index = {DimensionReference(dimension.name): dimension for dimension in semantic_model.dimensions} for measure in semantic_model.measures: - agg_time_dimension_ref = semantic_model.checked_agg_time_dimension_for_measure(measure.reference) - agg_time_dimension = dimension_index.get(agg_time_dimension_ref.dimension_reference) - assert ( - agg_time_dimension - ), f"Dimension '{agg_time_dimension_ref.element_name}' not found in semantic manifest." + try: + agg_time_dimension_ref = semantic_model.checked_agg_time_dimension_for_measure(measure.reference) + agg_time_dimension: Optional[Dimension] = dimension_index[ + agg_time_dimension_ref.dimension_reference + ] + except (AssertionError, KeyError): + # If the agg_time_dimension is not set or does not exist, this will be validated elsewhere. + # Here, swallow the error to avoid disrupting the validation process. + agg_time_dimension = None measure_to_agg_time_dimension[measure.reference] = agg_time_dimension metric_index = {MetricReference(metric.name): metric for metric in semantic_manifest.metrics} From 55242af821a486f1a22e37a88e09562d2ad5d266 Mon Sep 17 00:00:00 2001 From: Courtney Holcomb Date: Fri, 28 Jun 2024 17:33:12 -0700 Subject: [PATCH 5/5] Fix test semantic model --- tests/validations/test_measures.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/validations/test_measures.py b/tests/validations/test_measures.py index d01bf3fc..266894ce 100644 --- a/tests/validations/test_measures.py +++ b/tests/validations/test_measures.py @@ -72,6 +72,8 @@ def test_measures_only_exist_in_one_semantic_model() -> None: # noqa: D node_relation: schema_name: some_schema alias: source_table + defaults: + agg_time_dimension: ds entities: - name: example_entity type: primary