From 3eee0f56cb1e16ecdf262b020aedda0054dff5fb Mon Sep 17 00:00:00 2001 From: Courtney Holcomb Date: Fri, 28 Jun 2024 17:36:19 -0700 Subject: [PATCH] Include `SetDefaultGranularityRule` in `all_rules` (#302) ### Description Missed this - needs to be manually added to `all_rules` in order to get applied in MF. ### Checklist - [x] I have read [the contributing guide](https://github.com/dbt-labs/dbt-semantic-interfaces/blob/main/CONTRIBUTING.md) and understand what's expected of me - [x] I have signed the [CLA](https://docs.getdbt.com/docs/contributor-license-agreements) - [x] This PR includes tests, or tests are not required/relevant for this PR - [ ] I have run `changie new` to [create a changelog entry](https://github.com/dbt-labs/dbt-semantic-interfaces/blob/main/CONTRIBUTING.md#adding-a-changelog-entry) --- .../transformations/default_granularity.py | 7 ++- .../transformations/pydantic_rule_set.py | 4 ++ .../validations/metrics.py | 47 +++++++++++++------ pyproject.toml | 2 +- tests/validations/test_measures.py | 2 + 5 files changed, 46 insertions(+), 16 deletions(-) diff --git a/dbt_semantic_interfaces/transformations/default_granularity.py b/dbt_semantic_interfaces/transformations/default_granularity.py index 0e6b75fd..92a4de9c 100644 --- a/dbt_semantic_interfaces/transformations/default_granularity.py +++ b/dbt_semantic_interfaces/transformations/default_granularity.py @@ -34,7 +34,12 @@ def transform_model(semantic_manifest: PydanticSemanticManifest) -> PydanticSema seen_agg_time_dimensions: Set[TimeDimensionReference] = set() for semantic_model in semantic_manifest.semantic_models: for measure_ref in set(metric.measure_references).intersection(semantic_model.measure_references): - agg_time_dimension_ref = semantic_model.checked_agg_time_dimension_for_measure(measure_ref) + try: + agg_time_dimension_ref = semantic_model.checked_agg_time_dimension_for_measure(measure_ref) + except AssertionError: + # This indicates the agg_time_dimension is misconfigured, which will fail elsewhere. + # Do nothing here to avoid disrupting the validation process. + continue if agg_time_dimension_ref in seen_agg_time_dimensions: continue seen_agg_time_dimensions.add(agg_time_dimension_ref) diff --git a/dbt_semantic_interfaces/transformations/pydantic_rule_set.py b/dbt_semantic_interfaces/transformations/pydantic_rule_set.py index 7c2f4568..cc4fb501 100644 --- a/dbt_semantic_interfaces/transformations/pydantic_rule_set.py +++ b/dbt_semantic_interfaces/transformations/pydantic_rule_set.py @@ -20,6 +20,9 @@ from dbt_semantic_interfaces.transformations.cumulative_type_params import ( SetCumulativeTypeParamsRule, ) +from dbt_semantic_interfaces.transformations.default_granularity import ( + SetDefaultGranularityRule, +) from dbt_semantic_interfaces.transformations.names import LowerCaseNamesRule from dbt_semantic_interfaces.transformations.proxy_measure import CreateProxyMeasureRule from dbt_semantic_interfaces.transformations.rule_set import ( @@ -54,6 +57,7 @@ def secondary_rules(self) -> Sequence[SemanticManifestTransformRule[PydanticSema ConvertMedianToPercentileRule(), AddInputMetricMeasuresRule(), SetCumulativeTypeParamsRule(), + SetDefaultGranularityRule(), ) @property diff --git a/dbt_semantic_interfaces/validations/metrics.py b/dbt_semantic_interfaces/validations/metrics.py index 4e68afaa..275446eb 100644 --- a/dbt_semantic_interfaces/validations/metrics.py +++ b/dbt_semantic_interfaces/validations/metrics.py @@ -584,8 +584,8 @@ class DefaultGranularityRule(SemanticManifestValidationRule[SemanticManifestT], def _min_queryable_granularity_for_metric( metric: Metric, metric_index: Dict[MetricReference, Metric], - measure_to_agg_time_dimension: Dict[MeasureReference, Dimension], - ) -> TimeGranularity: + measure_to_agg_time_dimension: Dict[MeasureReference, Optional[Dimension]], + ) -> Optional[TimeGranularity]: """Get the minimum time granularity this metric is allowed to be queried with. This should be the largest granularity that any of the metric's agg_time_dimensions is defined at. @@ -594,14 +594,18 @@ def _min_queryable_granularity_for_metric( min_queryable_granularity: Optional[TimeGranularity] = None for measure_reference in PydanticMetric.all_input_measures_for_metric(metric=metric, metric_index=metric_index): agg_time_dimension = measure_to_agg_time_dimension.get(measure_reference) - assert agg_time_dimension, f"Measure '{measure_reference.element_name}' not found in semantic manifest." - if not agg_time_dimension.type_params: - continue - defined_time_granularity = agg_time_dimension.type_params.time_granularity + if not agg_time_dimension: + # This indicates the measure or agg_time_dimension were invalid, so we can't determine granularity. + return None + defined_time_granularity = ( + agg_time_dimension.type_params.time_granularity + if agg_time_dimension.type_params + else TimeGranularity.DAY + ) if not min_queryable_granularity or defined_time_granularity.to_int() > min_queryable_granularity.to_int(): min_queryable_granularity = defined_time_granularity - return min_queryable_granularity or TimeGranularity.DAY + return min_queryable_granularity @staticmethod @validate_safely( @@ -610,7 +614,7 @@ def _min_queryable_granularity_for_metric( def _validate_metric( metric: Metric, metric_index: Dict[MetricReference, Metric], - measure_to_agg_time_dimension: Dict[MeasureReference, Dimension], + measure_to_agg_time_dimension: Dict[MeasureReference, Optional[Dimension]], ) -> Sequence[ValidationIssue]: # noqa: D issues: List[ValidationIssue] = [] context = MetricContext( @@ -622,6 +626,17 @@ def _validate_metric( min_queryable_granularity = DefaultGranularityRule._min_queryable_granularity_for_metric( metric=metric, metric_index=metric_index, measure_to_agg_time_dimension=measure_to_agg_time_dimension ) + if not min_queryable_granularity: + issues.append( + ValidationError( + context=context, + message=( + f"Unable to validate `default_granularity` for metric '{metric.name}' due to " + "misconfiguration with measures or related agg_time_dimensions." + ), + ) + ) + return issues valid_granularities = [ granularity.name for granularity in TimeGranularity @@ -653,15 +668,19 @@ def validate_manifest(semantic_manifest: SemanticManifestT) -> Sequence[Validati """ issues: List[ValidationIssue] = [] - measure_to_agg_time_dimension: Dict[MeasureReference, Dimension] = {} + measure_to_agg_time_dimension: Dict[MeasureReference, Optional[Dimension]] = {} for semantic_model in semantic_manifest.semantic_models: dimension_index = {DimensionReference(dimension.name): dimension for dimension in semantic_model.dimensions} for measure in semantic_model.measures: - agg_time_dimension_ref = semantic_model.checked_agg_time_dimension_for_measure(measure.reference) - agg_time_dimension = dimension_index.get(agg_time_dimension_ref.dimension_reference) - assert ( - agg_time_dimension - ), f"Dimension '{agg_time_dimension_ref.element_name}' not found in semantic manifest." + try: + agg_time_dimension_ref = semantic_model.checked_agg_time_dimension_for_measure(measure.reference) + agg_time_dimension: Optional[Dimension] = dimension_index[ + agg_time_dimension_ref.dimension_reference + ] + except (AssertionError, KeyError): + # If the agg_time_dimension is not set or does not exist, this will be validated elsewhere. + # Here, swallow the error to avoid disrupting the validation process. + agg_time_dimension = None measure_to_agg_time_dimension[measure.reference] = agg_time_dimension metric_index = {MetricReference(metric.name): metric for metric in semantic_manifest.metrics} diff --git a/pyproject.toml b/pyproject.toml index 76903e1d..95af9f72 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "dbt-semantic-interfaces" -version = "0.6.2.dev2" +version = "0.6.2.dev3" description = 'The shared semantic layer definitions that dbt-core and MetricFlow use' readme = "README.md" requires-python = ">=3.8" diff --git a/tests/validations/test_measures.py b/tests/validations/test_measures.py index d01bf3fc..266894ce 100644 --- a/tests/validations/test_measures.py +++ b/tests/validations/test_measures.py @@ -72,6 +72,8 @@ def test_measures_only_exist_in_one_semantic_model() -> None: # noqa: D node_relation: schema_name: some_schema alias: source_table + defaults: + agg_time_dimension: ds entities: - name: example_entity type: primary