Skip to content

Commit

Permalink
Update tests to reflect class signature changes.
Browse files Browse the repository at this point in the history 
Tests cases need to be updated to handle signature changes to
the MetricFlowQueryParser, MetricFlowQuerySpec, WhereFilterSpec.

In cases where a filter needs to be created for a query, the query parser is
is used to generate the appropriate loookup.
  • Loading branch information
@plypaul
plypaul committed Nov 30, 2023
1 parent f9a67eb commit 92c5191
Show file tree
Hide file tree
Showing 10 changed files with 267 additions and 309 deletions.
168 changes: 45 additions & 123 deletions metricflow/test/dataflow/builder/test_dataflow_plan_builder.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,14 @@

import pytest
from _pytest.fixtures import FixtureRequest
from dbt_semantic_interfaces.implementations.filters.where_filter import PydanticWhereFilter
from dbt_semantic_interfaces.naming.keywords import METRIC_TIME_ELEMENT_NAME
from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity

from metricflow.dataflow.builder.dataflow_plan_builder import DataflowPlanBuilder
from metricflow.dataflow.dataflow_plan_to_text import dataflow_plan_as_text
from metricflow.dataset.dataset import DataSet
from metricflow.errors.errors import UnableToSatisfyQueryError
from metricflow.query.query_parser import MetricFlowQueryParser
from metricflow.specs.column_assoc import ColumnAssociationResolver
from metricflow.specs.specs import (
DimensionSpec,
Expand All @@ -20,7 +21,6 @@
OrderBySpec,
TimeDimensionSpec,
)
from metricflow.specs.where_filter_transform import WhereSpecFactory
from metricflow.test.dataflow_plan_to_svg import display_graph_if_requested
from metricflow.test.fixtures.setup_fixtures import MetricFlowTestSessionState
from metricflow.test.snapshot_utils import assert_plan_snapshot_text_equal
Expand Down Expand Up @@ -344,28 +344,15 @@ def test_where_constrained_plan( # noqa: D
mf_test_session_state: MetricFlowTestSessionState,
column_association_resolver: ColumnAssociationResolver,
dataflow_plan_builder: DataflowPlanBuilder,
query_parser: MetricFlowQueryParser,
) -> None:
"""Tests a simple plan getting a metric and a local dimension."""
dataflow_plan = dataflow_plan_builder.build_plan(
MetricFlowQuerySpec(
metric_specs=(MetricSpec(element_name="bookings"),),
dimension_specs=(
DimensionSpec(
element_name="is_instant",
entity_links=(EntityReference("booking"),),
),
),
where_constraint=(
WhereSpecFactory(
column_association_resolver=column_association_resolver,
).create_from_where_filter(
PydanticWhereFilter(
where_sql_template="{{ Dimension('listing__country_latest') }} = 'us'",
)
)
),
)
query_spec = query_parser.parse_and_validate_query(
metric_names=("bookings",),
group_by_names=("booking__is_instant",),
where_constraint_str="{{ Dimension('listing__country_latest') }} = 'us'",
)
dataflow_plan = dataflow_plan_builder.build_plan(query_spec)

assert_plan_snapshot_text_equal(
request=request,
Expand All @@ -386,29 +373,15 @@ def test_where_constrained_plan_time_dimension( # noqa: D
request: FixtureRequest,
mf_test_session_state: MetricFlowTestSessionState,
dataflow_plan_builder: DataflowPlanBuilder,
column_association_resolver: ColumnAssociationResolver,
query_parser: MetricFlowQueryParser,
) -> None:
"""Tests a simple plan getting a metric and a local dimension."""
dataflow_plan = dataflow_plan_builder.build_plan(
MetricFlowQuerySpec(
metric_specs=(MetricSpec(element_name="bookings"),),
dimension_specs=(
DimensionSpec(
element_name="is_instant",
entity_links=(EntityReference("booking"),),
),
),
where_constraint=(
WhereSpecFactory(
column_association_resolver=column_association_resolver,
).create_from_where_filter(
PydanticWhereFilter(
where_sql_template="{{ TimeDimension('metric_time', 'day') }} >= '2020-01-01'",
)
)
),
)
query_spec = query_parser.parse_and_validate_query(
metric_names=("bookings",),
group_by_names=("booking__is_instant",),
where_constraint_str="{{ TimeDimension('metric_time', 'day') }} >= '2020-01-01'",
)
dataflow_plan = dataflow_plan_builder.build_plan(query_spec)

assert_plan_snapshot_text_equal(
request=request,
Expand All @@ -430,28 +403,15 @@ def test_where_constrained_with_common_linkable_plan( # noqa: D
mf_test_session_state: MetricFlowTestSessionState,
column_association_resolver: ColumnAssociationResolver,
dataflow_plan_builder: DataflowPlanBuilder,
query_parser: MetricFlowQueryParser,
) -> None:
"""Tests a dataflow plan where the where clause has a common linkable with the query."""
dataflow_plan = dataflow_plan_builder.build_plan(
MetricFlowQuerySpec(
metric_specs=(MetricSpec(element_name="bookings"),),
dimension_specs=(
DimensionSpec(
element_name="country_latest",
entity_links=(EntityReference(element_name="listing"),),
),
),
where_constraint=(
WhereSpecFactory(
column_association_resolver=column_association_resolver,
).create_from_where_filter(
PydanticWhereFilter(
where_sql_template="{{ Dimension('listing__country_latest') }} = 'us'",
)
)
),
)
query_spec = query_parser.parse_and_validate_query(
metric_names=("bookings",),
group_by_names=("listing__country_latest",),
where_constraint_str="{{ Dimension('listing__country_latest') }} = 'us'",
)
dataflow_plan = dataflow_plan_builder.build_plan(query_spec)

assert_plan_snapshot_text_equal(
request=request,
Expand Down Expand Up @@ -580,34 +540,17 @@ def test_distinct_values_plan( # noqa: D
request: FixtureRequest,
mf_test_session_state: MetricFlowTestSessionState,
dataflow_plan_builder: DataflowPlanBuilder,
column_association_resolver: ColumnAssociationResolver,
query_parser: MetricFlowQueryParser,
) -> None:
"""Tests a plan to get distinct values of a dimension."""
dataflow_plan = dataflow_plan_builder.build_plan_for_distinct_values(
query_spec=MetricFlowQuerySpec(
dimension_specs=(
DimensionSpec(element_name="country_latest", entity_links=(EntityReference(element_name="listing"),)),
),
where_constraint=(
WhereSpecFactory(
column_association_resolver=column_association_resolver,
).create_from_where_filter(
PydanticWhereFilter(
where_sql_template="{{ Dimension('listing__country_latest') }} = 'us'",
)
)
),
order_by_specs=(
OrderBySpec(
instance_spec=DimensionSpec(
element_name="country_latest", entity_links=(EntityReference(element_name="listing"),)
),
descending=True,
),
),
limit=100,
)
query_spec = query_parser.parse_and_validate_query(
metric_names=(),
group_by_names=("listing__country_latest",),
where_constraint_str="{{ Dimension('listing__country_latest') }} = 'us'",
order_by_names=("-listing__country_latest",),
limit=100,
)
dataflow_plan = dataflow_plan_builder.build_plan_for_distinct_values(query_spec)

assert_plan_snapshot_text_equal(
request=request,
Expand All @@ -628,35 +571,16 @@ def test_distinct_values_plan_with_join( # noqa: D
request: FixtureRequest,
mf_test_session_state: MetricFlowTestSessionState,
dataflow_plan_builder: DataflowPlanBuilder,
column_association_resolver: ColumnAssociationResolver,
query_parser: MetricFlowQueryParser,
) -> None:
"""Tests a plan to get distinct values of 2 dimensions, where a join is required."""
dataflow_plan = dataflow_plan_builder.build_plan_for_distinct_values(
query_spec=MetricFlowQuerySpec(
dimension_specs=(
DimensionSpec(element_name="home_state_latest", entity_links=(EntityReference(element_name="user"),)),
DimensionSpec(element_name="is_lux_latest", entity_links=(EntityReference(element_name="listing"),)),
),
where_constraint=(
WhereSpecFactory(
column_association_resolver=column_association_resolver,
).create_from_where_filter(
PydanticWhereFilter(
where_sql_template="{{ Dimension('listing__country_latest') }} = 'us'",
)
)
),
order_by_specs=(
OrderBySpec(
instance_spec=DimensionSpec(
element_name="country_latest", entity_links=(EntityReference(element_name="listing"),)
),
descending=True,
),
),
limit=100,
)
query_spec = query_parser.parse_and_validate_query(
group_by_names=("user__home_state_latest", "listing__is_lux_latest"),
where_constraint_str="{{ Dimension('listing__country_latest') }} = 'us'",
order_by_names=("-listing__is_lux_latest",),
limit=100,
)
dataflow_plan = dataflow_plan_builder.build_plan_for_distinct_values(query_spec)

assert_plan_snapshot_text_equal(
request=request,
Expand All @@ -676,16 +600,15 @@ def test_distinct_values_plan_with_join( # noqa: D
def test_measure_constraint_plan(
request: FixtureRequest,
mf_test_session_state: MetricFlowTestSessionState,
query_parser: MetricFlowQueryParser,
dataflow_plan_builder: DataflowPlanBuilder,
) -> None:
"""Tests a plan for querying a metric with a constraint on one or more of its input measures."""
dataflow_plan = dataflow_plan_builder.build_plan(
MetricFlowQuerySpec(
metric_specs=(MetricSpec(element_name="lux_booking_value_rate_expr"),),
dimension_specs=(),
time_dimension_specs=(MTD_SPEC_DAY,),
),
query_spec = query_parser.parse_and_validate_query(
metric_names=("lux_booking_value_rate_expr",),
group_by_names=(METRIC_TIME_ELEMENT_NAME,),
)
dataflow_plan = dataflow_plan_builder.build_plan(query_spec)

assert_plan_snapshot_text_equal(
request=request,
Expand All @@ -705,16 +628,15 @@ def test_measure_constraint_plan(
def test_measure_constraint_with_reused_measure_plan(
request: FixtureRequest,
mf_test_session_state: MetricFlowTestSessionState,
query_parser: MetricFlowQueryParser,
dataflow_plan_builder: DataflowPlanBuilder,
) -> None:
"""Tests a plan for querying a metric with a constraint on one or more of its input measures."""
dataflow_plan = dataflow_plan_builder.build_plan(
MetricFlowQuerySpec(
metric_specs=(MetricSpec(element_name="instant_booking_value_ratio"),),
dimension_specs=(),
time_dimension_specs=(MTD_SPEC_DAY,),
),
query_spec = query_parser.parse_and_validate_query(
metric_names=("instant_booking_value_ratio",),
group_by_names=(METRIC_TIME_ELEMENT_NAME,),
)
dataflow_plan = dataflow_plan_builder.build_plan(query_spec)

assert_plan_snapshot_text_equal(
request=request,
Expand Down
29 changes: 10 additions & 19 deletions metricflow/test/dataflow/optimizer/source_scan/test_source_scan_optimizer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

import pytest
from _pytest.fixtures import FixtureRequest
from dbt_semantic_interfaces.implementations.filters.where_filter import PydanticWhereFilter
from dbt_semantic_interfaces.naming.keywords import METRIC_TIME_ELEMENT_NAME
from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity

from metricflow.dataflow.builder.dataflow_plan_builder import DataflowPlanBuilder
Expand All @@ -31,14 +31,14 @@
from metricflow.dataflow.dataflow_plan_to_text import dataflow_plan_as_text
from metricflow.dataflow.optimizer.source_scan.source_scan_optimizer import SourceScanOptimizer
from metricflow.dataset.dataset import DataSet
from metricflow.query.query_parser import MetricFlowQueryParser
from metricflow.specs.column_assoc import ColumnAssociationResolver
from metricflow.specs.specs import (
DimensionSpec,
EntityReference,
MetricFlowQuerySpec,
MetricSpec,
)
from metricflow.specs.where_filter_transform import WhereSpecFactory
from metricflow.test.dataflow_plan_to_svg import display_graph_if_requested
from metricflow.test.fixtures.setup_fixtures import MetricFlowTestSessionState
from metricflow.test.snapshot_utils import assert_plan_snapshot_text_equal
Expand Down Expand Up @@ -221,31 +221,22 @@ def test_constrained_metric_not_combined( # noqa: D
mf_test_session_state: MetricFlowTestSessionState,
column_association_resolver: ColumnAssociationResolver,
dataflow_plan_builder: DataflowPlanBuilder,
query_parser: MetricFlowQueryParser,
) -> None:
"""Tests that 2 metrics from the same semantic model but where 1 is constrained results in 2 scans.
If there is a constraint, need needs to be handled in a separate query because the constraint applies to all rows.
If there is a constraint for a metric, it needs to be handled in a separate query because the constraint applies to
all rows.
"""
query_spec = query_parser.parse_and_validate_query(
metric_names=("booking_value", "instant_booking_value"),
group_by_names=(METRIC_TIME_ELEMENT_NAME,),
)
check_optimization(
request=request,
mf_test_session_state=mf_test_session_state,
dataflow_plan_builder=dataflow_plan_builder,
query_spec=MetricFlowQuerySpec(
metric_specs=(
MetricSpec(element_name="booking_value"),
MetricSpec(
element_name="instant_booking_value",
constraint=(
WhereSpecFactory(
column_association_resolver=column_association_resolver,
).create_from_where_filter(
PydanticWhereFilter(where_sql_template="{{ Dimension('booking__is_instant') }} ")
)
),
),
),
dimension_specs=(DataSet.metric_time_dimension_spec(TimeGranularity.DAY),),
),
query_spec=query_spec,
expected_num_sources_in_unoptimized=2,
expected_num_sources_in_optimized=2,
)
Expand Down
24 changes: 24 additions & 0 deletions metricflow/test/fixtures/dataflow_fixtures.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from metricflow.plan_conversion.column_resolver import DunderColumnAssociationResolver
from metricflow.plan_conversion.time_spine import TimeSpineSource
from metricflow.protocols.sql_client import SqlClient
from metricflow.query.query_parser import MetricFlowQueryParser
from metricflow.specs.column_assoc import ColumnAssociationResolver
from metricflow.test.fixtures.model_fixtures import ConsistentIdObjectRepository
from metricflow.test.fixtures.setup_fixtures import MetricFlowTestSessionState
Expand Down Expand Up @@ -38,6 +39,18 @@ def dataflow_plan_builder( # noqa: D
)


@pytest.fixture
def query_parser( # noqa: D
simple_semantic_manifest_lookup: SemanticManifestLookup,
column_association_resolver: ColumnAssociationResolver,
consistent_id_object_repository: ConsistentIdObjectRepository,
) -> MetricFlowQueryParser:
return MetricFlowQueryParser(
column_association_resolver=column_association_resolver,
model=simple_semantic_manifest_lookup,
)


@pytest.fixture
def extended_date_dataflow_plan_builder( # noqa: D
extended_date_semantic_manifest_lookup: SemanticManifestLookup,
Expand Down Expand Up @@ -85,6 +98,17 @@ def scd_dataflow_plan_builder( # noqa: D
)


@pytest.fixture
def scd_query_parser( # noqa: D
scd_column_association_resolver: ColumnAssociationResolver,
scd_semantic_manifest_lookup: SemanticManifestLookup,
) -> MetricFlowQueryParser:
return MetricFlowQueryParser(
column_association_resolver=scd_column_association_resolver,
model=scd_semantic_manifest_lookup,
)


@pytest.fixture(scope="session")
def time_spine_source( # noqa: D
sql_client: SqlClient, mf_test_session_state: MetricFlowTestSessionState # noqa: F811
Expand Down
5 changes: 0 additions & 5 deletions metricflow/test/fixtures/model_fixtures.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,11 +64,6 @@ def query_parser_from_yaml(yaml_contents: List[YamlConfigFile]) -> MetricFlowQue
return MetricFlowQueryParser(
model=semantic_manifest_lookup,
column_association_resolver=DunderColumnAssociationResolver(semantic_manifest_lookup),
read_nodes=list(_data_set_to_read_nodes(create_data_sets(semantic_manifest_lookup)).values()),
node_output_resolver=DataflowPlanNodeOutputDataSetResolver(
column_association_resolver=DunderColumnAssociationResolver(semantic_manifest_lookup),
semantic_manifest_lookup=semantic_manifest_lookup,
),
)


Expand Down
Loading

0 comments on commit 92c5191

Please sign in to comment.