Create metricflow-semantics Package

metricflow/plan_conversion/node_processor.py

@@ -7,7 +7,7 @@
 from dbt_semantic_interfaces.references import EntityReference, TimeDimensionReference
 from metricflow_semantics.filters.time_constraint import TimeRangeConstraint
 from metricflow_semantics.mf_logging.pretty_print import mf_pformat
-from metricflow_semantics.model.semantics.semantic_model_join_evaluator import MAX_JOIN_HOPS, SemanticModelJoinEvaluator
+from metricflow_semantics.model.semantics.semantic_model_join_evaluator import MAX_JOIN_HOPS
 from metricflow_semantics.model.semantics.semantic_model_lookup import SemanticModelLookup
 from metricflow_semantics.specs.spec_classes import InstanceSpecSet, LinkableInstanceSpec, LinklessEntitySpec
 from metricflow_semantics.specs.spec_set_transforms import ToElementNameSet
@@ -22,6 +22,7 @@
 from metricflow.dataflow.nodes.filter_elements import FilterElementsNode
 from metricflow.dataflow.nodes.join_to_base import JoinDescription, JoinToBaseOutputNode
 from metricflow.dataflow.nodes.metric_time_transform import MetricTimeDimensionTransformNode
+from metricflow.validation.dataflow_join_validator import JoinDataflowOutputValidator
 
 logger = logging.getLogger(__name__)
 
@@ -84,7 +85,7 @@ def __init__(  # noqa: D107
         self._node_data_set_resolver = node_data_set_resolver
         self._partition_resolver = PartitionJoinResolver(semantic_model_lookup)
         self._semantic_model_lookup = semantic_model_lookup
-        self._join_evaluator = SemanticModelJoinEvaluator(semantic_model_lookup)
+        self._join_evaluator = JoinDataflowOutputValidator(semantic_model_lookup)
 
     def add_time_range_constraint(
         self,

metricflow/validation/dataflow_join_validator.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, List
+
+from dbt_semantic_interfaces.references import (
+    EntityReference,
+    SemanticModelReference,
+)
+from metricflow_semantics.instances import EntityInstance, InstanceSet
+from metricflow_semantics.mf_logging.pretty_print import mf_pformat
+from metricflow_semantics.model.semantics.semantic_model_join_evaluator import SemanticModelJoinEvaluator
+
+if TYPE_CHECKING:
+    from metricflow_semantics.model.semantics.semantic_model_lookup import SemanticModelLookup
+
+
+class JoinDataflowOutputValidator:
+    """Checks that the instances in the output of a join dataflow node is valid."""
+
+    def __init__(self, semantic_model_lookup: SemanticModelLookup) -> None:  # noqa: D107
+        self._join_evaluator = SemanticModelJoinEvaluator(semantic_model_lookup)
+
+    @staticmethod
+    def _semantic_model_of_entity_in_instance_set(
+        instance_set: InstanceSet,
+        entity_reference: EntityReference,
+    ) -> SemanticModelReference:
+        """Return the semantic model where the entity was defined in the instance set."""
+        matching_instances: List[EntityInstance] = []
+        for entity_instance in instance_set.entity_instances:
+            assert len(entity_instance.defined_from) == 1
+            if len(entity_instance.spec.entity_links) == 0 and entity_instance.spec.reference == entity_reference:
+                matching_instances.append(entity_instance)
+
+        assert len(matching_instances) == 1, (
+            f"Not exactly 1 matching entity instances found: {matching_instances} for {entity_reference} in "
+            f"{mf_pformat(instance_set)}"
+        )
+        return matching_instances[0].origin_semantic_model_reference.semantic_model_reference
+
+    def is_valid_instance_set_join(
+        self,
+        left_instance_set: InstanceSet,
+        right_instance_set: InstanceSet,
+        on_entity_reference: EntityReference,
+    ) -> bool:
+        """Return true if the instance sets can be joined using the given entity."""
+        return self._join_evaluator.is_valid_semantic_model_join(
+            left_semantic_model_reference=JoinDataflowOutputValidator._semantic_model_of_entity_in_instance_set(
+                instance_set=left_instance_set, entity_reference=on_entity_reference
+            ),
+            right_semantic_model_reference=JoinDataflowOutputValidator._semantic_model_of_entity_in_instance_set(
+                instance_set=right_instance_set,
+                entity_reference=on_entity_reference,
+            ),
+            on_entity_reference=on_entity_reference,
+        )

tests_metricflow/semantics/model/semantics/test_semantic_model_join_evaluator.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import Dict, Mapping, Sequence
+from typing import Dict, Sequence
 
 from dbt_semantic_interfaces.enum_extension import assert_values_exhausted
 from dbt_semantic_interfaces.protocols.entity import EntityType
@@ -13,8 +13,6 @@
     SemanticModelLink,
 )
 
-from tests_metricflow.fixtures.manifest_fixtures import MetricFlowEngineTestFixture, SemanticManifestSetup
-
 
 def _get_join_types_for_entity_type(entity_type: EntityType) -> Sequence[SemanticModelEntityJoinType]:
     """Exhaustively evaluate entity types and return a sequence of all possible join type pairs.
@@ -207,124 +205,6 @@ def test_semantic_model_join_validation_on_missing_entity(
     )
 
 
-def test_distinct_target_instance_set_join_validation(
-    mf_engine_test_fixture_mapping: Mapping[SemanticManifestSetup, MetricFlowEngineTestFixture],
-    simple_semantic_manifest_lookup: SemanticManifestLookup,
-) -> None:
-    """Tests instance set join validation to a PRIMARY or UNIQUE entity."""
-    foreign_user_instance_set = (
-        mf_engine_test_fixture_mapping[SemanticManifestSetup.SIMPLE_MANIFEST]
-        .data_set_mapping["listings_latest"]
-        .instance_set
-    )
-    primary_user_instance_set = (
-        mf_engine_test_fixture_mapping[SemanticManifestSetup.SIMPLE_MANIFEST]
-        .data_set_mapping["users_latest"]
-        .instance_set
-    )
-    unique_user_instance_set = (
-        mf_engine_test_fixture_mapping[SemanticManifestSetup.SIMPLE_MANIFEST].data_set_mapping["companies"].instance_set
-    )
-    user_entity_reference = EntityReference(element_name="user")
-    join_evaluator = SemanticModelJoinEvaluator(
-        semantic_model_lookup=simple_semantic_manifest_lookup.semantic_model_lookup
-    )
-
-    foreign_primary = join_evaluator.is_valid_instance_set_join(
-        left_instance_set=foreign_user_instance_set,
-        right_instance_set=primary_user_instance_set,
-        on_entity_reference=user_entity_reference,
-    )
-    primary_primary = join_evaluator.is_valid_instance_set_join(
-        left_instance_set=primary_user_instance_set,
-        right_instance_set=primary_user_instance_set,
-        on_entity_reference=user_entity_reference,
-    )
-    unique_primary = join_evaluator.is_valid_instance_set_join(
-        left_instance_set=unique_user_instance_set,
-        right_instance_set=primary_user_instance_set,
-        on_entity_reference=user_entity_reference,
-    )
-    foreign_unique = join_evaluator.is_valid_instance_set_join(
-        left_instance_set=foreign_user_instance_set,
-        right_instance_set=unique_user_instance_set,
-        on_entity_reference=user_entity_reference,
-    )
-    primary_unique = join_evaluator.is_valid_instance_set_join(
-        left_instance_set=primary_user_instance_set,
-        right_instance_set=unique_user_instance_set,
-        on_entity_reference=user_entity_reference,
-    )
-    unique_unique = join_evaluator.is_valid_instance_set_join(
-        left_instance_set=unique_user_instance_set,
-        right_instance_set=unique_user_instance_set,
-        on_entity_reference=user_entity_reference,
-    )
-
-    results = {
-        "foreign to primary": foreign_primary,
-        "primary to primary": primary_primary,
-        "unique to primary": unique_primary,
-        "foreign to unique": foreign_unique,
-        "primary to unique": primary_unique,
-        "unique to unique": unique_unique,
-    }
-    assert all(results.values()), (
-        f"All instance set level join types for primary and unique targets should be valid, but we found "
-        f"at least one that was not! Incorrectly failing types: {[k for k,v in results.items() if not v]}."
-    )
-
-
-def test_foreign_target_instance_set_join_validation(
-    mf_engine_test_fixture_mapping: Mapping[SemanticManifestSetup, MetricFlowEngineTestFixture],
-    simple_semantic_manifest_lookup: SemanticManifestLookup,
-) -> None:
-    """Tests semantic model join validation to FOREIGN entity types."""
-    foreign_user_instance_set = (
-        mf_engine_test_fixture_mapping[SemanticManifestSetup.SIMPLE_MANIFEST]
-        .data_set_mapping["listings_latest"]
-        .instance_set
-    )
-    primary_user_instance_set = (
-        mf_engine_test_fixture_mapping[SemanticManifestSetup.SIMPLE_MANIFEST]
-        .data_set_mapping["users_latest"]
-        .instance_set
-    )
-    unique_user_instance_set = (
-        mf_engine_test_fixture_mapping[SemanticManifestSetup.SIMPLE_MANIFEST].data_set_mapping["companies"].instance_set
-    )
-    user_entity_reference = EntityReference(element_name="user")
-    join_evaluator = SemanticModelJoinEvaluator(
-        semantic_model_lookup=simple_semantic_manifest_lookup.semantic_model_lookup
-    )
-
-    foreign_foreign = join_evaluator.is_valid_instance_set_join(
-        left_instance_set=foreign_user_instance_set,
-        right_instance_set=foreign_user_instance_set,
-        on_entity_reference=user_entity_reference,
-    )
-    primary_foreign = join_evaluator.is_valid_instance_set_join(
-        left_instance_set=primary_user_instance_set,
-        right_instance_set=foreign_user_instance_set,
-        on_entity_reference=user_entity_reference,
-    )
-    unique_foreign = join_evaluator.is_valid_instance_set_join(
-        left_instance_set=unique_user_instance_set,
-        right_instance_set=foreign_user_instance_set,
-        on_entity_reference=user_entity_reference,
-    )
-
-    results = {
-        "foreign to foreign": foreign_foreign,
-        "primary to foreign": primary_foreign,
-        "unique to foreign": unique_foreign,
-    }
-    assert not any(results.values()), (
-        f"All semantic model level joins against foreign targets should be invalid, but we found at least one "
-        f"that was not! Incorrectly passing types: {[k for k,v in results.items() if v]}."
-    )
-
-
 def test_get_joinable_semantic_models_single_hop(  # noqa: D103
     partitioned_multi_hop_join_semantic_manifest_lookup: SemanticManifestLookup,
 ) -> None:
@@ -498,88 +378,3 @@ def test_natural_entity_semantic_model_validation(scd_semantic_manifest_lookup:
         f"joins marked invalid: {[k for k,v in valid_joins.items() if not v]}. Invalid joins marked valid: "
         f"{[k for k, v in invalid_joins.items() if v]}."
     )
-
-
-def test_natural_entity_instance_set_validation(
-    mf_engine_test_fixture_mapping: Mapping[SemanticManifestSetup, MetricFlowEngineTestFixture],
-    scd_semantic_manifest_lookup: SemanticManifestLookup,
-) -> None:
-    """Tests instance set validation for NATURAL target entity types.
-
-    These tests rely on the scd_semantic_manifest_lookup, which makes extensive use of NATURAL key types.
-    """
-    natural_user_instance_set = (
-        mf_engine_test_fixture_mapping[SemanticManifestSetup.SCD_MANIFEST]
-        .data_set_mapping["primary_accounts"]
-        .instance_set
-    )
-    primary_user_instance_set = (
-        mf_engine_test_fixture_mapping[SemanticManifestSetup.SCD_MANIFEST].data_set_mapping["users_latest"].instance_set
-    )
-    foreign_user_instance_set = (
-        mf_engine_test_fixture_mapping[SemanticManifestSetup.SCD_MANIFEST]
-        .data_set_mapping["bookings_source"]
-        .instance_set
-    )
-    unique_user_instance_set = (
-        mf_engine_test_fixture_mapping[SemanticManifestSetup.SCD_MANIFEST].data_set_mapping["companies"].instance_set
-    )
-    user_entity_reference = EntityReference(element_name="user")
-    join_evaluator = SemanticModelJoinEvaluator(
-        semantic_model_lookup=scd_semantic_manifest_lookup.semantic_model_lookup
-    )
-
-    # Valid cases
-    natural_primary = join_evaluator.is_valid_instance_set_join(
-        left_instance_set=natural_user_instance_set,
-        right_instance_set=primary_user_instance_set,
-        on_entity_reference=user_entity_reference,
-    )
-    natural_unique = join_evaluator.is_valid_instance_set_join(
-        left_instance_set=natural_user_instance_set,
-        right_instance_set=unique_user_instance_set,
-        on_entity_reference=user_entity_reference,
-    )
-    foreign_natural = join_evaluator.is_valid_instance_set_join(
-        left_instance_set=foreign_user_instance_set,
-        right_instance_set=natural_user_instance_set,
-        on_entity_reference=user_entity_reference,
-    )
-    primary_natural = join_evaluator.is_valid_instance_set_join(
-        left_instance_set=primary_user_instance_set,
-        right_instance_set=natural_user_instance_set,
-        on_entity_reference=user_entity_reference,
-    )
-    unique_natural = join_evaluator.is_valid_instance_set_join(
-        left_instance_set=unique_user_instance_set,
-        right_instance_set=natural_user_instance_set,
-        on_entity_reference=user_entity_reference,
-    )
-    # Invalid cases
-    natural_foreign = join_evaluator.is_valid_instance_set_join(
-        left_instance_set=natural_user_instance_set,
-        right_instance_set=foreign_user_instance_set,
-        on_entity_reference=user_entity_reference,
-    )
-    natural_natural = join_evaluator.is_valid_instance_set_join(
-        left_instance_set=natural_user_instance_set,
-        right_instance_set=natural_user_instance_set,
-        on_entity_reference=user_entity_reference,
-    )
-
-    valid_joins = {
-        "natural to primary": natural_primary,
-        "natural to unique": natural_unique,
-        "foreign to natural": foreign_natural,
-        "primary to natural": primary_natural,
-        "unique to natural": unique_natural,
-    }
-    invalid_joins = {
-        "natural to foreign": natural_foreign,
-        "natural to natural": natural_natural,
-    }
-    assert all(valid_joins.values()) and not any(invalid_joins.values()), (
-        f"Found unexpected join validator results when validating joins involving natural key comparisons! Valid "
-        f"joins marked invalid: {[k for k,v in valid_joins.items() if not v]}. Invalid joins marked valid: "
-        f"{[k for k, v in invalid_joins.items() if v]}."
-    )