Create metricflow-semantics Package

.changes/unreleased/Features-20240424-145949.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Create `metricflow-semantics` package.
+time: 2024-04-24T14:59:49.8435-07:00
+custom:
+  Author: plypaul
+  Issue: "1150"

Makefile

@@ -9,7 +9,9 @@ ADDITIONAL_PYTEST_OPTIONS =
 
 # Pytest that can populate the persistent source schema
 USE_PERSISTENT_SOURCE_SCHEMA = --use-persistent-source-schema
-POPULATE_PERSISTENT_SOURCE_SCHEMA = tests/source_schema_tools.py::populate_source_schema
+TESTS_METRICFLOW = tests_metricflow
+TESTS_METRICFLOW_SEMANTICS = tests_metricflow_semantics
+POPULATE_PERSISTENT_SOURCE_SCHEMA = $(TESTS_METRICFLOW)/source_schema_tools.py::populate_source_schema
 
 # Install Hatch package / project manager
 .PHONY: install-hatch
@@ -19,32 +21,33 @@ install-hatch:
 # Testing and linting
 .PHONY: test
 test:
-	hatch -v run dev-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) tests/
+	cd metricflow-semantics && hatch -v run dev-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) $(TESTS_METRICFLOW_SEMANTICS)/
+	hatch -v run dev-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) $(TESTS_METRICFLOW)/
 
 .PHONY: test-postgresql
 test-postgresql:
-	hatch -v run postgres-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) tests/
+	hatch -v run postgres-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) $(TESTS_METRICFLOW)/
 
 # Engine-specific test environments.
 .PHONY: test-bigquery
 test-bigquery:
-	hatch -v run bigquery-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) tests/
+	hatch -v run bigquery-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) $(TESTS_METRICFLOW)/
 
 .PHONY: populate-persistent-source-schema-bigquery
 populate-persistent-source-schema-bigquery:
 	hatch -v run bigquery-env:pytest -vv $(ADDITIONAL_PYTEST_OPTIONS) $(USE_PERSISTENT_SOURCE_SCHEMA) $(POPULATE_PERSISTENT_SOURCE_SCHEMA)
 
 .PHONY: test-databricks
 test-databricks:
-	hatch -v run databricks-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) tests/
+	hatch -v run databricks-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) $(TESTS_METRICFLOW)/
 
 .PHONY: populate-persistent-source-schema-databricks
 populate-persistent-source-schema-databricks:
 	hatch -v run databricks-env:pytest -vv $(ADDITIONAL_PYTEST_OPTIONS) $(USE_PERSISTENT_SOURCE_SCHEMA) $(POPULATE_PERSISTENT_SOURCE_SCHEMA)
 
 .PHONY: test-redshift
 test-redshift:
-	hatch -v run redshift-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) tests/
+	hatch -v run redshift-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) $(TESTS_METRICFLOW)/
 
 .PHONY: populate-persistent-source-schema-redshift
 populate-persistent-source-schema-redshift:
@@ -53,15 +56,15 @@ populate-persistent-source-schema-redshift:
 
 .PHONY: test-snowflake
 test-snowflake:
-	hatch -v run snowflake-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) tests/
+	hatch -v run snowflake-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) $(TESTS_METRICFLOW)/
 
 .PHONY: populate-persistent-source-schema-snowflake
 populate-persistent-source-schema-snowflake:
 	hatch -v run snowflake-env:pytest -vv $(ADDITIONAL_PYTEST_OPTIONS) $(USE_PERSISTENT_SOURCE_SCHEMA) $(POPULATE_PERSISTENT_SOURCE_SCHEMA)
 
 .PHONY: test-trino
 test-trino:
-	hatch -v run trino-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) tests/
+	hatch -v run trino-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) $(TESTS_METRICFLOW)/
 
 .PHONY: lint
 lint:
@@ -79,12 +82,12 @@ trino:
 # Re-generate test snapshots using all supported SQL engines.
 .PHONY: regenerate-test-snapshots
 regenerate-test-snapshots:
-	hatch -v run dev-env:python tests/generate_snapshots.py
+	hatch -v run dev-env:python tests_metricflow/generate_snapshots.py
 
 # Populate persistent source schemas for all relevant SQL engines.
 .PHONY: populate-persistent-source-schemas
 populate-persistent-source-schemas:
-	hatch -v run dev-env:python tests/populate_persistent_source_schemas.py
+	hatch -v run dev-env:python $(TESTS_METRICFLOW)/populate_persistent_source_schemas.py
 
 # Re-generate snapshots for the default SQL engine.
 .PHONY: test-snap

dbt-metricflow/dbt_metricflow/cli/cli_context.py

@@ -6,11 +6,11 @@
 from typing import Dict, Optional
 
 from dbt_semantic_interfaces.protocols.semantic_manifest import SemanticManifest
+from metricflow_semantics.model.semantic_manifest_lookup import SemanticManifestLookup
 
 from dbt_metricflow.cli.dbt_connectors.adapter_backed_client import AdapterBackedSqlClient
 from dbt_metricflow.cli.dbt_connectors.dbt_config_accessor import dbtArtifacts, dbtProjectMetadata
 from metricflow.engine.metricflow_engine import MetricFlowEngine
-from metricflow.model.semantic_manifest_lookup import SemanticManifestLookup
 from metricflow.protocols.sql_client import SqlClient
 
 logger = logging.getLogger(__name__)

dbt-metricflow/dbt_metricflow/cli/dbt_connectors/adapter_backed_client.py

@@ -8,12 +8,13 @@
 from dbt.adapters.base.impl import BaseAdapter
 from dbt.exceptions import DbtDatabaseError
 from dbt_semantic_interfaces.enum_extension import assert_values_exhausted
+from metricflow_semantics.errors.error_classes import SqlBindParametersNotSupportedError
+from metricflow_semantics.mf_logging.formatting import indent
+from metricflow_semantics.mf_logging.pretty_print import mf_pformat
+from metricflow_semantics.random_id import random_id
+from metricflow_semantics.sql.sql_bind_parameters import SqlBindParameters
 
-from metricflow.errors.errors import SqlBindParametersNotSupportedError
-from metricflow.mf_logging.formatting import indent
-from metricflow.mf_logging.pretty_print import mf_pformat
 from metricflow.protocols.sql_client import SqlEngine
-from metricflow.random_id import random_id
 from metricflow.sql.render.big_query import BigQuerySqlQueryPlanRenderer
 from metricflow.sql.render.databricks import DatabricksSqlQueryPlanRenderer
 from metricflow.sql.render.duckdb_renderer import DuckDbSqlQueryPlanRenderer
@@ -22,7 +23,6 @@
 from metricflow.sql.render.snowflake import SnowflakeSqlQueryPlanRenderer
 from metricflow.sql.render.sql_plan_renderer import SqlQueryPlanRenderer
 from metricflow.sql.render.trino import TrinoSqlQueryPlanRenderer
-from metricflow.sql.sql_bind_parameters import SqlBindParameters
 from metricflow.sql_request.sql_request_attributes import SqlRequestId
 
 logger = logging.getLogger(__name__)

dbt-metricflow/dbt_metricflow/cli/dbt_connectors/dbt_config_accessor.py

@@ -12,12 +12,11 @@
 from dbt.config.project import Project
 from dbt.config.runtime import load_profile, load_project
 from dbt_semantic_interfaces.protocols.semantic_manifest import SemanticManifest
+from metricflow_semantics.errors.error_classes import ModelCreationException
+from metricflow_semantics.mf_logging.pretty_print import mf_pformat
+from metricflow_semantics.model.dbt_manifest_parser import parse_manifest_from_dbt_generated_manifest
 from typing_extensions import Self
 
-from metricflow.errors.errors import ModelCreationException
-from metricflow.mf_logging.pretty_print import mf_pformat
-from metricflow.model.dbt_manifest_parser import parse_manifest_from_dbt_generated_manifest
-
 logger = logging.getLogger(__name__)
 
 

dbt-metricflow/dbt_metricflow/cli/main.py

@@ -18,6 +18,7 @@
 from dbt_semantic_interfaces.validations.semantic_manifest_validator import SemanticManifestValidator
 from dbt_semantic_interfaces.validations.validator_helpers import SemanticManifestValidationResults
 from halo import Halo
+from metricflow_semantics.dag.dag_visualization import display_dag_as_svg
 from update_checker import UpdateChecker
 
 import dbt_metricflow.cli.custom_click_types as click_custom
@@ -35,11 +36,10 @@
     query_options,
     start_end_time_options,
 )
-from metricflow.dag.dag_visualization import display_dag_as_svg
 from metricflow.engine.metricflow_engine import MetricFlowExplainResult, MetricFlowQueryRequest, MetricFlowQueryResult
-from metricflow.model.data_warehouse_model_validator import DataWarehouseModelValidator
 from metricflow.telemetry.models import TelemetryLevel
 from metricflow.telemetry.reporter import TelemetryReporter, log_call
+from metricflow.validation.data_warehouse_model_validator import DataWarehouseModelValidator
 
 logger = logging.getLogger(__name__)
 

metricflow-semantics/LICENSE

@@ -0,0 +1,90 @@
+License text copyright © 2017 MariaDB Corporation Ab, All Rights Reserved.
+“Business Source License” is a trademark of MariaDB Corporation Ab.
+
+Licensed Work: dbt MetricFlow
+
+Change Date: The change date for each release of the Licensed Work will
+be the 3rd anniversary of the initial release date of that
+release of the Licensed Work.
+
+Change License: Apache 2.0 https://www.apache.org/licenses/LICENSE-2.0
+
+Additional Use Grant: The Licensor hereby grants you the right to use the
+Licensed Work for production purposes that consist of
+your internal business purposes, but not for the purpose
+of providing managed services or software-as-a-service
+to others.
+
+Terms
+
+The Licensor hereby grants you the right to copy, modify, create derivative
+works, redistribute, and make non-production use of the Licensed Work. The
+Licensor may make an Additional Use Grant, above, permitting limited production
+use.
+
+Effective on the Change Date, or the fourth anniversary of the first publicly
+available distribution of a specific version of the Licensed Work under this
+License, whichever comes first, the Licensor hereby grants you rights under the
+terms of the Change License, and the rights granted in the paragraph above
+terminate.
+
+If your use of the Licensed Work does not comply with the requirements currently
+in effect as described in this License, you must purchase a commercial license
+from the Licensor, its affiliated entities, or authorized resellers, or you must
+refrain from using the Licensed Work.
+
+All copies of the original and modified Licensed Work, and derivative works of
+the Licensed Work, are subject to this License. This License applies separately
+for each version of the Licensed Work and the Change Date may vary for each
+version of the Licensed Work released by Licensor.
+
+You must conspicuously display this License on each original or modified copy of
+the Licensed Work. If you receive the Licensed Work in original or modified form
+from a third party, the terms and conditions set forth in this License apply to
+your use of that work.
+
+Any use of the Licensed Work in violation of this License will automatically
+terminate your rights under this License for the current and all other versions
+of the Licensed Work.
+
+This License does not grant you any right in any trademark or logo of Licensor
+or its affiliates (provided that you may use a trademark or logo of Licensor as
+expressly required by this License).TO THE EXTENT PERMITTED BY APPLICABLE LAW,
+THE LICENSED WORK IS PROVIDED ON AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL
+WARRANTIES AND CONDITIONS, EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION)
+WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE,
+NON-INFRINGEMENT, AND TITLE.MariaDB hereby grants you permission to use this
+License’s text to license your works, and to refer to it using the trademark
+“Business Source License”, as long as you comply with the Covenants of Licensor
+below.
+
+Covenants of Licensor
+
+In consideration of the right to use this License’s text and the “Business
+Source License” name and trademark, Licensor covenants to MariaDB, and to all
+other recipients of the licensed work to be provided by Licensor:
+
+To specify as the Change License the GPL Version 2.0 or any later version, or a
+license that is compatible with GPL Version 2.0 or a later version, where
+“compatible” means that software provided under the Change License can be
+included in a program with software provided under GPL Version 2.0 or a later
+version. Licensor may specify additional Change Licenses without limitation.
+
+To either: (a) specify an additional grant of rights to use that does not impose
+any additional restriction on the right granted in this License, as the
+Additional Use Grant; or (b) insert the text “None”.
+
+To specify a Change Date.
+
+Not to modify this License in any other way.
+
+Notice
+
+The Business Source License (this document, or the “License”) is not an Open
+Source license. However, the Licensed Work will eventually be made available
+under an Open Source License, as stated in this License.
+
+For more information on the use of the Business Source License for MariaDB
+products, please visit the MariaDB Business Source License FAQ. For more
+information on the use of the Business Source License generally, please visit
+the Adopting and Developing Business Source License FAQ.

metricflow-semantics/README.md

@@ -0,0 +1,16 @@
+<p align="center">
+    <a target="_blank" href="https://twitter.com/dbt_labs">
+    <img src="https://img.shields.io/twitter/follow/dbt_labs?labelColor=image.png&color=163B36&logo=twitter&style=flat">
+  </a>
+    <a target="_blank" href="https://www.getdbt.com/community/">
+    <img src="https://img.shields.io/badge/Slack-join-163B36">
+  </a>
+    <a href="https://github.com/psf/black"><img src="https://img.shields.io/badge/code%20style-black-000000.svg" /></a>
+</p>
+
+# Welcome to metricflow-semantics
+
+This repo encapsulates the modules needed for semantic resolution for MetricFlow queries.
+
+## Repo use cases
+- Resolving the dbt model dependencies of a (saved) query in MetricFlow for DAG dependency management.

metricflow/dag/dag_to_text.py → metricflow-semantics/metricflow_semantics/dag/dag_to_text.py

@@ -12,9 +12,9 @@
 import jinja2
 
 if typing.TYPE_CHECKING:
-    from metricflow.dag.mf_dag import DagNode, DagNodeT, DisplayedProperty, MetricFlowDag
+    from metricflow_semantics.dag.mf_dag import DagNode, DagNodeT, DisplayedProperty, MetricFlowDag
 
-from metricflow.mf_logging.pretty_print import mf_pformat
+from metricflow_semantics.mf_logging.pretty_print import mf_pformat
 
 logger = logging.getLogger(__name__)
 

metricflow/dag/dag_visualization.py → metricflow-semantics/metricflow_semantics/dag/dag_visualization.py

@@ -6,8 +6,8 @@
 
 import graphviz
 
-from metricflow.dag.mf_dag import DagNode, MetricFlowDag
-from metricflow.random_id import random_id
+from metricflow_semantics.dag.mf_dag import DagNode, MetricFlowDag
+from metricflow_semantics.random_id import random_id
 
 logger = logging.getLogger(__name__)
 DagNodeT = TypeVar("DagNodeT", bound=DagNode)

metricflow/dag/mf_dag.py → metricflow-semantics/metricflow_semantics/dag/mf_dag.py

@@ -11,10 +11,10 @@
 
 import jinja2
 
-from metricflow.dag.dag_to_text import MetricFlowDagTextFormatter
-from metricflow.dag.id_prefix import IdPrefix
-from metricflow.dag.sequential_id import SequentialIdGenerator
-from metricflow.visitor import VisitorOutputT
+from metricflow_semantics.dag.dag_to_text import MetricFlowDagTextFormatter
+from metricflow_semantics.dag.id_prefix import IdPrefix
+from metricflow_semantics.dag.sequential_id import SequentialIdGenerator
+from metricflow_semantics.visitor import VisitorOutputT
 
 logger = logging.getLogger(__name__)
 

metricflow/dag/sequential_id.py → metricflow-semantics/metricflow_semantics/dag/sequential_id.py

@@ -6,7 +6,7 @@
 
 from typing_extensions import override
 
-from metricflow.dag.id_prefix import IdPrefix
+from metricflow_semantics.dag.id_prefix import IdPrefix
 
 
 @dataclass(frozen=True)

metricflow/filters/time_constraint.py → metricflow-semantics/metricflow_semantics/filters/time_constraint.py

@@ -9,7 +9,7 @@
 from dbt_semantic_interfaces.dataclass_serialization import SerializableDataclass
 from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity
 
-from metricflow.time.time_granularity import offset_period
+from metricflow_semantics.time.time_granularity import offset_period
 
 logger = logging.getLogger(__name__)
 

metricflow/instances.py → metricflow-semantics/metricflow_semantics/instances.py

@@ -9,9 +9,9 @@
 from dbt_semantic_interfaces.dataclass_serialization import SerializableDataclass
 from dbt_semantic_interfaces.references import MetricModelReference, SemanticModelElementReference
 
-from metricflow.aggregation_properties import AggregationState
-from metricflow.specs.column_assoc import ColumnAssociation
-from metricflow.specs.specs import (
+from metricflow_semantics.aggregation_properties import AggregationState
+from metricflow_semantics.specs.column_assoc import ColumnAssociation
+from metricflow_semantics.specs.spec_classes import (
     DimensionSpec,
     EntitySpec,
     GroupByMetricSpec,

metricflow/mf_logging/pretty_print.py → metricflow-semantics/metricflow_semantics/mf_logging/pretty_print.py

@@ -9,7 +9,7 @@
 
 from pydantic import BaseModel
 
-from metricflow.mf_logging.formatting import indent
+from metricflow_semantics.mf_logging.formatting import indent
 
 logger = logging.getLogger(__name__)
 
@@ -36,7 +36,8 @@ def __init__(
 
     @staticmethod
     def _is_pydantic_base_model(obj: Any):  # type:ignore
-        return isinstance(obj, BaseModel)
+        # Checking the attribute as the BaseModel check fails for newer version of Pydantic.
+        return isinstance(obj, BaseModel) or hasattr(obj, "__config__")
 
     def _handle_sequence_obj(self, list_like_obj: Union[list, tuple], remaining_line_width: Optional[int]) -> str:
         """Pretty prints a sequence object i.e. list or tuple.

metricflow/model/dbt_manifest_parser.py → metricflow-semantics/metricflow_semantics/model/dbt_manifest_parser.py

@@ -16,7 +16,7 @@
     PydanticSemanticManifestTransformer,
 )
 
-from metricflow.model.transformations.dedupe_metric_input_measures import DedupeMetricInputMeasuresRule
+from metricflow_semantics.model.transformations.dedupe_metric_input_measures import DedupeMetricInputMeasuresRule
 
 
 def parse_manifest_from_dbt_generated_manifest(manifest_json_string: str) -> PydanticSemanticManifest: