Create metricflow-semantics Package

.changes/unreleased/Under the Hood-20240425-020137.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Create `metricflow-semantics` package.
+time: 2024-04-25T02:01:37.532124-07:00
+custom:
+  Author: plypaul
+  Issue: "1150"

.github/workflows/cd-push-to-pypi.yaml

@@ -21,7 +21,7 @@ jobs:
         with:
           python-version: "${{ env.PYTHON_VERSION }}"
 
-      - name: Hatch Publish
+      - name: Hatch Publish `metricflow`
         run: hatch build && hatch publish
         env:
           HATCH_INDEX_USER: __token__

.github/workflows/ci-metricflow-unit-tests.yaml

@@ -16,7 +16,6 @@ jobs:
       matrix:
         python-version: ["3.8", "3.11"]
     steps:
-
       - name: Check-out the repo
         uses: actions/checkout@v3
 
@@ -62,3 +61,25 @@ jobs:
 
       - name: Check success
         run: test ${{ needs.metricflow-unit-tests-duckdb.result }} = 'success' -a ${{ needs.metricflow-unit-tests-postgres.result }} = 'success'
+
+  metricflow-package-tests:
+    name: MetricFlow Packages Tests
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [ "3.8", "3.11" ]
+    steps:
+
+      - name: Check-out the repo
+        uses: actions/checkout@v3
+
+      - name: Setup Python ${{ inputs.python-version }} Environment
+        uses: ./.github/actions/setup-python-env
+        with:
+          python-version: "${{ matrix.python-version }}"
+
+      - name: Test `metricflow-semantics` Package
+        run: bash scripts/ci_tests/metricflow_semantics_package_test.sh
+
+      - name: Test `metricflow` Package
+        run: bash scripts/ci_tests/metricflow_package_test.sh

Makefile

@@ -9,7 +9,9 @@ ADDITIONAL_PYTEST_OPTIONS =
 
 # Pytest that can populate the persistent source schema
 USE_PERSISTENT_SOURCE_SCHEMA = --use-persistent-source-schema
-POPULATE_PERSISTENT_SOURCE_SCHEMA = tests/source_schema_tools.py::populate_source_schema
+TESTS_METRICFLOW = tests_metricflow
+TESTS_METRICFLOW_SEMANTICS = tests_metricflow_semantics
+POPULATE_PERSISTENT_SOURCE_SCHEMA = $(TESTS_METRICFLOW)/source_schema_tools.py::populate_source_schema
 
 # Install Hatch package / project manager
 .PHONY: install-hatch
@@ -19,32 +21,33 @@ install-hatch:
 # Testing and linting
 .PHONY: test
 test:
-	hatch -v run dev-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) tests/
+	cd metricflow-semantics && hatch -v run dev-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) $(TESTS_METRICFLOW_SEMANTICS)/
+	hatch -v run dev-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) $(TESTS_METRICFLOW)/
 
 .PHONY: test-postgresql
 test-postgresql:
-	hatch -v run postgres-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) tests/
+	hatch -v run postgres-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) $(TESTS_METRICFLOW)/
 
 # Engine-specific test environments.
 .PHONY: test-bigquery
 test-bigquery:
-	hatch -v run bigquery-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) tests/
+	hatch -v run bigquery-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) $(TESTS_METRICFLOW)/
 
 .PHONY: populate-persistent-source-schema-bigquery
 populate-persistent-source-schema-bigquery:
 	hatch -v run bigquery-env:pytest -vv $(ADDITIONAL_PYTEST_OPTIONS) $(USE_PERSISTENT_SOURCE_SCHEMA) $(POPULATE_PERSISTENT_SOURCE_SCHEMA)
 
 .PHONY: test-databricks
 test-databricks:
-	hatch -v run databricks-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) tests/
+	hatch -v run databricks-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) $(TESTS_METRICFLOW)/
 
 .PHONY: populate-persistent-source-schema-databricks
 populate-persistent-source-schema-databricks:
 	hatch -v run databricks-env:pytest -vv $(ADDITIONAL_PYTEST_OPTIONS) $(USE_PERSISTENT_SOURCE_SCHEMA) $(POPULATE_PERSISTENT_SOURCE_SCHEMA)
 
 .PHONY: test-redshift
 test-redshift:
-	hatch -v run redshift-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) tests/
+	hatch -v run redshift-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) $(TESTS_METRICFLOW)/
 
 .PHONY: populate-persistent-source-schema-redshift
 populate-persistent-source-schema-redshift:
@@ -53,15 +56,15 @@ populate-persistent-source-schema-redshift:
 
 .PHONY: test-snowflake
 test-snowflake:
-	hatch -v run snowflake-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) tests/
+	hatch -v run snowflake-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) $(TESTS_METRICFLOW)/
 
 .PHONY: populate-persistent-source-schema-snowflake
 populate-persistent-source-schema-snowflake:
 	hatch -v run snowflake-env:pytest -vv $(ADDITIONAL_PYTEST_OPTIONS) $(USE_PERSISTENT_SOURCE_SCHEMA) $(POPULATE_PERSISTENT_SOURCE_SCHEMA)
 
 .PHONY: test-trino
 test-trino:
-	hatch -v run trino-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) tests/
+	hatch -v run trino-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) $(TESTS_METRICFLOW)/
 
 .PHONY: lint
 lint:
@@ -79,12 +82,12 @@ trino:
 # Re-generate test snapshots using all supported SQL engines.
 .PHONY: regenerate-test-snapshots
 regenerate-test-snapshots:
-	hatch -v run dev-env:python tests/generate_snapshots.py
+	hatch -v run dev-env:python tests_metricflow/generate_snapshots.py
 
 # Populate persistent source schemas for all relevant SQL engines.
 .PHONY: populate-persistent-source-schemas
 populate-persistent-source-schemas:
-	hatch -v run dev-env:python tests/populate_persistent_source_schemas.py
+	hatch -v run dev-env:python $(TESTS_METRICFLOW)/populate_persistent_source_schemas.py
 
 # Re-generate snapshots for the default SQL engine.
 .PHONY: test-snap

dbt-metricflow/dbt_metricflow/cli/cli_context.py

@@ -6,11 +6,11 @@
 from typing import Dict, Optional
 
 from dbt_semantic_interfaces.protocols.semantic_manifest import SemanticManifest
+from metricflow_semantics.model.semantic_manifest_lookup import SemanticManifestLookup
 
 from dbt_metricflow.cli.dbt_connectors.adapter_backed_client import AdapterBackedSqlClient
 from dbt_metricflow.cli.dbt_connectors.dbt_config_accessor import dbtArtifacts, dbtProjectMetadata
 from metricflow.engine.metricflow_engine import MetricFlowEngine
-from metricflow.model.semantic_manifest_lookup import SemanticManifestLookup
 from metricflow.protocols.sql_client import SqlClient
 
 logger = logging.getLogger(__name__)

dbt-metricflow/dbt_metricflow/cli/dbt_connectors/adapter_backed_client.py

@@ -8,12 +8,13 @@
 from dbt.adapters.base.impl import BaseAdapter
 from dbt.exceptions import DbtDatabaseError
 from dbt_semantic_interfaces.enum_extension import assert_values_exhausted
+from metricflow_semantics.errors.error_classes import SqlBindParametersNotSupportedError
+from metricflow_semantics.mf_logging.formatting import indent
+from metricflow_semantics.mf_logging.pretty_print import mf_pformat
+from metricflow_semantics.random_id import random_id
+from metricflow_semantics.sql.sql_bind_parameters import SqlBindParameters
 
-from metricflow.errors.errors import SqlBindParametersNotSupportedError
-from metricflow.mf_logging.formatting import indent
-from metricflow.mf_logging.pretty_print import mf_pformat
 from metricflow.protocols.sql_client import SqlEngine
-from metricflow.random_id import random_id
 from metricflow.sql.render.big_query import BigQuerySqlQueryPlanRenderer
 from metricflow.sql.render.databricks import DatabricksSqlQueryPlanRenderer
 from metricflow.sql.render.duckdb_renderer import DuckDbSqlQueryPlanRenderer
@@ -22,7 +23,6 @@
 from metricflow.sql.render.snowflake import SnowflakeSqlQueryPlanRenderer
 from metricflow.sql.render.sql_plan_renderer import SqlQueryPlanRenderer
 from metricflow.sql.render.trino import TrinoSqlQueryPlanRenderer
-from metricflow.sql.sql_bind_parameters import SqlBindParameters
 from metricflow.sql_request.sql_request_attributes import SqlRequestId
 
 logger = logging.getLogger(__name__)

dbt-metricflow/dbt_metricflow/cli/dbt_connectors/dbt_config_accessor.py

@@ -12,12 +12,11 @@
 from dbt.config.project import Project
 from dbt.config.runtime import load_profile, load_project
 from dbt_semantic_interfaces.protocols.semantic_manifest import SemanticManifest
+from metricflow_semantics.errors.error_classes import ModelCreationException
+from metricflow_semantics.mf_logging.pretty_print import mf_pformat
+from metricflow_semantics.model.dbt_manifest_parser import parse_manifest_from_dbt_generated_manifest
 from typing_extensions import Self
 
-from metricflow.errors.errors import ModelCreationException
-from metricflow.mf_logging.pretty_print import mf_pformat
-from metricflow.model.dbt_manifest_parser import parse_manifest_from_dbt_generated_manifest
-
 logger = logging.getLogger(__name__)
 
 

dbt-metricflow/dbt_metricflow/cli/main.py

@@ -18,6 +18,7 @@
 from dbt_semantic_interfaces.validations.semantic_manifest_validator import SemanticManifestValidator
 from dbt_semantic_interfaces.validations.validator_helpers import SemanticManifestValidationResults
 from halo import Halo
+from metricflow_semantics.dag.dag_visualization import display_dag_as_svg
 from update_checker import UpdateChecker
 
 import dbt_metricflow.cli.custom_click_types as click_custom
@@ -35,11 +36,10 @@
     query_options,
     start_end_time_options,
 )
-from metricflow.dag.dag_visualization import display_dag_as_svg
 from metricflow.engine.metricflow_engine import MetricFlowExplainResult, MetricFlowQueryRequest, MetricFlowQueryResult
-from metricflow.model.data_warehouse_model_validator import DataWarehouseModelValidator
 from metricflow.telemetry.models import TelemetryLevel
 from metricflow.telemetry.reporter import TelemetryReporter, log_call
+from metricflow.validation.data_warehouse_model_validator import DataWarehouseModelValidator
 
 logger = logging.getLogger(__name__)
 

metricflow-semantics/LICENSE

@@ -0,0 +1,90 @@
+License text copyright © 2017 MariaDB Corporation Ab, All Rights Reserved.
+“Business Source License” is a trademark of MariaDB Corporation Ab.
+
+Licensed Work: dbt MetricFlow
+
+Change Date: The change date for each release of the Licensed Work will
+be the 3rd anniversary of the initial release date of that
+release of the Licensed Work.
+
+Change License: Apache 2.0 https://www.apache.org/licenses/LICENSE-2.0
+
+Additional Use Grant: The Licensor hereby grants you the right to use the
+Licensed Work for production purposes that consist of
+your internal business purposes, but not for the purpose
+of providing managed services or software-as-a-service
+to others.
+
+Terms
+
+The Licensor hereby grants you the right to copy, modify, create derivative
+works, redistribute, and make non-production use of the Licensed Work. The
+Licensor may make an Additional Use Grant, above, permitting limited production
+use.
+
+Effective on the Change Date, or the fourth anniversary of the first publicly
+available distribution of a specific version of the Licensed Work under this
+License, whichever comes first, the Licensor hereby grants you rights under the
+terms of the Change License, and the rights granted in the paragraph above
+terminate.
+
+If your use of the Licensed Work does not comply with the requirements currently
+in effect as described in this License, you must purchase a commercial license
+from the Licensor, its affiliated entities, or authorized resellers, or you must
+refrain from using the Licensed Work.
+
+All copies of the original and modified Licensed Work, and derivative works of
+the Licensed Work, are subject to this License. This License applies separately
+for each version of the Licensed Work and the Change Date may vary for each
+version of the Licensed Work released by Licensor.
+
+You must conspicuously display this License on each original or modified copy of
+the Licensed Work. If you receive the Licensed Work in original or modified form
+from a third party, the terms and conditions set forth in this License apply to
+your use of that work.
+
+Any use of the Licensed Work in violation of this License will automatically
+terminate your rights under this License for the current and all other versions
+of the Licensed Work.
+
+This License does not grant you any right in any trademark or logo of Licensor
+or its affiliates (provided that you may use a trademark or logo of Licensor as
+expressly required by this License).TO THE EXTENT PERMITTED BY APPLICABLE LAW,
+THE LICENSED WORK IS PROVIDED ON AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL
+WARRANTIES AND CONDITIONS, EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION)
+WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE,
+NON-INFRINGEMENT, AND TITLE.MariaDB hereby grants you permission to use this
+License’s text to license your works, and to refer to it using the trademark
+“Business Source License”, as long as you comply with the Covenants of Licensor
+below.
+
+Covenants of Licensor
+
+In consideration of the right to use this License’s text and the “Business
+Source License” name and trademark, Licensor covenants to MariaDB, and to all
+other recipients of the licensed work to be provided by Licensor:
+
+To specify as the Change License the GPL Version 2.0 or any later version, or a
+license that is compatible with GPL Version 2.0 or a later version, where
+“compatible” means that software provided under the Change License can be
+included in a program with software provided under GPL Version 2.0 or a later
+version. Licensor may specify additional Change Licenses without limitation.
+
+To either: (a) specify an additional grant of rights to use that does not impose
+any additional restriction on the right granted in this License, as the
+Additional Use Grant; or (b) insert the text “None”.
+
+To specify a Change Date.
+
+Not to modify this License in any other way.
+
+Notice
+
+The Business Source License (this document, or the “License”) is not an Open
+Source license. However, the Licensed Work will eventually be made available
+under an Open Source License, as stated in this License.
+
+For more information on the use of the Business Source License for MariaDB
+products, please visit the MariaDB Business Source License FAQ. For more
+information on the use of the Business Source License generally, please visit
+the Adopting and Developing Business Source License FAQ.

metricflow-semantics/README.md

@@ -0,0 +1,15 @@
+<p align="center">
+    <a target="_blank" href="https://twitter.com/dbt_labs">
+    <img src="https://img.shields.io/twitter/follow/dbt_labs?labelColor=image.png&color=163B36&logo=twitter&style=flat">
+  </a>
+    <a target="_blank" href="https://www.getdbt.com/community/">
+    <img src="https://img.shields.io/badge/Slack-join-163B36">
+  </a>
+    <a href="https://github.com/psf/black"><img src="https://img.shields.io/badge/code%20style-black-000000.svg" /></a>
+</p>
+
+# Welcome to metricflow-semantics
+
+`metricflow-semantics` encapsulates the modules needed to resolve the semantics of MetricFlow queries.
+## Repo use cases
+- Resolving the dbt model dependencies of a (saved) query in MetricFlow for DAG dependency management.

metricflow/dag/dag_to_text.py → metricflow-semantics/metricflow_semantics/dag/dag_to_text.py

@@ -12,9 +12,9 @@
 import jinja2
 
 if typing.TYPE_CHECKING:
-    from metricflow.dag.mf_dag import DagNode, DagNodeT, DisplayedProperty, MetricFlowDag
+    from metricflow_semantics.dag.mf_dag import DagNode, DagNodeT, DisplayedProperty, MetricFlowDag
 
-from metricflow.mf_logging.pretty_print import mf_pformat
+from metricflow_semantics.mf_logging.pretty_print import mf_pformat
 
 logger = logging.getLogger(__name__)
 

metricflow/dag/dag_visualization.py → metricflow-semantics/metricflow_semantics/dag/dag_visualization.py

@@ -6,8 +6,8 @@
 
 import graphviz
 
-from metricflow.dag.mf_dag import DagNode, MetricFlowDag
-from metricflow.random_id import random_id
+from metricflow_semantics.dag.mf_dag import DagNode, MetricFlowDag
+from metricflow_semantics.random_id import random_id
 
 logger = logging.getLogger(__name__)
 DagNodeT = TypeVar("DagNodeT", bound=DagNode)

metricflow/dag/mf_dag.py → metricflow-semantics/metricflow_semantics/dag/mf_dag.py

@@ -11,10 +11,10 @@
 
 import jinja2
 
-from metricflow.dag.dag_to_text import MetricFlowDagTextFormatter
-from metricflow.dag.id_prefix import IdPrefix
-from metricflow.dag.sequential_id import SequentialIdGenerator
-from metricflow.visitor import VisitorOutputT
+from metricflow_semantics.dag.dag_to_text import MetricFlowDagTextFormatter
+from metricflow_semantics.dag.id_prefix import IdPrefix
+from metricflow_semantics.dag.sequential_id import SequentialIdGenerator
+from metricflow_semantics.visitor import VisitorOutputT
 
 logger = logging.getLogger(__name__)
 

metricflow/dag/sequential_id.py → metricflow-semantics/metricflow_semantics/dag/sequential_id.py

@@ -6,7 +6,7 @@
 
 from typing_extensions import override
 
-from metricflow.dag.id_prefix import IdPrefix
+from metricflow_semantics.dag.id_prefix import IdPrefix
 
 
 @dataclass(frozen=True)

metricflow/filters/time_constraint.py → metricflow-semantics/metricflow_semantics/filters/time_constraint.py

@@ -9,7 +9,7 @@
 from dbt_semantic_interfaces.dataclass_serialization import SerializableDataclass
 from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity
 
-from metricflow.time.time_granularity import offset_period
+from metricflow_semantics.time.time_granularity import offset_period
 
 logger = logging.getLogger(__name__)
 

metricflow/instances.py → metricflow-semantics/metricflow_semantics/instances.py

@@ -9,9 +9,9 @@
 from dbt_semantic_interfaces.dataclass_serialization import SerializableDataclass
 from dbt_semantic_interfaces.references import MetricModelReference, SemanticModelElementReference
 
-from metricflow.aggregation_properties import AggregationState
-from metricflow.specs.column_assoc import ColumnAssociation
-from metricflow.specs.specs import (
+from metricflow_semantics.aggregation_properties import AggregationState
+from metricflow_semantics.specs.column_assoc import ColumnAssociation
+from metricflow_semantics.specs.spec_classes import (
     DimensionSpec,
     EntitySpec,
     GroupByMetricSpec,