Merge branch 'main' into time_dimension_filter

.changes/unreleased/Under the Hood-20231107-184138.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Removed DatePart Enum and change imports to depend on DSI version instead.
+time: 2023-11-07T18:41:38.606807-06:00
+custom:
+  Author: DevonFulcher
+  Issue: None

Makefile

@@ -74,6 +74,11 @@ postgresql postgres:
 regenerate-test-snapshots:
 	hatch -v run dev-env:python metricflow/test/generate_snapshots.py
 
+# Populate persistent source schemas for all relevant SQL engines.
+.PHONY: populate-persistent-source-schemas
+populate-persistent-source-schemas:
+	hatch -v run dev-env:python metricflow/test/populate_persistent_source_schemas.py
+
 # Re-generate snapshots for the default SQL engine.
 .PHONY: test-snap
 test-snap:

metricflow/dataset/convert_semantic_model.py

@@ -10,6 +10,7 @@
 from dbt_semantic_interfaces.protocols.measure import Measure
 from dbt_semantic_interfaces.protocols.semantic_model import SemanticModel
 from dbt_semantic_interfaces.references import SemanticModelElementReference, SemanticModelReference
+from dbt_semantic_interfaces.type_enums.date_part import DatePart
 from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity
 
 from metricflow.aggregation_properties import AggregationState
@@ -47,7 +48,6 @@
     SqlSelectStatementNode,
     SqlTableFromClauseNode,
 )
-from metricflow.time.date_part import DatePart
 
 logger = logging.getLogger(__name__)
 

metricflow/dataset/dataset.py

@@ -4,12 +4,12 @@
 from typing import Optional, Sequence
 
 from dbt_semantic_interfaces.references import TimeDimensionReference
+from dbt_semantic_interfaces.type_enums.date_part import DatePart
 from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity
 from dbt_semantic_interfaces.validations.unique_valid_name import MetricFlowReservedKeywords
 
 from metricflow.instances import InstanceSet, TimeDimensionInstance
 from metricflow.specs.specs import TimeDimensionSpec
-from metricflow.time.date_part import DatePart
 
 logger = logging.getLogger(__name__)
 

metricflow/naming/linkable_spec_name.py

@@ -4,10 +4,9 @@
 from dataclasses import dataclass
 from typing import Optional, Tuple
 
+from dbt_semantic_interfaces.type_enums.date_part import DatePart
 from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity
 
-from metricflow.time.date_part import DatePart
-
 DUNDER = "__"
 
 logger = logging.getLogger(__name__)

metricflow/plan_conversion/instance_converters.py

@@ -9,6 +9,7 @@
 from typing import Dict, List, Optional, Sequence, Tuple
 
 from dbt_semantic_interfaces.references import SemanticModelReference
+from dbt_semantic_interfaces.type_enums.date_part import DatePart
 from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity
 from more_itertools import bucket
 
@@ -47,7 +48,6 @@
     SqlFunctionExpression,
 )
 from metricflow.sql.sql_plan import SqlSelectColumn
-from metricflow.time.date_part import DatePart
 
 logger = logging.getLogger(__name__)
 

metricflow/protocols/query_parameter.py

@@ -3,8 +3,7 @@
 from typing import Optional, Protocol, Union, runtime_checkable
 
 from dbt_semantic_interfaces.type_enums import TimeGranularity
-
-from metricflow.time.date_part import DatePart
+from dbt_semantic_interfaces.type_enums.date_part import DatePart
 
 
 @runtime_checkable

metricflow/query/query_parser.py

@@ -21,6 +21,7 @@
     MetricReference,
     TimeDimensionReference,
 )
+from dbt_semantic_interfaces.type_enums.date_part import DatePart
 from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity
 
 from metricflow.assert_one_arg import assert_exactly_one_arg_set
@@ -55,7 +56,6 @@
     WhereFilterSpec,
 )
 from metricflow.specs.where_filter_transform import WhereSpecFactory
-from metricflow.time.date_part import DatePart
 from metricflow.time.time_granularity_solver import (
     PartialTimeDimensionSpec,
     RequestTimeGranularityException,

metricflow/specs/query_param_implementations.py

@@ -4,13 +4,13 @@
 from typing import Optional
 
 from dbt_semantic_interfaces.protocols import ProtocolHint
+from dbt_semantic_interfaces.type_enums.date_part import DatePart
 from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity
 from typing_extensions import override
 
 from metricflow.naming.linkable_spec_name import StructuredLinkableSpecName
 from metricflow.protocols.query_parameter import InputOrderByParameter
 from metricflow.protocols.query_parameter import SavedQueryParameter as SavedQueryParameterProtocol
-from metricflow.time.date_part import DatePart
 
 
 @dataclass(frozen=True)

metricflow/specs/specs.py

@@ -26,14 +26,14 @@
     TimeDimensionReference,
 )
 from dbt_semantic_interfaces.type_enums.aggregation_type import AggregationType
+from dbt_semantic_interfaces.type_enums.date_part import DatePart
 from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity
 
 from metricflow.aggregation_properties import AggregationState
 from metricflow.filters.time_constraint import TimeRangeConstraint
 from metricflow.naming.linkable_spec_name import StructuredLinkableSpecName
 from metricflow.sql.sql_bind_parameters import SqlBindParameters
 from metricflow.sql.sql_column_type import SqlColumnType
-from metricflow.time.date_part import DatePart
 from metricflow.visitor import VisitorOutputT
 
 

metricflow/sql/render/big_query.py

@@ -4,6 +4,7 @@
 from typing import Collection
 
 from dbt_semantic_interfaces.enum_extension import assert_values_exhausted
+from dbt_semantic_interfaces.type_enums.date_part import DatePart
 from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity
 from typing_extensions import override
 
@@ -25,7 +26,6 @@
     SqlSubtractTimeIntervalExpression,
 )
 from metricflow.sql.sql_plan import SqlSelectColumn
-from metricflow.time.date_part import DatePart
 
 
 class BigQuerySqlExpressionRenderer(DefaultSqlExpressionRenderer):

metricflow/sql/render/databricks.py

@@ -3,6 +3,7 @@
 from typing import Collection
 
 from dbt_semantic_interfaces.enum_extension import assert_values_exhausted
+from dbt_semantic_interfaces.type_enums.date_part import DatePart
 from typing_extensions import override
 
 from metricflow.errors.errors import UnsupportedEngineFeatureError
@@ -13,7 +14,6 @@
 )
 from metricflow.sql.render.sql_plan_renderer import DefaultSqlQueryPlanRenderer
 from metricflow.sql.sql_exprs import SqlPercentileExpression, SqlPercentileFunctionType
-from metricflow.time.date_part import DatePart
 
 
 class DatabricksSqlExpressionRenderer(DefaultSqlExpressionRenderer):

metricflow/sql/render/expr_renderer.py

@@ -8,6 +8,7 @@
 from typing import Collection, List
 
 import jinja2
+from dbt_semantic_interfaces.type_enums.date_part import DatePart
 from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity
 from typing_extensions import override
 
@@ -37,7 +38,6 @@
     SqlWindowFunctionExpression,
 )
 from metricflow.sql.sql_plan import SqlSelectColumn
-from metricflow.time.date_part import DatePart
 
 logger = logging.getLogger(__name__)
 

metricflow/sql/render/redshift.py

@@ -3,6 +3,7 @@
 from typing import Collection
 
 from dbt_semantic_interfaces.enum_extension import assert_values_exhausted
+from dbt_semantic_interfaces.type_enums.date_part import DatePart
 from typing_extensions import override
 
 from metricflow.errors.errors import UnsupportedEngineFeatureError
@@ -19,7 +20,6 @@
     SqlPercentileExpression,
     SqlPercentileFunctionType,
 )
-from metricflow.time.date_part import DatePart
 
 
 class RedshiftSqlExpressionRenderer(DefaultSqlExpressionRenderer):

metricflow/sql/render/snowflake.py

@@ -3,6 +3,7 @@
 from typing import Collection
 
 from dbt_semantic_interfaces.enum_extension import assert_values_exhausted
+from dbt_semantic_interfaces.type_enums.date_part import DatePart
 from typing_extensions import override
 
 from metricflow.errors.errors import UnsupportedEngineFeatureError
@@ -14,7 +15,6 @@
 from metricflow.sql.render.sql_plan_renderer import DefaultSqlQueryPlanRenderer
 from metricflow.sql.sql_bind_parameters import SqlBindParameters
 from metricflow.sql.sql_exprs import SqlGenerateUuidExpression, SqlPercentileExpression, SqlPercentileFunctionType
-from metricflow.time.date_part import DatePart
 
 
 class SnowflakeSqlExpressionRenderer(DefaultSqlExpressionRenderer):

metricflow/sql/sql_exprs.py

@@ -12,6 +12,7 @@
 from dbt_semantic_interfaces.enum_extension import assert_values_exhausted
 from dbt_semantic_interfaces.protocols.measure import MeasureAggregationParameters
 from dbt_semantic_interfaces.type_enums.aggregation_type import AggregationType
+from dbt_semantic_interfaces.type_enums.date_part import DatePart
 from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity
 
 from metricflow.dag.id_generation import (
@@ -35,7 +36,6 @@
 )
 from metricflow.dag.mf_dag import DagNode, DisplayedProperty, NodeId
 from metricflow.sql.sql_bind_parameters import SqlBindParameters
-from metricflow.time.date_part import DatePart
 from metricflow.visitor import Visitable, VisitorOutputT
 
 

metricflow/test/generate_snapshots.py

@@ -37,7 +37,7 @@
 import logging
 import os
 from dataclasses import dataclass
-from typing import Optional, Sequence
+from typing import Callable, Optional, Sequence
 
 from dbt_semantic_interfaces.enum_extension import assert_values_exhausted
 from dbt_semantic_interfaces.implementations.base import FrozenBaseModel
@@ -107,7 +107,11 @@ def run_command(command: str) -> None:  # noqa: D
         raise RuntimeError(f"Error running command: {command}")
 
 
-def run_tests(test_configuration: MetricFlowTestConfiguration) -> None:  # noqa: D
+def set_engine_env_variables(test_configuration: MetricFlowTestConfiguration) -> None:
+    """Set connection env variables dynamically for the engine being used.
+
+    Requires MF_TEST_ENGINE_CREDENTIALS env variable to be set with creds for all engines.
+    """
     if test_configuration.credential_set.engine_url is None:
         if "MF_SQL_ENGINE_URL" in os.environ:
             del os.environ["MF_SQL_ENGINE_URL"]
@@ -120,6 +124,10 @@ def run_tests(test_configuration: MetricFlowTestConfiguration) -> None:  # noqa:
     else:
         os.environ["MF_SQL_ENGINE_PASSWORD"] = test_configuration.credential_set.engine_password
 
+
+def run_tests(test_configuration: MetricFlowTestConfiguration) -> None:  # noqa: D
+    set_engine_env_variables(test_configuration)
+
     if test_configuration.engine is SqlEngine.DUCKDB:
         # DuckDB is fast, so generate all snapshots, including the engine-agnostic ones
         run_command(f"pytest -x -vv -n 4 --overwrite-snapshots -k 'not itest' {TEST_DIRECTORY}")
@@ -145,7 +153,7 @@ def run_tests(test_configuration: MetricFlowTestConfiguration) -> None:  # noqa:
         assert_values_exhausted(test_configuration.engine)
 
 
-def run_cli() -> None:  # noqa: D
+def run_cli(function_to_run: Callable) -> None:  # noqa: D
     # Setup logging.
     dev_format = "%(asctime)s %(levelname)s %(filename)s:%(lineno)d [%(threadName)s] - %(message)s"
     logging.basicConfig(level=logging.INFO, format=dev_format)
@@ -165,8 +173,8 @@ def run_cli() -> None:  # noqa: D
         logger.info(
             f"Running tests for {test_configuration.engine} with URL: {test_configuration.credential_set.engine_url}"
         )
-        run_tests(test_configuration)
+        function_to_run(test_configuration)
 
 
 if __name__ == "__main__":
-    run_cli()
+    run_cli(run_tests)

metricflow/test/integration/test_configured_cases.py

@@ -11,6 +11,7 @@
 from dbt_semantic_interfaces.enum_extension import assert_values_exhausted
 from dbt_semantic_interfaces.implementations.elements.measure import PydanticMeasureAggregationParameters
 from dbt_semantic_interfaces.test_utils import as_datetime
+from dbt_semantic_interfaces.type_enums.date_part import DatePart
 from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity
 
 from metricflow.engine.metricflow_engine import MetricFlowEngine, MetricFlowQueryRequest
@@ -43,7 +44,6 @@
 from metricflow.test.time.configurable_time_source import (
     ConfigurableTimeSource,
 )
-from metricflow.time.date_part import DatePart
 
 logger = logging.getLogger(__name__)
 

metricflow/test/populate_persistent_source_schemas.py

@@ -0,0 +1,45 @@
+"""Script to help generate persistent source schemas with test data for all relevant engines."""
+
+from __future__ import annotations
+
+import logging
+import os
+
+from dbt_semantic_interfaces.enum_extension import assert_values_exhausted
+
+from metricflow.protocols.sql_client import SqlEngine
+from metricflow.test.generate_snapshots import (
+    MetricFlowTestConfiguration,
+    run_cli,
+    run_command,
+    set_engine_env_variables,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def populate_schemas(test_configuration: MetricFlowTestConfiguration) -> None:  # noqa: D
+    set_engine_env_variables(test_configuration)
+
+    if test_configuration.engine is SqlEngine.DUCKDB or test_configuration.engine is SqlEngine.POSTGRES:
+        # DuckDB & Postgres don't use persistent source schema
+        return None
+    elif (
+        test_configuration.engine is SqlEngine.SNOWFLAKE
+        or test_configuration.engine is SqlEngine.BIGQUERY
+        or test_configuration.engine is SqlEngine.DATABRICKS
+        or test_configuration.engine is SqlEngine.REDSHIFT
+    ):
+        engine_name = test_configuration.engine.value.lower()
+        os.environ["MF_TEST_ADAPTER_TYPE"] = engine_name
+        hatch_env = f"{engine_name}-env"
+        run_command(
+            f"hatch -v run {hatch_env}:pytest -vv --use-persistent-source-schema "
+            "metricflow/test/source_schema_tools.py::populate_source_schema"
+        )
+    else:
+        assert_values_exhausted(test_configuration.engine)
+
+
+if __name__ == "__main__":
+    run_cli(populate_schemas)

metricflow/test/query/test_query_parser.py

@@ -8,6 +8,7 @@
 from dbt_semantic_interfaces.parsing.objects import YamlConfigFile
 from dbt_semantic_interfaces.references import EntityReference
 from dbt_semantic_interfaces.test_utils import as_datetime
+from dbt_semantic_interfaces.type_enums.date_part import DatePart
 from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity
 
 from metricflow.errors.errors import UnableToSatisfyQueryError
@@ -30,7 +31,6 @@
 from metricflow.test.fixtures.model_fixtures import query_parser_from_yaml
 from metricflow.test.model.example_project_configuration import EXAMPLE_PROJECT_CONFIGURATION_YAML_CONFIG_FILE
 from metricflow.test.time.metric_time_dimension import MTD
-from metricflow.time.date_part import DatePart
 from metricflow.time.time_granularity_solver import RequestTimeGranularityException
 
 logger = logging.getLogger(__name__)

metricflow/test/query_rendering/test_granularity_date_part_rendering.py

@@ -8,6 +8,7 @@
 
 import pytest
 from _pytest.fixtures import FixtureRequest
+from dbt_semantic_interfaces.type_enums.date_part import DatePart
 from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity
 
 from metricflow.dataflow.builder.dataflow_plan_builder import DataflowPlanBuilder
@@ -20,7 +21,6 @@
 )
 from metricflow.test.fixtures.setup_fixtures import MetricFlowTestSessionState
 from metricflow.test.query_rendering.compare_rendered_query import convert_and_check
-from metricflow.time.date_part import DatePart
 
 
 @pytest.mark.sql_engine_snapshot

metricflow/test/sql/test_sql_expr_render.py

@@ -4,6 +4,7 @@
 import textwrap
 
 import pytest
+from dbt_semantic_interfaces.type_enums.date_part import DatePart
 from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity
 
 from metricflow.sql.render.expr_renderer import DefaultSqlExpressionRenderer
@@ -30,7 +31,6 @@
     SqlWindowFunctionExpression,
     SqlWindowOrderByArgument,
 )
-from metricflow.time.date_part import DatePart
 
 logger = logging.getLogger(__name__)