Add test to check dev dependencies between metricflow and metricflow-semantics

.github/workflows/ci-metricflow-unit-tests.yaml

@@ -84,3 +84,20 @@ jobs:
 
       - name: Test `metricflow` Package
         run: bash scripts/ci_tests/metricflow_package_test.sh
+
+
+  metricflow-project-configuration-tests:
+    name: MetricFlow Project Configuration Tests
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Check-out the repo
+        uses: actions/checkout@v3
+
+      - name: Setup Python ${{ inputs.python-version }} Environment
+        uses: ./.github/actions/setup-python-env
+        with:
+          python-version: 3.12
+
+      - name: Check Dev Dependencies
+        run: hatch run dev-env:python scripts/ci_tests/metricflow_semantics_package_test.sh

metricflow-semantics/pyproject.toml

@@ -26,23 +26,28 @@ classifiers = [
 ]
 
 # Dependencies are specified through the `hatch-requirements-txt` plug-in.
-dynamic = ["dependencies", "optional-dependencies"]
+dynamic = ["dependencies"]
 
 [tool.hatch.metadata.hooks.requirements_txt]
 files = [
   "extra-hatch-configuration/requirements.txt",
 ]
 
-[tool.hatch.metadata.hooks.requirements_txt.optional-dependencies]
-dev-packages = [
-  "extra-hatch-configuration/requirements-dev-packages.txt",
-]
-
 [tool.hatch.envs.dev-env]
 description = "Environment for development. Includes a DuckDB-backed client."
 
-features = [
-  "dev-packages",
+dependencies = [
+  # Developer tools
+  "mypy>=1.7.0, <1.8.0",
+  "pre-commit>=3.2.2, <3.3.0",
+  "pytest-mock>=3.14.0, <3.15.0",
+  "pytest-xdist>=3.6.0, <3.7.0",
+  "pytest>=8.0.0, < 9.0.0",
+  "PyYAML>=6.0, <7.0.0",
+  "tabulate>=0.8.9",
+  "types-PyYAML",
+  "types-python-dateutil",
+  "types-tabulate",
 ]
 
 

pyproject.toml

@@ -40,35 +40,6 @@ files = [
 Documentation = "https://docs.getdbt.com/docs/build/about-metricflow"
 "Source Code" = "https://github.com/dbt-labs/metricflow"
 
-[tool.hatch.metadata.hooks.requirements_txt.optional-dependencies]
-dbt-bigquery = [
-  "dbt-metricflow/extra-hatch-configuration/requirements-dbt-bigquery.txt"
-]
-dbt-databricks = [
-  "dbt-metricflow/extra-hatch-configuration/requirements-dbt-databricks.txt"
-]
-dbt-duckdb = [
-  "dbt-metricflow/extra-hatch-configuration/requirements-dbt-duckdb.txt"
-]
-dbt-postgres = [
-  "dbt-metricflow/extra-hatch-configuration/requirements-dbt-postgres.txt"
-]
-dbt-redshift = [
-  "dbt-metricflow/extra-hatch-configuration/requirements-dbt-redshift.txt"
-]
-dbt-snowflake = [
-  "dbt-metricflow/extra-hatch-configuration/requirements-dbt-snowflake.txt"
-]
-dbt-trino = [
-  "dbt-metricflow/extra-hatch-configuration/requirements-dbt-trino.txt"
-]
-dev-packages = [
-  "metricflow-semantics/extra-hatch-configuration/requirements-dev-packages.txt",
-  "dbt-metricflow/extra-hatch-configuration/requirements-cli.txt"
-]
-trino-sql-client-packages = [
-  "extra-hatch-configuration/requirements-trino-sql-client-packages.txt"
-]
 
 # There are files that hatch will always include as well (e.g. the LICENSE file).
 # See hatch build docs for more details.
@@ -91,11 +62,28 @@ packages = ["metricflow", "metricflow-semantics/metricflow_semantics"]
 description = "Environment for development. Includes a DuckDB-backed client."
 run = "run-coverage --no-cov"
 
-features = [
-  "dev-packages",
-  "dbt-duckdb",
+dependencies = [
+  # Developer tools
+  "mypy>=1.7.0, <1.8.0",
+  "pre-commit>=3.2.2, <3.3.0",
+  "pytest-mock>=3.14.0, <3.15.0",
+  "pytest-xdist>=3.6.0, <3.7.0",
+  "pytest>=8.0.0, < 9.0.0",
+  "PyYAML>=6.0, <7.0.0",
+  "tabulate>=0.8.9",
+  "types-PyYAML",
+  "types-python-dateutil",
+  "types-tabulate",
+  # CLI-related
+  "halo>=0.0.31, <0.1.0",
+  "update-checker>=0.18.0, <0.19.0",
+  # Bug with mypy: https://github.com/pallets/click/issues/2558#issuecomment-1656546003
+  "click>=8.1.6",
+  "dbt-core @ git+https://github.com/dbt-labs/dbt-core@e53420c1d073dc81609ae7aa84cef6ee09650576#subdirectory=core",
+  "dbt-duckdb>=1.8.0, <1.9.0",
 ]
 
+
 [tool.hatch.envs.dev-env.env-vars]
 MF_TEST_ADAPTER_TYPE="duckdb"
 MF_SQL_ENGINE_URL="duckdb://"
@@ -112,31 +100,21 @@ MF_TEST_ADAPTER_TYPE="postgres"
 
 [tool.hatch.envs.postgres-env]
 description = "Dev environment for working with Postgres adapter"
-
-features = [
-  "dev-packages",
-  "dbt-postgres",
-]
-[tool.hatch.metadata]
-allow-direct-references = true
+template = "dev-env"
+extra-dependencies = ["dbt-postgres>=1.8.0, <1.9.0"]
 
 
-
-# NOTE: All of the below should have their authentication credentials
-# configured independently of the hatch env construction
+## NOTE: All environments below should have their authentication credentials
+## configured independently of the hatch env construction
 
 [tool.hatch.envs.bigquery-env.env-vars]
 PYTHONPATH="metricflow-semantics:dbt-metricflow"
 MF_TEST_ADAPTER_TYPE="bigquery"
 
 [tool.hatch.envs.bigquery-env]
 description = "Dev environment for working with the BigQuery adapter"
-
-features = [
-  "dev-packages",
-  "dbt-bigquery",
-]
-
+template = "dev-env"
+extra-dependencies = ["dbt-bigquery>=1.8.0, <1.9.0"]
 
 [tool.hatch.envs.databricks-env.env-vars]
 PYTHONPATH="metricflow-semantics:dbt-metricflow"
@@ -145,27 +123,22 @@ MF_TEST_ADAPTER_TYPE="databricks"
 
 [tool.hatch.envs.databricks-env]
 description = "Dev environment for working with the Databricks adapter"
+template = "dev-env"
 
+# Needed as dbt-databricks 1.8.1 depends on dbt-core~=1.8.0 while this uses dbt-core 1.9.0a1
 pre-install-commands = [
   "pip install dbt-databricks"
 ]
 
-features = [
-  "dev-packages",
-]
-
 
 [tool.hatch.envs.redshift-env.env-vars]
 PYTHONPATH="metricflow-semantics:dbt-metricflow"
 MF_TEST_ADAPTER_TYPE="redshift"
 
 [tool.hatch.envs.redshift-env]
 description = "Dev environment for working with the Redshift adapter"
-
-features = [
-  "dev-packages",
-  "dbt-redshift"
-]
+template = "dev-env"
+extra-dependencies = ["dbt-redshift>=1.8.0, <1.9.0"]
 
 
 [tool.hatch.envs.snowflake-env.env-vars]
@@ -174,11 +147,8 @@ MF_TEST_ADAPTER_TYPE="snowflake"
 
 [tool.hatch.envs.snowflake-env]
 description = "Dev environment for working with Snowflake adapter"
-
-features = [
-  "dev-packages",
-  "dbt-snowflake",
-]
+template = "dev-env"
+extra-dependencies = ["dbt-snowflake>=1.8.0, <1.9.0"]
 
 
 [tool.hatch.envs.trino-env.env-vars]
@@ -189,12 +159,8 @@ DBT_ENV_SECRET_CATALOG="memory"
 
 [tool.hatch.envs.trino-env]
 description = "Dev environment for working with the Trino adapter"
-
-features = [
-  "dev-packages",
-  "trino-sql-client-packages",
-  "dbt-trino"
-]
+template = "dev-env"
+extra-dependencies = ["trino>=0.328.0, <0.329.0", "dbt-trino>=1.8.0, <1.9.0"]
 
 
 [tool.black]

scripts/ci_tests/dev_dependency_test.py

@@ -0,0 +1,35 @@
+"""Rough test to check that dev dependencies between `metricflow` and `metricflow-semantics` are the similar."""
+from __future__ import annotations
+
+import logging
+from typing import Any, Sequence
+
+import tomllib
+
+TomlType = dict[str, Any]  # type: ignore[misc]
+
+
+def _get_dev_dependencies(toml_obj: TomlType) -> Sequence[str]:
+    return toml_obj["tool"]["hatch"]["envs"]["dev-env"]["dependencies"]
+
+
+def check_dev_dependencies() -> None:  # noqa: D103
+    with open("pyproject.toml", "rb") as fp:
+        metricflow_pyproject_toml = tomllib.load(fp)
+
+    with open("metricflow-semantics/pyproject.toml", "rb") as fp:
+        metricflow_semantics_pyproject_toml = tomllib.load(fp)
+    metricflow_dev_env_dependencies = set(_get_dev_dependencies(metricflow_pyproject_toml))
+    metricflow_semantics_dev_env_dependencies = set(_get_dev_dependencies(metricflow_semantics_pyproject_toml))
+
+    if not metricflow_dev_env_dependencies.issuperset(metricflow_semantics_dev_env_dependencies):
+        raise ValueError(
+            f"Dev dependencies of `metricflow` are not a superset of those listed in `metricflow-semantics`"
+            f"\n\n{metricflow_dev_env_dependencies=}"
+            f"\n\n{metricflow_semantics_dev_env_dependencies=}"
+        )
+
+
+if __name__ == "__main__":
+    logging.basicConfig(format="%(asctime)s - %(levelname)s -  %(message)s", level=logging.INFO)
+    check_dev_dependencies()