From 264b2bfc4a66cd3403b019c223c2e3056d1a8336 Mon Sep 17 00:00:00 2001 From: Paul Yang Date: Fri, 21 Jun 2024 10:11:51 -0700 Subject: [PATCH 1/3] Move dev dependencies to respective environments in `pyproject.toml`. --- ...requirements-trino-sql-client-packages.txt | 1 - .../requirements-dev-packages.txt | 11 -- metricflow-semantics/pyproject.toml | 2 +- pyproject.toml | 102 ++++++------------ 4 files changed, 35 insertions(+), 81 deletions(-) delete mode 100644 extra-hatch-configuration/requirements-trino-sql-client-packages.txt delete mode 100644 metricflow-semantics/extra-hatch-configuration/requirements-dev-packages.txt diff --git a/extra-hatch-configuration/requirements-trino-sql-client-packages.txt b/extra-hatch-configuration/requirements-trino-sql-client-packages.txt deleted file mode 100644 index ce89b89bcc..0000000000 --- a/extra-hatch-configuration/requirements-trino-sql-client-packages.txt +++ /dev/null @@ -1 +0,0 @@ -trino>=0.328.0, <0.329.0 diff --git a/metricflow-semantics/extra-hatch-configuration/requirements-dev-packages.txt b/metricflow-semantics/extra-hatch-configuration/requirements-dev-packages.txt deleted file mode 100644 index 006feca2e5..0000000000 --- a/metricflow-semantics/extra-hatch-configuration/requirements-dev-packages.txt +++ /dev/null @@ -1,11 +0,0 @@ -# Developer tools -mypy>=1.7.0, <1.8.0 -pre-commit>=3.2.2, <3.3.0 -pytest-mock>=3.14.0, <3.15.0 -pytest-xdist>=3.6.0, <3.7.0 -pytest>=8.0.0, < 9.0.0 -PyYAML>=6.0, <7.0.0 -tabulate>=0.8.9 -types-PyYAML -types-python-dateutil -types-tabulate diff --git a/metricflow-semantics/pyproject.toml b/metricflow-semantics/pyproject.toml index dec34d0d87..7b2fcf9f71 100644 --- a/metricflow-semantics/pyproject.toml +++ b/metricflow-semantics/pyproject.toml @@ -26,7 +26,7 @@ classifiers = [ ] # Dependencies are specified through the `hatch-requirements-txt` plug-in. -dynamic = ["dependencies", "optional-dependencies"] +dynamic = ["dependencies"] [tool.hatch.metadata.hooks.requirements_txt] files = [ diff --git a/pyproject.toml b/pyproject.toml index 2374530bb7..1f0b5fbe72 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,35 +40,6 @@ files = [ Documentation = "https://docs.getdbt.com/docs/build/about-metricflow" "Source Code" = "https://github.com/dbt-labs/metricflow" -[tool.hatch.metadata.hooks.requirements_txt.optional-dependencies] -dbt-bigquery = [ - "dbt-metricflow/extra-hatch-configuration/requirements-dbt-bigquery.txt" -] -dbt-databricks = [ - "dbt-metricflow/extra-hatch-configuration/requirements-dbt-databricks.txt" -] -dbt-duckdb = [ - "dbt-metricflow/extra-hatch-configuration/requirements-dbt-duckdb.txt" -] -dbt-postgres = [ - "dbt-metricflow/extra-hatch-configuration/requirements-dbt-postgres.txt" -] -dbt-redshift = [ - "dbt-metricflow/extra-hatch-configuration/requirements-dbt-redshift.txt" -] -dbt-snowflake = [ - "dbt-metricflow/extra-hatch-configuration/requirements-dbt-snowflake.txt" -] -dbt-trino = [ - "dbt-metricflow/extra-hatch-configuration/requirements-dbt-trino.txt" -] -dev-packages = [ - "metricflow-semantics/extra-hatch-configuration/requirements-dev-packages.txt", - "dbt-metricflow/extra-hatch-configuration/requirements-cli.txt" -] -trino-sql-client-packages = [ - "extra-hatch-configuration/requirements-trino-sql-client-packages.txt" -] # There are files that hatch will always include as well (e.g. the LICENSE file). # See hatch build docs for more details. @@ -91,11 +62,28 @@ packages = ["metricflow", "metricflow-semantics/metricflow_semantics"] description = "Environment for development. Includes a DuckDB-backed client." run = "run-coverage --no-cov" -features = [ - "dev-packages", - "dbt-duckdb", +dependencies = [ + # Developer tools + "mypy>=1.7.0, <1.8.0", + "pre-commit>=3.2.2, <3.3.0", + "pytest-mock>=3.14.0, <3.15.0", + "pytest-xdist>=3.6.0, <3.7.0", + "pytest>=8.0.0, < 9.0.0", + "PyYAML>=6.0, <7.0.0", + "tabulate>=0.8.9", + "types-PyYAML", + "types-python-dateutil", + "types-tabulate", + # CLI-related + "halo>=0.0.31, <0.1.0", + "update-checker>=0.18.0, <0.19.0", + # Bug with mypy: https://github.com/pallets/click/issues/2558#issuecomment-1656546003 + "click>=8.1.6", + "dbt-core @ git+https://github.com/dbt-labs/dbt-core@e53420c1d073dc81609ae7aa84cef6ee09650576#subdirectory=core", + "dbt-duckdb>=1.8.0, <1.9.0", ] + [tool.hatch.envs.dev-env.env-vars] MF_TEST_ADAPTER_TYPE="duckdb" MF_SQL_ENGINE_URL="duckdb://" @@ -112,18 +100,12 @@ MF_TEST_ADAPTER_TYPE="postgres" [tool.hatch.envs.postgres-env] description = "Dev environment for working with Postgres adapter" - -features = [ - "dev-packages", - "dbt-postgres", -] -[tool.hatch.metadata] -allow-direct-references = true +template = "dev-env" +extra-dependencies = ["dbt-postgres>=1.8.0, <1.9.0"] - -# NOTE: All of the below should have their authentication credentials -# configured independently of the hatch env construction +## NOTE: All environments below should have their authentication credentials +## configured independently of the hatch env construction [tool.hatch.envs.bigquery-env.env-vars] PYTHONPATH="metricflow-semantics:dbt-metricflow" @@ -131,12 +113,8 @@ MF_TEST_ADAPTER_TYPE="bigquery" [tool.hatch.envs.bigquery-env] description = "Dev environment for working with the BigQuery adapter" - -features = [ - "dev-packages", - "dbt-bigquery", -] - +template = "dev-env" +extra-dependencies = ["dbt-bigquery>=1.8.0, <1.9.0"] [tool.hatch.envs.databricks-env.env-vars] PYTHONPATH="metricflow-semantics:dbt-metricflow" @@ -145,15 +123,13 @@ MF_TEST_ADAPTER_TYPE="databricks" [tool.hatch.envs.databricks-env] description = "Dev environment for working with the Databricks adapter" +template = "dev-env" +# Needed as dbt-databricks 1.8.1 depends on dbt-core~=1.8.0 while this uses dbt-core 1.9.0a1 pre-install-commands = [ "pip install dbt-databricks" ] -features = [ - "dev-packages", -] - [tool.hatch.envs.redshift-env.env-vars] PYTHONPATH="metricflow-semantics:dbt-metricflow" @@ -161,11 +137,8 @@ MF_TEST_ADAPTER_TYPE="redshift" [tool.hatch.envs.redshift-env] description = "Dev environment for working with the Redshift adapter" - -features = [ - "dev-packages", - "dbt-redshift" -] +template = "dev-env" +extra-dependencies = ["dbt-redshift>=1.8.0, <1.9.0"] [tool.hatch.envs.snowflake-env.env-vars] @@ -174,11 +147,8 @@ MF_TEST_ADAPTER_TYPE="snowflake" [tool.hatch.envs.snowflake-env] description = "Dev environment for working with Snowflake adapter" - -features = [ - "dev-packages", - "dbt-snowflake", -] +template = "dev-env" +extra-dependencies = ["dbt-snowflake>=1.8.0, <1.9.0"] [tool.hatch.envs.trino-env.env-vars] @@ -189,12 +159,8 @@ DBT_ENV_SECRET_CATALOG="memory" [tool.hatch.envs.trino-env] description = "Dev environment for working with the Trino adapter" - -features = [ - "dev-packages", - "trino-sql-client-packages", - "dbt-trino" -] +template = "dev-env" +extra-dependencies = ["trino>=0.328.0, <0.329.0", "dbt-trino>=1.8.0, <1.9.0"] [tool.black] From 2a3cf7da6e41463eccec012df316b1f4edbf0bea Mon Sep 17 00:00:00 2001 From: Paul Yang Date: Fri, 21 Jun 2024 10:13:48 -0700 Subject: [PATCH 2/3] Update dependencies for `metricflow-semantics`. --- metricflow-semantics/pyproject.toml | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/metricflow-semantics/pyproject.toml b/metricflow-semantics/pyproject.toml index 7b2fcf9f71..6ac8e6293f 100644 --- a/metricflow-semantics/pyproject.toml +++ b/metricflow-semantics/pyproject.toml @@ -33,16 +33,21 @@ files = [ "extra-hatch-configuration/requirements.txt", ] -[tool.hatch.metadata.hooks.requirements_txt.optional-dependencies] -dev-packages = [ - "extra-hatch-configuration/requirements-dev-packages.txt", -] - [tool.hatch.envs.dev-env] description = "Environment for development. Includes a DuckDB-backed client." -features = [ - "dev-packages", +dependencies = [ + # Developer tools + "mypy>=1.7.0, <1.8.0", + "pre-commit>=3.2.2, <3.3.0", + "pytest-mock>=3.14.0, <3.15.0", + "pytest-xdist>=3.6.0, <3.7.0", + "pytest>=8.0.0, < 9.0.0", + "PyYAML>=6.0, <7.0.0", + "tabulate>=0.8.9", + "types-PyYAML", + "types-python-dateutil", + "types-tabulate", ] From 6623940d26af35346a98b31e2867019c31fe8585 Mon Sep 17 00:00:00 2001 From: Paul Yang Date: Fri, 21 Jun 2024 11:46:17 -0700 Subject: [PATCH 3/3] Add CI test to check dev dependencies between `metricflow` and `metricflow-semantics`. --- .../workflows/ci-metricflow-unit-tests.yaml | 17 +++++++++ scripts/ci_tests/dev_dependency_test.py | 35 +++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 scripts/ci_tests/dev_dependency_test.py diff --git a/.github/workflows/ci-metricflow-unit-tests.yaml b/.github/workflows/ci-metricflow-unit-tests.yaml index 14cf9fea68..26604c3c41 100644 --- a/.github/workflows/ci-metricflow-unit-tests.yaml +++ b/.github/workflows/ci-metricflow-unit-tests.yaml @@ -84,3 +84,20 @@ jobs: - name: Test `metricflow` Package run: bash scripts/ci_tests/metricflow_package_test.sh + + + metricflow-project-configuration-tests: + name: MetricFlow Project Configuration Tests + runs-on: ubuntu-latest + steps: + + - name: Check-out the repo + uses: actions/checkout@v3 + + - name: Setup Python ${{ inputs.python-version }} Environment + uses: ./.github/actions/setup-python-env + with: + python-version: 3.12 + + - name: Check Dev Dependencies + run: hatch run dev-env:python scripts/ci_tests/metricflow_semantics_package_test.sh diff --git a/scripts/ci_tests/dev_dependency_test.py b/scripts/ci_tests/dev_dependency_test.py new file mode 100644 index 0000000000..70842a356e --- /dev/null +++ b/scripts/ci_tests/dev_dependency_test.py @@ -0,0 +1,35 @@ +"""Rough test to check that dev dependencies between `metricflow` and `metricflow-semantics` are the similar.""" +from __future__ import annotations + +import logging +from typing import Any, Sequence + +import tomllib + +TomlType = dict[str, Any] # type: ignore[misc] + + +def _get_dev_dependencies(toml_obj: TomlType) -> Sequence[str]: + return toml_obj["tool"]["hatch"]["envs"]["dev-env"]["dependencies"] + + +def check_dev_dependencies() -> None: # noqa: D103 + with open("pyproject.toml", "rb") as fp: + metricflow_pyproject_toml = tomllib.load(fp) + + with open("metricflow-semantics/pyproject.toml", "rb") as fp: + metricflow_semantics_pyproject_toml = tomllib.load(fp) + metricflow_dev_env_dependencies = set(_get_dev_dependencies(metricflow_pyproject_toml)) + metricflow_semantics_dev_env_dependencies = set(_get_dev_dependencies(metricflow_semantics_pyproject_toml)) + + if not metricflow_dev_env_dependencies.issuperset(metricflow_semantics_dev_env_dependencies): + raise ValueError( + f"Dev dependencies of `metricflow` are not a superset of those listed in `metricflow-semantics`" + f"\n\n{metricflow_dev_env_dependencies=}" + f"\n\n{metricflow_semantics_dev_env_dependencies=}" + ) + + +if __name__ == "__main__": + logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s", level=logging.INFO) + check_dev_dependencies()