diff --git a/.changes/unreleased/Features-20240603-174544.yaml b/.changes/unreleased/Features-20240603-174544.yaml new file mode 100644 index 0000000000..a70200fe89 --- /dev/null +++ b/.changes/unreleased/Features-20240603-174544.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Update `dbt-metricflow` dependencies to use `dbt*` 1.8 packages +time: 2024-06-03T17:45:44.215307-07:00 +custom: + Author: plypaul + Issue: "1243" diff --git a/.github/workflows/ci-metricflow-unit-tests.yaml b/.github/workflows/ci-metricflow-unit-tests.yaml index 9106eaf83a..d135da5927 100644 --- a/.github/workflows/ci-metricflow-unit-tests.yaml +++ b/.github/workflows/ci-metricflow-unit-tests.yaml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.11"] + python-version: ["3.8", "3.12"] steps: - name: Check-out the repo uses: actions/checkout@v3 @@ -46,10 +46,10 @@ jobs: - name: Check-out the repo uses: actions/checkout@v3 - - name: Test w/ Python 3.11 + - name: Test w/ Python 3.12 uses: ./.github/actions/run-mf-tests with: - python-version: "3.11" + python-version: "3.12" make-target: "test-postgresql" metricflow-unit-tests: diff --git a/dbt-metricflow/dbt_metricflow/cli/dbt_connectors/adapter_backed_client.py b/dbt-metricflow/dbt_metricflow/cli/dbt_connectors/adapter_backed_client.py index 4dcc8a8f84..98d76916b9 100644 --- a/dbt-metricflow/dbt_metricflow/cli/dbt_connectors/adapter_backed_client.py +++ b/dbt-metricflow/dbt_metricflow/cli/dbt_connectors/adapter_backed_client.py @@ -5,7 +5,7 @@ import time from dbt.adapters.base.impl import BaseAdapter -from dbt.exceptions import DbtDatabaseError +from dbt_common.exceptions.base import DbtDatabaseError from dbt_semantic_interfaces.enum_extension import assert_values_exhausted from metricflow_semantics.errors.error_classes import SqlBindParametersNotSupportedError from metricflow_semantics.mf_logging.formatting import indent @@ -132,9 +132,8 @@ def query( Args: stmt: The SQL query statement to run. This should produce output via a SELECT - sql_bind_parameters: The parameter replacement mapping for filling in - concrete values for SQL query parameters. - extra_tags: An object containing JSON serialized tags meant for annotating queries. + sql_bind_parameters: The parameter replacement mapping for filling in concrete values for SQL query + parameters. """ start = time.time() request_id = SqlRequestId(f"mf_rid__{random_id()}") diff --git a/dbt-metricflow/extra-hatch-configuration/requirements-cli.txt b/dbt-metricflow/extra-hatch-configuration/requirements-cli.txt new file mode 100644 index 0000000000..ae9d0ae471 --- /dev/null +++ b/dbt-metricflow/extra-hatch-configuration/requirements-cli.txt @@ -0,0 +1,13 @@ +# Internal dependencies +dbt-core>=1.8.0, <1.9.0 + +# dsi version should be fixed by MetricFlow/dbt-core, not set here +dbt-semantic-interfaces + +# CLI-related +Jinja2>=3.1.3 +halo>=0.0.31, <0.1.0 +update-checker>=0.18.0, <0.19.0 + +# Bug with mypy: https://github.com/pallets/click/issues/2558#issuecomment-1656546003 +click>=8.1.6 diff --git a/dbt-metricflow/extra-hatch-configuration/requirements-dbt-bigquery.txt b/dbt-metricflow/extra-hatch-configuration/requirements-dbt-bigquery.txt new file mode 100644 index 0000000000..3ef6a2994c --- /dev/null +++ b/dbt-metricflow/extra-hatch-configuration/requirements-dbt-bigquery.txt @@ -0,0 +1 @@ +dbt-bigquery>=1.8.0, <1.9.0 diff --git a/dbt-metricflow/extra-hatch-configuration/requirements-dbt-databricks.txt b/dbt-metricflow/extra-hatch-configuration/requirements-dbt-databricks.txt new file mode 100644 index 0000000000..ab8e6417ea --- /dev/null +++ b/dbt-metricflow/extra-hatch-configuration/requirements-dbt-databricks.txt @@ -0,0 +1 @@ +dbt-databricks>=1.8.0, <1.9.0 diff --git a/dbt-metricflow/extra-hatch-configuration/requirements-dbt-duckdb.txt b/dbt-metricflow/extra-hatch-configuration/requirements-dbt-duckdb.txt new file mode 100644 index 0000000000..a91d11d94d --- /dev/null +++ b/dbt-metricflow/extra-hatch-configuration/requirements-dbt-duckdb.txt @@ -0,0 +1 @@ +dbt-duckdb>=1.8.0, <1.9.0 diff --git a/dbt-metricflow/extra-hatch-configuration/requirements-dbt-postgres.txt b/dbt-metricflow/extra-hatch-configuration/requirements-dbt-postgres.txt new file mode 100644 index 0000000000..832aa9e91f --- /dev/null +++ b/dbt-metricflow/extra-hatch-configuration/requirements-dbt-postgres.txt @@ -0,0 +1 @@ +dbt-postgres>=1.8.0, <1.9.0 diff --git a/dbt-metricflow/extra-hatch-configuration/requirements-dbt-redshift.txt b/dbt-metricflow/extra-hatch-configuration/requirements-dbt-redshift.txt new file mode 100644 index 0000000000..7658ca24f1 --- /dev/null +++ b/dbt-metricflow/extra-hatch-configuration/requirements-dbt-redshift.txt @@ -0,0 +1 @@ +dbt-redshift>=1.8.0, <1.9.0 diff --git a/dbt-metricflow/extra-hatch-configuration/requirements-dbt-snowflake.txt b/dbt-metricflow/extra-hatch-configuration/requirements-dbt-snowflake.txt new file mode 100644 index 0000000000..937f1b52fc --- /dev/null +++ b/dbt-metricflow/extra-hatch-configuration/requirements-dbt-snowflake.txt @@ -0,0 +1 @@ +dbt-snowflake>=1.8.0, <1.9.0 diff --git a/dbt-metricflow/extra-hatch-configuration/requirements-dbt-trino.txt b/dbt-metricflow/extra-hatch-configuration/requirements-dbt-trino.txt new file mode 100644 index 0000000000..bc3390eacb --- /dev/null +++ b/dbt-metricflow/extra-hatch-configuration/requirements-dbt-trino.txt @@ -0,0 +1 @@ +dbt-trino>=1.8.0, <1.9.0 diff --git a/dbt-metricflow/extra-hatch-configuration/requirements-metricflow.txt b/dbt-metricflow/extra-hatch-configuration/requirements-metricflow.txt new file mode 100644 index 0000000000..ac4dc48b30 --- /dev/null +++ b/dbt-metricflow/extra-hatch-configuration/requirements-metricflow.txt @@ -0,0 +1 @@ +metricflow>=0.206.0.dev3, <0.207.0 diff --git a/dbt-metricflow/pyproject.toml b/dbt-metricflow/pyproject.toml index 497aad6542..f7fa9fa95e 100644 --- a/dbt-metricflow/pyproject.toml +++ b/dbt-metricflow/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["hatchling~=1.14.0"] +requires = ["hatchling~=1.14.0", "hatch-requirements-txt >= 0.4.1, <0.5.0"] build-backend = "hatchling.build" [project] @@ -7,7 +7,7 @@ name = "dbt-metricflow" version = "0.6.0" description = "Execute commands against the MetricFlow semantic layer with dbt." readme = "README.md" -requires-python = ">=3.8,<3.12" +requires-python = ">=3.8,<3.13" license = "BUSL-1.1" authors = [ { name = "dbt Labs", email = "info@dbtlabs.com" }, @@ -20,22 +20,18 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", ] -dependencies = [ - # cli dependencies - "Jinja2>=3.1.3", - "click>=7.1.2", - "halo>=0.0.31, <0.1.0", - "update-checker>=0.18.0, <0.19.0", - # Internal dependencies - "dbt-core>=1.7.4, <1.8.0", - "metricflow>=0.205.0, <0.206.0", +# Dependencies are specified through the `hatch-requirements-txt` plug-in. +dynamic = ["dependencies", "optional-dependencies"] - # dsi version should be fixed by MetricFlow/dbt-core, not set here - "dbt-semantic-interfaces", +[tool.hatch.metadata.hooks.requirements_txt] +files = [ + "extra-hatch-configuration/requirements-cli.txt", + "extra-hatch-configuration/requirements-metricflow.txt", ] [project.urls] @@ -44,27 +40,27 @@ dependencies = [ [project.scripts] mf = 'dbt_metricflow.cli.main:cli' -[project.optional-dependencies] -bigquery = [ - "dbt-bigquery>=1.7.0, <1.8.0" +[tool.hatch.metadata.hooks.requirements_txt.optional-dependencies] +dbt-bigquery = [ + "extra-hatch-configuration/requirements-dbt-bigquery.txt" ] -databricks = [ - "dbt-databricks>=1.7.0, <1.8.0" +dbt-databricks = [ + "extra-hatch-configuration/requirements-dbt-databricks.txt" ] -duckdb = [ - "dbt-duckdb>=1.7.0, <1.8.0" +dbt-duckdb = [ + "extra-hatch-configuration/requirements-dbt-duckdb.txt" ] -postgres = [ - "dbt-postgres>=1.7.0, <1.8.0" +dbt-postgres = [ + "extra-hatch-configuration/requirements-dbt-postgres.txt" ] -redshift = [ - "dbt-redshift>=1.7.0, <1.8.0" +dbt-redshift = [ + "extra-hatch-configuration/requirements-dbt-redshift.txt" ] -snowflake = [ - "dbt-snowflake>=1.7.0, <1.8.0" +dbt-snowflake = [ + "extra-hatch-configuration/requirements-dbt-snowflake.txt" ] -trino = [ - "dbt-trino>=1.7.0, <1.8.0" +dbt-trino = [ + "extra-hatch-configuration/requirements-dbt-trino.txt" ] [tool.hatch.build.targets.sdist] diff --git a/extra-hatch-configuration/requirements-dev-packages.txt b/extra-hatch-configuration/requirements-dev-packages.txt new file mode 100644 index 0000000000..b0ddd88f8d --- /dev/null +++ b/extra-hatch-configuration/requirements-dev-packages.txt @@ -0,0 +1,9 @@ +# Developer tools +mypy>=1.7.0, <1.8.0 +pre-commit>=3.2.2, <3.3.0 +pytest-mock>=3.7.0, <3.8.0 +pytest-xdist>=3.2.1, <3.3.0 +pytest>=7.1.1, < 7.2.0 +types-PyYAML +types-python-dateutil +types-tabulate diff --git a/extra-hatch-configuration/requirements-sql-client-packages.txt b/extra-hatch-configuration/requirements-sql-client-packages.txt new file mode 100644 index 0000000000..8fa34decb8 --- /dev/null +++ b/extra-hatch-configuration/requirements-sql-client-packages.txt @@ -0,0 +1,4 @@ +# These are currently separate for ease of removal, but due to the way Python +# handles import statements they are required in all test environments +SQLAlchemy>=1.4.42, <1.5.0 +sqlalchemy2-stubs>=0.0.2a21, <0.0.3 diff --git a/extra-hatch-configuration/requirements-trino-sql-client-packages.txt b/extra-hatch-configuration/requirements-trino-sql-client-packages.txt new file mode 100644 index 0000000000..ce89b89bcc --- /dev/null +++ b/extra-hatch-configuration/requirements-trino-sql-client-packages.txt @@ -0,0 +1 @@ +trino>=0.328.0, <0.329.0 diff --git a/pyproject.toml b/pyproject.toml index 5baa9d9ae2..1c19397bed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["hatchling~=1.14.0"] +requires = ["hatchling~=1.14.0", "hatch-requirements-txt >= 0.4.1, <0.5.0"] build-backend = "hatchling.build" [project] @@ -39,50 +39,44 @@ dependencies = [ "typing_extensions>=4.4, <5.0", ] +# Optional dependencies are specified through the `hatch-requirements-txt` plug-in. +dynamic = ["optional-dependencies"] + [project.urls] Documentation = "https://docs.getdbt.com/docs/build/about-metricflow" "Source Code" = "https://github.com/dbt-labs/metricflow" -[project.optional-dependencies] +[tool.hatch.metadata.hooks.requirements_txt.optional-dependencies] +dbt-bigquery = [ + "dbt-metricflow/extra-hatch-configuration/requirements-dbt-bigquery.txt" +] +dbt-databricks = [ + "dbt-metricflow/extra-hatch-configuration/requirements-dbt-databricks.txt" +] +dbt-duckdb = [ + "dbt-metricflow/extra-hatch-configuration/requirements-dbt-duckdb.txt" +] +dbt-postgres = [ + "dbt-metricflow/extra-hatch-configuration/requirements-dbt-postgres.txt" +] +dbt-redshift = [ + "dbt-metricflow/extra-hatch-configuration/requirements-dbt-redshift.txt" +] +dbt-snowflake = [ + "dbt-metricflow/extra-hatch-configuration/requirements-dbt-snowflake.txt" +] +dbt-trino = [ + "dbt-metricflow/extra-hatch-configuration/requirements-dbt-trino.txt" +] dev-packages = [ - # Developer tools - "mypy>=1.7.0, <1.8.0", - "pre-commit>=3.2.2, <3.3.0", - "pytest-mock>=3.7.0, <3.8.0", - "pytest-xdist>=3.2.1, <3.3.0", - "pytest>=7.1.1, < 7.2.0", - "types-PyYAML", - "types-python-dateutil", - "types-tabulate", - - # Test and CLI development dependencies. - # These should generally match what's in dbt-metricflow/pyproject.toml, but may - # diverge during upgrade phases or for other reasons - # - # Note we do not currently depend on dbt-core because updates across minor version - # boundaries cause problems with tests against future versions of dbt-semantic-interfaces. - # In future we will refine this packaging to depend on stable dbt-adapter interfaces in - # the development builds, but those packages do not yet support all of our needs so we - # hack around the problem with editable pre-installs in the various dev environments. - # Note this only works because we don't use the dbt packages for anything other than - # managing warehouse connections and dispatching queries, so this is not a pattern - # anyone else should be emulating even as a short-term hack. - - "halo>=0.0.31, <0.1.0", - "update-checker>=0.18.0, <0.19.0", - - # Bug with mypy: https://github.com/pallets/click/issues/2558#issuecomment-1656546003 - "click>=8.1.6", -] - -# These are currently separate for ease of removal, but due to the way Python -# handles import statements they are required in all test environments + "extra-hatch-configuration/requirements-dev-packages.txt", + "dbt-metricflow/extra-hatch-configuration/requirements-cli.txt" +] sql-client-packages = [ - "SQLAlchemy>=1.4.42, <1.5.0", - "sqlalchemy2-stubs>=0.0.2a21, <0.0.3", + "extra-hatch-configuration/requirements-sql-client-packages.txt" ] trino-sql-client-packages = [ - "trino>=0.327.0, <0.328.0", + "extra-hatch-configuration/requirements-trino-sql-client-packages.txt" ] # There are files that hatch will always include as well (e.g. the LICENSE file). @@ -102,30 +96,26 @@ packages = ["metricflow", "metricflow-semantics/metricflow_semantics"] # Environment setup -# Due to our current wonky package organization, we need to do an editable install -# of dbt-metricflow in every test running environment in order for the tests to work. -# In theory we can install the adapter extra for dbt-metricflow and rely on that, but -# for ease of managing versioning through upgrade phases we retain the explicit -# dependencies on the specific adapter versions defined here. - [tool.hatch.envs.dev-env] description = "Environment for development. Includes a DuckDB-backed client." - -pre-install-commands = [ - "pip install -e ./metricflow-semantics", - "pip install -e ./dbt-metricflow[duckdb]", -] +run = "run-coverage --no-cov" features = [ "dev-packages", "sql-client-packages", + "dbt-duckdb", ] [tool.hatch.envs.dev-env.env-vars] MF_TEST_ADAPTER_TYPE="duckdb" MF_SQL_ENGINE_URL="duckdb://" +# This allows us to use the classes in the `dbt-metricflow` package for tests without installing the package. +# `dbt-metricflow` can't be installed as it has `metricflow` as a dependency. +PYTHONPATH="metricflow-semantics:dbt-metricflow" + [tool.hatch.envs.postgres-env.env-vars] +PYTHONPATH="metricflow-semantics:dbt-metricflow" MF_SQL_ENGINE_URL="postgresql://metricflow@localhost:5432/metricflow" MF_SQL_ENGINE_PASSWORD="metricflowing" MF_TEST_ADAPTER_TYPE="postgres" @@ -133,84 +123,75 @@ MF_TEST_ADAPTER_TYPE="postgres" [tool.hatch.envs.postgres-env] description = "Dev environment for working with Postgres adapter" -pre-install-commands = [ - "pip install -e ./metricflow-semantics", - "pip install -e ./dbt-metricflow[postgres]" -] - features = [ "dev-packages", "sql-client-packages", + "dbt-postgres", ] + # NOTE: All of the below should have their authentication credentials # configured independently of the hatch env construction [tool.hatch.envs.bigquery-env.env-vars] +PYTHONPATH="metricflow-semantics:dbt-metricflow" MF_TEST_ADAPTER_TYPE="bigquery" [tool.hatch.envs.bigquery-env] description = "Dev environment for working with the BigQuery adapter" -pre-install-commands = [ - "pip install -e ./metricflow-semantics", - "pip install -e ./dbt-metricflow[bigquery]" -] - features = [ "dev-packages", "sql-client-packages", + "dbt-bigquery", ] + [tool.hatch.envs.databricks-env.env-vars] +PYTHONPATH="metricflow-semantics:dbt-metricflow" MF_TEST_ADAPTER_TYPE="databricks" + [tool.hatch.envs.databricks-env] description = "Dev environment for working with the Databricks adapter" -pre-install-commands = [ - "pip install -e ./metricflow-semantics", - "pip install -e ./dbt-metricflow[databricks]" -] - features = [ "dev-packages", "sql-client-packages", + "dbt-databricks", ] + [tool.hatch.envs.redshift-env.env-vars] +PYTHONPATH="metricflow-semantics:dbt-metricflow" MF_TEST_ADAPTER_TYPE="redshift" [tool.hatch.envs.redshift-env] description = "Dev environment for working with the Redshift adapter" -pre-install-commands = [ - "pip install -e ./metricflow-semantics", - "pip install -e ./dbt-metricflow[redshift]" -] - features = [ "dev-packages", "sql-client-packages", + "dbt-redshift" ] + [tool.hatch.envs.snowflake-env.env-vars] +PYTHONPATH="metricflow-semantics:dbt-metricflow" MF_TEST_ADAPTER_TYPE="snowflake" [tool.hatch.envs.snowflake-env] description = "Dev environment for working with Snowflake adapter" -pre-install-commands = [ - "pip install -e ./metricflow-semantics", - "pip install -e ./dbt-metricflow[snowflake]" -] - features = [ "dev-packages", "sql-client-packages", + "dbt-snowflake", ] + [tool.hatch.envs.trino-env.env-vars] +PYTHONPATH="metricflow-semantics:dbt-metricflow" MF_TEST_ADAPTER_TYPE = "trino" MF_SQL_ENGINE_URL = "trino://trino@localhost:8080/" DBT_ENV_SECRET_CATALOG="memory" @@ -218,24 +199,21 @@ DBT_ENV_SECRET_CATALOG="memory" [tool.hatch.envs.trino-env] description = "Dev environment for working with the Trino adapter" -pre-install-commands = [ - "pip install -e ./metricflow-semantics", - "pip install -e ./dbt-metricflow[trino]" -] - features = [ "dev-packages", "sql-client-packages", "trino-sql-client-packages", + "dbt-trino" ] + [tool.black] line-length = 120 +[tool.pytest.ini_options] # Many deprecation warnings come from 3rd-party libraries and make the # output of pytest noisy. Since no action is going to be taken, hide those # warnings. -[tool.pytest.ini_options] filterwarnings = [ "ignore:Deprecated call to.*", "ignore:pkg_resources is deprecated as an API"