diff --git a/.changes/unreleased/Breaking Changes-20240307-095610.yaml b/.changes/unreleased/Breaking Changes-20240307-095610.yaml new file mode 100644 index 0000000000..c5925aa377 --- /dev/null +++ b/.changes/unreleased/Breaking Changes-20240307-095610.yaml @@ -0,0 +1,6 @@ +kind: Breaking Changes +body: dd Python 3.12 Support / Remove Python 3.8 Support +time: 2024-03-07T09:56:10.062042-08:00 +custom: + Author: plypaul + Issue: "1065" diff --git a/.github/workflows/cd-push-to-pypi.yaml b/.github/workflows/cd-push-to-pypi.yaml index 7468e1efe1..22a5f00c24 100644 --- a/.github/workflows/cd-push-to-pypi.yaml +++ b/.github/workflows/cd-push-to-pypi.yaml @@ -6,7 +6,7 @@ on: - "*" env: - PYTHON_VERSION: "3.8" + PYTHON_VERSION: "3.9" jobs: pypi-publish: diff --git a/.github/workflows/cd-sql-engine-populate-persistent-source-schema.yaml b/.github/workflows/cd-sql-engine-populate-persistent-source-schema.yaml index cfae1da5ef..f5381189b3 100644 --- a/.github/workflows/cd-sql-engine-populate-persistent-source-schema.yaml +++ b/.github/workflows/cd-sql-engine-populate-persistent-source-schema.yaml @@ -15,7 +15,7 @@ on: env: # Unclear on how to make 'Reload Test Data in SQL Engines' a constant here as it does not work here. - PYTHON_VERSION: "3.8" + PYTHON_VERSION: "3.9" ADDITIONAL_PYTEST_OPTIONS: "--log-cli-level info" jobs: diff --git a/.github/workflows/cd-sql-engine-tests.yaml b/.github/workflows/cd-sql-engine-tests.yaml index bb0782bc21..002ec9d6fa 100644 --- a/.github/workflows/cd-sql-engine-tests.yaml +++ b/.github/workflows/cd-sql-engine-tests.yaml @@ -8,7 +8,7 @@ on: types: [labeled] env: - PYTHON_VERSION: "3.8" + PYTHON_VERSION: "3.9" EXTERNAL_ENGINE_TEST_PARALLELISM: 8 ADDITIONAL_PYTEST_OPTIONS: "--use-persistent-source-schema" diff --git a/.github/workflows/ci-linting.yaml b/.github/workflows/ci-linting.yaml index bbbf20eb2a..097346f214 100644 --- a/.github/workflows/ci-linting.yaml +++ b/.github/workflows/ci-linting.yaml @@ -18,7 +18,7 @@ jobs: name: Run Pre-Commit Linting Hooks runs-on: ubuntu-latest env: - python-version: "3.8" + python-version: "3.9" steps: - name: Check-out the repo diff --git a/.github/workflows/ci-metricflow-unit-tests.yaml b/.github/workflows/ci-metricflow-unit-tests.yaml index 34291067f3..157e81f7d3 100644 --- a/.github/workflows/ci-metricflow-unit-tests.yaml +++ b/.github/workflows/ci-metricflow-unit-tests.yaml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.11"] + python-version: ["3.9", "3.12"] steps: - name: Check-out the repo @@ -47,10 +47,10 @@ jobs: - name: Check-out the repo uses: actions/checkout@v3 - - name: Test w/ Python 3.11 + - name: Test w/ Python 3.12 uses: ./.github/actions/run-mf-tests with: - python-version: "3.11" + python-version: "3.12" make-target: "test-postgresql" metricflow-unit-tests: diff --git a/metricflow/test/compare_df.py b/metricflow/test/compare_df.py index ce34fbd419..58f99b6026 100644 --- a/metricflow/test/compare_df.py +++ b/metricflow/test/compare_df.py @@ -1,5 +1,6 @@ from __future__ import annotations +import datetime import logging import math from typing import SupportsFloat @@ -22,22 +23,38 @@ def _dataframes_contain_same_data( for c in range(expected.shape[0]): for r in range(expected.shape[1]): + expected_value = expected.iloc[c, r] + actual_value = actual.iloc[c, r] + # NaNs can't be compared for equality. - if pd.isna(expected.iloc[c, r]) and pd.isna(actual.iloc[c, r]): - pass - elif isinstance(expected.iloc[c, r], SupportsFloat) and isinstance(actual.iloc[c, r], SupportsFloat): - if not math.isclose(expected.iloc[c, r], actual.iloc[c, r], rel_tol=1e-6): - return False - elif ( - isinstance(expected.iloc[c, r], pd.Timestamp) - and isinstance(actual.iloc[c, r], pd.Timestamp) + if pd.isna(expected_value) and pd.isna(actual_value): + continue + + if ( + isinstance(expected_value, SupportsFloat) + and isinstance(actual_value, SupportsFloat) + and math.isclose(expected_value, actual_value, rel_tol=1e-6) + ): + continue + + # Convert dates to timestamps as Pandas will give a warning when comparing dates and timestamps. + if isinstance(expected_value, datetime.date): + expected_value = pd.Timestamp(expected_value) + + if isinstance(actual_value, datetime.date): + actual_value = pd.Timestamp(actual_value) + + if ( + isinstance(expected_value, pd.Timestamp) + and isinstance(actual_value, pd.Timestamp) # If expected has no tz but actual is UTC, remove timezone. Some engines add UTC by default. - and actual.iloc[c, r].tzname() == "UTC" - and expected.iloc[c, r].tzname() is None + and actual_value.tzname() == "UTC" + and expected_value.tzname() is None ): - if actual.iloc[c, r].tz_localize(None) != expected.iloc[c, r].tz_localize(None): - return False - elif expected.iloc[c, r] != actual.iloc[c, r]: + actual_value = actual_value.tz_localize(None) + expected_value = expected_value.tz_localize(None) + + if expected_value != actual_value: return False return True diff --git a/pyproject.toml b/pyproject.toml index 3e7b57718b..f964fef641 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "metricflow" version = "0.205.0" description = "Translates a simple metric definition into reusable SQL and executes it against the SQL engine of your choice." readme = "README.md" -requires-python = ">=3.8,<3.12" +requires-python = ">=3.9,<=3.12" license = "BUSL-1.1" keywords = [] authors = [ @@ -17,10 +17,10 @@ authors = [ classifiers = [ "Development Status :: 4 - Beta", "Programming Language :: Python", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", ] @@ -33,9 +33,10 @@ dependencies = [ "graphviz>=0.18.2, <0.21", "halo>=0.0.31, <0.1.0", "more-itertools>=8.10.0, <10.2.0", - "pandas>=1.5.0, <1.6.0", + # pandas 2.2 has a bug https://github.com/databricks/databricks-sql-python/issues/334, so using 2.1. + "pandas>=2.1, <2.2", "pydantic>=1.10.0, <1.11.0", - "python-dateutil>=2.8.2, <2.9.0", + "python-dateutil~=2.9.0", "rapidfuzz>=3.0, <4.0", "ruamel.yaml>=0.17.21, <0.18.0", "tabulate>=0.8.9", @@ -56,11 +57,11 @@ dev-packages = [ "pre-commit>=3.2.2, <3.3.0", # Bug with mypy: https://github.com/pallets/click/issues/2558#issuecomment-1656546003 "click>=8.1.6", - "pytest-mock>=3.7.0, <3.8.0", - "pytest-xdist>=3.2.1, <3.3.0", - "pytest>=7.1.1, < 7.2.0", - "types-PyYAML", + "pytest-mock~=3.12", + "pytest-xdist~=3.5", + "pytest~=8.0", "types-python-dateutil", + "types-PyYAML", "types-tabulate", ] diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000000..20f66b876b --- /dev/null +++ b/pytest.ini @@ -0,0 +1,17 @@ +[pytest] +filterwarnings = +; The filter below ignores deprecation warnings from dbt packages. This should be removed once those are updated. + ignore:datetime\.datetime\.utcnow\(\) is deprecated:DeprecationWarning:dbt.*: +; +; The filter below ignores deprecation warnings from the 'halo' package. +; The latest release of 'halo' is 0.0.31 and it has not been updated since 2020-11-09. +; We may want to migrate away from this package if that package is not kept up to date. +; + ignore:setDaemon\(\) is deprecated, set the daemon attribute instead:DeprecationWarning:halo.*: +; +; The filter below ignores deprecation warnings from the protobuff package. Based on +; https://github.com/protocolbuffers/protobuf/issues/15077, it seems like it'll be resolved in a release after +; 4.25.3, so this filter can be removed then. +; + ignore:Type google\._upb\._message\.ScalarMapContainer uses PyType_Spec with a metaclass that has custom tp_new:DeprecationWarning + ignore:Type google\._upb\._message\.MessageMapContainer uses PyType_Spec with a metaclass that has custom tp_new:DeprecationWarning