Add Python 3.12 Support / Remove Python 3.8 Support

.changes/unreleased/Breaking Changes-20240307-095610.yaml

@@ -0,0 +1,6 @@
+kind: Breaking Changes
+body: dd Python 3.12 Support / Remove Python 3.8 Support
+time: 2024-03-07T09:56:10.062042-08:00
+custom:
+  Author: plypaul
+  Issue: "1065"

.github/workflows/cd-push-to-pypi.yaml

@@ -6,7 +6,7 @@ on:
       - "*"
 
 env:
-  PYTHON_VERSION: "3.8"
+  PYTHON_VERSION: "3.9"
 
 jobs:
   pypi-publish:

.github/workflows/cd-sql-engine-populate-persistent-source-schema.yaml

@@ -15,7 +15,7 @@ on:
 
 env:
   # Unclear on how to make 'Reload Test Data in SQL Engines' a constant here as it does not work here.
-  PYTHON_VERSION: "3.8"
+  PYTHON_VERSION: "3.9"
   ADDITIONAL_PYTEST_OPTIONS: "--log-cli-level info"
 
 jobs:

.github/workflows/cd-sql-engine-tests.yaml

@@ -8,7 +8,7 @@ on:
     types: [labeled]
 
 env:
-  PYTHON_VERSION: "3.8"
+  PYTHON_VERSION: "3.9"
   EXTERNAL_ENGINE_TEST_PARALLELISM: 8
   ADDITIONAL_PYTEST_OPTIONS: "--use-persistent-source-schema"
 

.github/workflows/ci-linting.yaml

@@ -18,7 +18,7 @@ jobs:
     name: Run Pre-Commit Linting Hooks
     runs-on: ubuntu-latest
     env:
-      python-version: "3.8"
+      python-version: "3.9"
     steps:
 
       - name: Check-out the repo

.github/workflows/ci-metricflow-unit-tests.yaml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.8", "3.11"]
+        python-version: ["3.9", "3.12"]
     steps:
 
       - name: Check-out the repo
@@ -47,10 +47,10 @@ jobs:
       - name: Check-out the repo
         uses: actions/checkout@v3
 
-      - name: Test w/ Python 3.11
+      - name: Test w/ Python 3.12
         uses: ./.github/actions/run-mf-tests
         with:
-          python-version: "3.11"
+          python-version: "3.12"
           make-target: "test-postgresql"
 
   metricflow-unit-tests:

metricflow/test/compare_df.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import datetime
 import logging
 import math
 from typing import SupportsFloat
@@ -22,22 +23,38 @@ def _dataframes_contain_same_data(
 
     for c in range(expected.shape[0]):
         for r in range(expected.shape[1]):
+            expected_value = expected.iloc[c, r]
+            actual_value = actual.iloc[c, r]
+
             # NaNs can't be compared for equality.
-            if pd.isna(expected.iloc[c, r]) and pd.isna(actual.iloc[c, r]):
-                pass
-            elif isinstance(expected.iloc[c, r], SupportsFloat) and isinstance(actual.iloc[c, r], SupportsFloat):
-                if not math.isclose(expected.iloc[c, r], actual.iloc[c, r], rel_tol=1e-6):
-                    return False
-            elif (
-                isinstance(expected.iloc[c, r], pd.Timestamp)
-                and isinstance(actual.iloc[c, r], pd.Timestamp)
+            if pd.isna(expected_value) and pd.isna(actual_value):
+                continue
+
+            if (
+                isinstance(expected_value, SupportsFloat)
+                and isinstance(actual_value, SupportsFloat)
+                and math.isclose(expected_value, actual_value, rel_tol=1e-6)
+            ):
+                continue
+
+            # Convert dates to timestamps as Pandas will give a warning when comparing dates and timestamps.
+            if isinstance(expected_value, datetime.date):
+                expected_value = pd.Timestamp(expected_value)
+
+            if isinstance(actual_value, datetime.date):
+                actual_value = pd.Timestamp(actual_value)
+
+            if (
+                isinstance(expected_value, pd.Timestamp)
+                and isinstance(actual_value, pd.Timestamp)
                 # If expected has no tz but actual is UTC, remove timezone. Some engines add UTC by default.
-                and actual.iloc[c, r].tzname() == "UTC"
-                and expected.iloc[c, r].tzname() is None
+                and actual_value.tzname() == "UTC"
+                and expected_value.tzname() is None
             ):
-                if actual.iloc[c, r].tz_localize(None) != expected.iloc[c, r].tz_localize(None):
-                    return False
-            elif expected.iloc[c, r] != actual.iloc[c, r]:
+                actual_value = actual_value.tz_localize(None)
+                expected_value = expected_value.tz_localize(None)
+
+            if expected_value != actual_value:
                 return False
     return True
 

pyproject.toml

@@ -7,7 +7,7 @@ name = "metricflow"
 version = "0.205.0"
 description = "Translates a simple metric definition into reusable SQL and executes it against the SQL engine of your choice."
 readme = "README.md"
-requires-python = ">=3.8,<3.12"
+requires-python = ">=3.9,<=3.12"
 license = "BUSL-1.1"
 keywords = []
 authors = [
@@ -17,10 +17,10 @@ authors = [
 classifiers = [
   "Development Status :: 4 - Beta",
   "Programming Language :: Python",
-  "Programming Language :: Python :: 3.8",
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
   "Programming Language :: Python :: Implementation :: CPython",
   "Programming Language :: Python :: Implementation :: PyPy",
 ]
@@ -33,9 +33,10 @@ dependencies = [
   "graphviz>=0.18.2, <0.21",
   "halo>=0.0.31, <0.1.0",
   "more-itertools>=8.10.0, <10.2.0",
-  "pandas>=1.5.0, <1.6.0",
+  # pandas 2.2 has a bug https://github.com/databricks/databricks-sql-python/issues/334, so using 2.1.
+  "pandas>=2.1, <2.2",
   "pydantic>=1.10.0, <1.11.0",
-  "python-dateutil>=2.8.2, <2.9.0",
+  "python-dateutil~=2.9.0",
   "rapidfuzz>=3.0, <4.0",
   "ruamel.yaml>=0.17.21, <0.18.0",
   "tabulate>=0.8.9",
@@ -56,11 +57,11 @@ dev-packages = [
   "pre-commit>=3.2.2, <3.3.0",
   # Bug with mypy: https://github.com/pallets/click/issues/2558#issuecomment-1656546003
   "click>=8.1.6",
-  "pytest-mock>=3.7.0, <3.8.0",
-  "pytest-xdist>=3.2.1, <3.3.0",
-  "pytest>=7.1.1, < 7.2.0",
-  "types-PyYAML",
+  "pytest-mock~=3.12",
+  "pytest-xdist~=3.5",
+  "pytest~=8.0",
   "types-python-dateutil",
+  "types-PyYAML",
   "types-tabulate",
 ]
 

pytest.ini

@@ -0,0 +1,17 @@
+[pytest]
+filterwarnings =
+;   The filter below ignores deprecation warnings from dbt packages. This should be removed once those are updated.
+    ignore:datetime\.datetime\.utcnow\(\) is deprecated:DeprecationWarning:dbt.*:
+;
+;   The filter  below ignores deprecation warnings from the 'halo' package.
+;   The latest release of 'halo' is 0.0.31 and it has not been updated since 2020-11-09.
+;   We may want to migrate away from this package if that package is not kept up to date.
+;
+    ignore:setDaemon\(\) is deprecated, set the daemon attribute instead:DeprecationWarning:halo.*:
+;
+;   The filter below ignores deprecation warnings from the protobuff package. Based on
+;   https://github.com/protocolbuffers/protobuf/issues/15077, it seems like it'll be resolved in a release after
+;   4.25.3, so this filter can be removed then.
+;
+    ignore:Type google\._upb\._message\.ScalarMapContainer uses PyType_Spec with a metaclass that has custom tp_new:DeprecationWarning
+    ignore:Type google\._upb\._message\.MessageMapContainer uses PyType_Spec with a metaclass that has custom tp_new:DeprecationWarning