From 4f6464a6d168c519239be48e3313002d6eefe918 Mon Sep 17 00:00:00 2001 From: Quigley Malcolm Date: Thu, 21 Dec 2023 12:30:39 -0800 Subject: [PATCH 1/5] Add a pydantic shim for handling pydantic 1 vs pydantic 2 We are going to need to support pydantic 1 for some time. We think currently that we'll have to support it for atleast another year. However, we also need to begin supporting pydantic 2. Thus we need this shim to have the flexibility of supporting both. I have also run `hatch build` and ensured that this shim _does_ get included in the resulting tarball. In the upcoming commits, we'll switch over direct pydantic imports to instead use this shim, and then loosen the version in the pyproject config. --- pydantic_shim.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 pydantic_shim.py diff --git a/pydantic_shim.py b/pydantic_shim.py new file mode 100644 index 00000000..3ea00720 --- /dev/null +++ b/pydantic_shim.py @@ -0,0 +1,26 @@ +from importlib.metadata import version + +pydantic_version = version("pydantic") +# Pydantic uses semantic versioning, i.e. .., and we need to know the major +pydantic_major = pydantic_version.split(".")[0] + +if pydantic_major == "1": + from pydantic import ( # type: ignore # noqa + BaseModel, + Extra, + Field, + create_model, + root_validator, + validator, + ) +elif pydantic_major == "2": + from pydantic.v1 import ( # type: ignore # noqa + BaseModel, + Extra, + Field, + create_model, + root_validator, + validator, + ) +else: + raise RuntimeError(f"Currently only pydantic 1 and 2 are supported, found pydantic {pydantic_version}") From 39fb6ea49cd7508e07c05341ddb61ee2531e6911 Mon Sep 17 00:00:00 2001 From: Quigley Malcolm Date: Thu, 21 Dec 2023 12:47:53 -0800 Subject: [PATCH 2/5] Migrate to importing from pydantic_shim instead of pydantic directly --- dbt_semantic_interfaces/dataclass_serialization.py | 5 ++--- dbt_semantic_interfaces/implementations/base.py | 3 +-- dbt_semantic_interfaces/implementations/export.py | 2 +- dbt_semantic_interfaces/implementations/metric.py | 3 +-- .../implementations/project_configuration.py | 2 +- dbt_semantic_interfaces/implementations/semantic_model.py | 2 +- dbt_semantic_interfaces/validations/validator_helpers.py | 2 +- 7 files changed, 8 insertions(+), 11 deletions(-) diff --git a/dbt_semantic_interfaces/dataclass_serialization.py b/dbt_semantic_interfaces/dataclass_serialization.py index 44ada7b1..8207f4a3 100644 --- a/dbt_semantic_interfaces/dataclass_serialization.py +++ b/dbt_semantic_interfaces/dataclass_serialization.py @@ -19,11 +19,10 @@ get_type_hints, ) -import pydantic -from pydantic import BaseModel from typing_extensions import TypeAlias from dbt_semantic_interfaces.pretty_print import pformat_big_objects +from pydantic_shim import BaseModel, create_model logger = logging.getLogger(__name__) @@ -347,7 +346,7 @@ def _convert_dataclass_type_to_pydantic_type( logger.debug( f"Creating Pydantic model {class_name} with fields:\n{pformat_big_objects(fields_for_pydantic_model)}" ) - pydantic_model = pydantic.create_model(class_name, **fields_for_pydantic_model) # type: ignore + pydantic_model = create_model(class_name, **fields_for_pydantic_model) # type: ignore logger.debug(f"Finished creating Pydantic model {class_name}") logger.debug(f"Finished converting {dataclass_type.__name__} to a pydantic class") return pydantic_model diff --git a/dbt_semantic_interfaces/implementations/base.py b/dbt_semantic_interfaces/implementations/base.py index 31c9faeb..25195649 100644 --- a/dbt_semantic_interfaces/implementations/base.py +++ b/dbt_semantic_interfaces/implementations/base.py @@ -5,13 +5,12 @@ from abc import ABC, abstractmethod from typing import Any, Callable, ClassVar, Generator, Generic, Type, TypeVar -from pydantic import BaseModel, root_validator - from dbt_semantic_interfaces.errors import ParsingException from dbt_semantic_interfaces.parsing.yaml_loader import ( PARSING_CONTEXT_KEY, ParsingContext, ) +from pydantic_shim import BaseModel, root_validator # Type alias for the implicit "Any" type used as input and output for Pydantic's parsing API PydanticParseableValueType = Any # type: ignore[misc] diff --git a/dbt_semantic_interfaces/implementations/export.py b/dbt_semantic_interfaces/implementations/export.py index 7a06d1b0..c586c6e0 100644 --- a/dbt_semantic_interfaces/implementations/export.py +++ b/dbt_semantic_interfaces/implementations/export.py @@ -2,7 +2,6 @@ from typing import Optional -from pydantic import Field from typing_extensions import override from dbt_semantic_interfaces.implementations.base import HashableBaseModel @@ -11,6 +10,7 @@ from dbt_semantic_interfaces.type_enums.export_destination_type import ( ExportDestinationType, ) +from pydantic_shim import Field class PydanticExportConfig(HashableBaseModel, ProtocolHint[ExportConfig]): diff --git a/dbt_semantic_interfaces/implementations/metric.py b/dbt_semantic_interfaces/implementations/metric.py index 366648a4..1bedf40a 100644 --- a/dbt_semantic_interfaces/implementations/metric.py +++ b/dbt_semantic_interfaces/implementations/metric.py @@ -2,8 +2,6 @@ from typing import List, Optional, Sequence -from pydantic import Field - from dbt_semantic_interfaces.enum_extension import assert_values_exhausted from dbt_semantic_interfaces.errors import ParsingException from dbt_semantic_interfaces.implementations.base import ( @@ -22,6 +20,7 @@ MetricType, TimeGranularity, ) +from pydantic_shim import Field class PydanticMetricInputMeasure(PydanticCustomInputParser, HashableBaseModel): diff --git a/dbt_semantic_interfaces/implementations/project_configuration.py b/dbt_semantic_interfaces/implementations/project_configuration.py index 3e52fd91..93693502 100644 --- a/dbt_semantic_interfaces/implementations/project_configuration.py +++ b/dbt_semantic_interfaces/implementations/project_configuration.py @@ -3,7 +3,6 @@ from typing import List, Optional from importlib_metadata import version -from pydantic import validator from typing_extensions import override from dbt_semantic_interfaces.implementations.base import ( @@ -20,6 +19,7 @@ ) from dbt_semantic_interfaces.protocols import ProtocolHint from dbt_semantic_interfaces.protocols.project_configuration import ProjectConfiguration +from pydantic_shim import validator class PydanticProjectConfiguration(HashableBaseModel, ModelWithMetadataParsing, ProtocolHint[ProjectConfiguration]): diff --git a/dbt_semantic_interfaces/implementations/semantic_model.py b/dbt_semantic_interfaces/implementations/semantic_model.py index 45b90ae0..933acb55 100644 --- a/dbt_semantic_interfaces/implementations/semantic_model.py +++ b/dbt_semantic_interfaces/implementations/semantic_model.py @@ -2,7 +2,6 @@ from typing import Any, List, Optional, Sequence -from pydantic import validator from typing_extensions import override from dbt_semantic_interfaces.implementations.base import ( @@ -25,6 +24,7 @@ SemanticModelReference, TimeDimensionReference, ) +from pydantic_shim import validator class NodeRelation(HashableBaseModel): diff --git a/dbt_semantic_interfaces/validations/validator_helpers.py b/dbt_semantic_interfaces/validations/validator_helpers.py index 6ea60f48..1f816cc3 100644 --- a/dbt_semantic_interfaces/validations/validator_helpers.py +++ b/dbt_semantic_interfaces/validations/validator_helpers.py @@ -20,7 +20,6 @@ ) import click -from pydantic import BaseModel, Extra from dbt_semantic_interfaces.implementations.base import FrozenBaseModel from dbt_semantic_interfaces.protocols import Metadata, SemanticManifestT, SemanticModel @@ -30,6 +29,7 @@ SemanticModelReference, ) from dbt_semantic_interfaces.type_enums import DimensionType +from pydantic_shim import BaseModel, Extra VALIDATE_SAFELY_ERROR_STR_TMPLT = ". Issue occurred in method `{method_name}` called with {arguments_str}" ValidationContextJSON = Dict[str, Union[str, int, None]] From 0924b6a456e78e5dd668ec0e4416e3d54a205b2b Mon Sep 17 00:00:00 2001 From: Quigley Malcolm Date: Thu, 21 Dec 2023 12:57:28 -0800 Subject: [PATCH 3/5] Expand pydantic depedency specification to include pydantic 2 --- .changes/unreleased/Dependencies-20231221-130418.yaml | 6 ++++++ pyproject.toml | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Dependencies-20231221-130418.yaml diff --git a/.changes/unreleased/Dependencies-20231221-130418.yaml b/.changes/unreleased/Dependencies-20231221-130418.yaml new file mode 100644 index 00000000..e478c08d --- /dev/null +++ b/.changes/unreleased/Dependencies-20231221-130418.yaml @@ -0,0 +1,6 @@ +kind: Dependencies +body: Add support for pydantic 2 (and continue support for pydantic 1) +time: 2023-12-21T13:04:18.792592-08:00 +custom: + Author: QMalcolm bernardcooke53 esciara + PR: "134" diff --git a/pyproject.toml b/pyproject.toml index a892f39d..aa6611c6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ classifiers = [ "Programming Language :: Python :: Implementation :: PyPy", ] dependencies = [ - "pydantic~=1.10", + "pydantic>=1.10,<3", "jsonschema~=4.0", "PyYAML~=6.0", "more-itertools>=8.0,<11.0", From 458f6d3a030fa9b79e753314d027682ebb0ace27 Mon Sep 17 00:00:00 2001 From: Quigley Malcolm Date: Thu, 21 Dec 2023 15:29:09 -0800 Subject: [PATCH 4/5] Test both pydantic 1 and 2 in github actions --- .github/workflows/ci-pytest.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-pytest.yaml b/.github/workflows/ci-pytest.yaml index 20c9bcaa..77c41fc7 100644 --- a/.github/workflows/ci-pytest.yaml +++ b/.github/workflows/ci-pytest.yaml @@ -11,11 +11,12 @@ on: jobs: pytest: - name: Run Tests / Python ${{ matrix.python-version }} + name: Run Tests / Python ${{ matrix.python-version }} / Pydantic ~= ${{ matrix.pydantic-version }} strategy: matrix: python-version: ["3.8", "3.9", "3.10", "3.11"] + pydantic-version: ["1.10", "2.0"] runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -26,5 +27,7 @@ jobs: - name: Install Hatch shell: bash run: pip3 install hatch + - name: Set pydantic Version ~= ${{ matrix.pydantic-version }} + run: hatch run dev-env:pip install "pydantic~=${{ matrix.pydantic-version }}" - name: Run Python Tests run: hatch run dev-env:pytest tests From 1e6245aa1c01f9d95ddc7f15b5f17a1a2a867f1a Mon Sep 17 00:00:00 2001 From: Quigley Malcolm Date: Thu, 21 Dec 2023 15:59:09 -0800 Subject: [PATCH 5/5] Use Pydantic 1 for ci-pre-commit workflow This is necessary because the typing for v1 stuff breaks when using Pydantic 2. We can stop doing this once we actually migrate off of Pydantic 1 implementations. --- .github/workflows/ci-pre-commit.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci-pre-commit.yaml b/.github/workflows/ci-pre-commit.yaml index e59004fe..bfd95a99 100644 --- a/.github/workflows/ci-pre-commit.yaml +++ b/.github/workflows/ci-pre-commit.yaml @@ -16,5 +16,8 @@ jobs: steps: - uses: actions/checkout@v3 - uses: ./.github/actions/setup-python-env + # This step is necessary so long as we're allowing Pydantic 1 and Pydantic 2 via shimming + - name: Force Pydantic 1 + run: hatch run dev-env:pip install "pydantic~=1.10" - name: Run Pre-commit Hooks run: hatch run dev-env:pre-commit run --show-diff-on-failure --color=always --all-files