From 8fd8bd1de84b3b7b475dac39fa858f37c2f5a440 Mon Sep 17 00:00:00 2001 From: Jim Bosch Date: Thu, 21 Dec 2023 14:44:42 -0500 Subject: [PATCH 1/5] Drop support for Pydantic v1. --- pyproject.toml | 2 +- python/lsst/pipe/base/_task_metadata.py | 5 ++--- python/lsst/pipe/base/graph/quantumNode.py | 4 ++-- python/lsst/pipe/base/pipeline_graph/io.py | 13 ++++++------- python/lsst/pipe/base/tests/mocks/_storage_class.py | 12 ++++++------ requirements.txt | 2 +- 6 files changed, 18 insertions(+), 20 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 09c22ae63..f7ca41193 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ dependencies = [ "lsst-daf-butler", "lsst-pex-config", "astropy", - "pydantic <3.0", + "pydantic >=2,<3.0", "networkx", "pyyaml >= 5.1", "numpy >= 1.17", diff --git a/python/lsst/pipe/base/_task_metadata.py b/python/lsst/pipe/base/_task_metadata.py index 3daa2f2d1..98aec0be2 100644 --- a/python/lsst/pipe/base/_task_metadata.py +++ b/python/lsst/pipe/base/_task_metadata.py @@ -33,9 +33,8 @@ from collections.abc import Collection, Iterator, Mapping, Sequence from typing import Any, Protocol -from lsst.daf.butler._compat import _BaseModelCompat from lsst.utils.introspection import find_outside_stacklevel -from pydantic import Field, StrictBool, StrictFloat, StrictInt, StrictStr +from pydantic import BaseModel, Field, StrictBool, StrictFloat, StrictInt, StrictStr # The types allowed in a Task metadata field are restricted # to allow predictable serialization. @@ -60,7 +59,7 @@ def _isListLike(v: Any) -> bool: return isinstance(v, Sequence) and not isinstance(v, str) -class TaskMetadata(_BaseModelCompat): +class TaskMetadata(BaseModel): """Dict-like object for storing task metadata. Metadata can be stored at two levels: single task or task plus subtasks. diff --git a/python/lsst/pipe/base/graph/quantumNode.py b/python/lsst/pipe/base/graph/quantumNode.py index e3804164a..a20fc2c86 100644 --- a/python/lsst/pipe/base/graph/quantumNode.py +++ b/python/lsst/pipe/base/graph/quantumNode.py @@ -33,6 +33,7 @@ from dataclasses import dataclass from typing import Any, NewType +import pydantic from lsst.daf.butler import ( DatasetRef, DimensionRecord, @@ -41,7 +42,6 @@ Quantum, SerializedQuantum, ) -from lsst.daf.butler._compat import _BaseModelCompat from lsst.utils.introspection import find_outside_stacklevel from ..pipeline import TaskDef @@ -188,7 +188,7 @@ def _replace_quantum(self, quantum: Quantum) -> None: _fields_set = {"quantum", "taskLabel", "nodeId"} -class SerializedQuantumNode(_BaseModelCompat): +class SerializedQuantumNode(pydantic.BaseModel): """Model representing a `QuantumNode` in serializable form.""" quantum: SerializedQuantum diff --git a/python/lsst/pipe/base/pipeline_graph/io.py b/python/lsst/pipe/base/pipeline_graph/io.py index bfaba6664..391267b4e 100644 --- a/python/lsst/pipe/base/pipeline_graph/io.py +++ b/python/lsst/pipe/base/pipeline_graph/io.py @@ -42,7 +42,6 @@ import networkx import pydantic from lsst.daf.butler import DatasetType, DimensionConfig, DimensionGroup, DimensionUniverse -from lsst.daf.butler._compat import _BaseModelCompat from .. import automatic_connection_constants as acc from ._dataset_types import DatasetTypeNode @@ -85,7 +84,7 @@ def expect_not_none(value: _U | None, msg: str) -> _U: return value -class SerializedEdge(_BaseModelCompat): +class SerializedEdge(pydantic.BaseModel): """Struct used to represent a serialized `Edge` in a `PipelineGraph`. All `ReadEdge` and `WriteEdge` state not included here is instead @@ -204,7 +203,7 @@ def deserialize_write_edge( ) -class SerializedTaskInitNode(_BaseModelCompat): +class SerializedTaskInitNode(pydantic.BaseModel): """Struct used to represent a serialized `TaskInitNode` in a `PipelineGraph`. @@ -305,7 +304,7 @@ def deserialize( ) -class SerializedTaskNode(_BaseModelCompat): +class SerializedTaskNode(pydantic.BaseModel): """Struct used to represent a serialized `TaskNode` in a `PipelineGraph`. The task label is serialized by the context in which a @@ -461,7 +460,7 @@ def deserialize( ) -class SerializedDatasetTypeNode(_BaseModelCompat): +class SerializedDatasetTypeNode(pydantic.BaseModel): """Struct used to represent a serialized `DatasetTypeNode` in a `PipelineGraph`. @@ -583,7 +582,7 @@ def deserialize( return None -class SerializedTaskSubset(_BaseModelCompat): +class SerializedTaskSubset(pydantic.BaseModel): """Struct used to represent a serialized `TaskSubset` in a `PipelineGraph`. The subsetlabel is serialized by the context in which a @@ -634,7 +633,7 @@ def deserialize_task_subset(self, label: str, xgraph: networkx.MultiDiGraph) -> return TaskSubset(xgraph, label, members, self.description) -class SerializedPipelineGraph(_BaseModelCompat): +class SerializedPipelineGraph(pydantic.BaseModel): """Struct used to represent a serialized `PipelineGraph`.""" version: str = ".".join(str(v) for v in _IO_VERSION_INFO) diff --git a/python/lsst/pipe/base/tests/mocks/_storage_class.py b/python/lsst/pipe/base/tests/mocks/_storage_class.py index a238fb657..7901d2f96 100644 --- a/python/lsst/pipe/base/tests/mocks/_storage_class.py +++ b/python/lsst/pipe/base/tests/mocks/_storage_class.py @@ -41,6 +41,7 @@ from collections.abc import Callable, Iterable, Mapping from typing import Any, cast +import pydantic from lsst.daf.butler import ( DataIdValue, DatasetComponent, @@ -54,7 +55,6 @@ StorageClassDelegate, StorageClassFactory, ) -from lsst.daf.butler._compat import _BaseModelCompat from lsst.daf.butler.formatters.json import JsonFormatter from lsst.utils.introspection import get_full_type_name @@ -117,7 +117,7 @@ def is_mock_name(name: str) -> bool: # access to complex real storage classes (and their pytypes) to test against. -class MockDataset(_BaseModelCompat): +class MockDataset(pydantic.BaseModel): """The in-memory dataset type used by `MockStorageClass`.""" dataset_id: uuid.UUID | None @@ -189,9 +189,9 @@ def make_derived(self, **kwargs: Any) -> MockDataset: The newly-mocked dataset. """ dataset_type_updates = { - k: kwargs.pop(k) for k in list(kwargs) if k in SerializedDatasetType.model_fields # type: ignore + k: kwargs.pop(k) for k in list(kwargs) if k in SerializedDatasetType.model_fields } - kwargs.setdefault("dataset_type", self.dataset_type.copy(update=dataset_type_updates)) + kwargs.setdefault("dataset_type", self.dataset_type.model_copy(update=dataset_type_updates)) # Fields below are those that should not be propagated to the derived # dataset, because they're not about the intrinsic on-disk thing. kwargs.setdefault("converted_from", None) @@ -200,10 +200,10 @@ def make_derived(self, **kwargs: Any) -> MockDataset: # Also use setdefault on the ref in case caller wants to override that # directly, but this is expected to be rare enough that it's not worth # it to try to optimize out the work above to make derived_ref. - return self.copy(update=kwargs) + return self.model_copy(update=kwargs) -class MockDatasetQuantum(_BaseModelCompat): +class MockDatasetQuantum(pydantic.BaseModel): """Description of the quantum that produced a mock dataset. This is also used to represent task-init operations for init-output mock diff --git a/requirements.txt b/requirements.txt index 1c1d5f118..ba69f6ea8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ pyyaml >= 5.1 -pydantic < 3.0 +pydantic >=2,<3.0 numpy >= 1.17 networkx frozendict From 8221da7c72562ed9f9abe528f303b295bbcfe2a5 Mon Sep 17 00:00:00 2001 From: Jim Bosch Date: Thu, 21 Dec 2023 15:17:14 -0500 Subject: [PATCH 2/5] Pin documenteer==0.8.2. This version claims to be compatible with Pydantic v2 and it works for daf_butler, though there are reasons to doubt it's fully compatible. --- .github/workflows/build_docs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_docs.yaml b/.github/workflows/build_docs.yaml index 5b9eb6008..3e4f26972 100644 --- a/.github/workflows/build_docs.yaml +++ b/.github/workflows/build_docs.yaml @@ -31,7 +31,7 @@ jobs: pip install wheel - name: Install documenteer - run: pip install 'documenteer[pipelines]>=0.8' + run: pip install 'documenteer[pipelines]==0.8.2' - name: Install dependencies run: | From 8a0d0c198b344cb91145f74ed5d4ce47a6cfbb2f Mon Sep 17 00:00:00 2001 From: Jim Bosch Date: Thu, 21 Dec 2023 16:07:39 -0500 Subject: [PATCH 3/5] Add pydantic to local-doc-build intersphinx. --- doc/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/conf.py b/doc/conf.py index 2a065df7a..88fae6a47 100755 --- a/doc/conf.py +++ b/doc/conf.py @@ -9,3 +9,4 @@ intersphinx_mapping["networkx"] = ("https://networkx.org/documentation/stable/", None) # noqa: F405 intersphinx_mapping["lsst"] = ("https://pipelines.lsst.io/v/weekly/", None) # noqa: F405 +intersphinx_mapping["pydantic"] = ("https://docs.pydantic.dev/latest/", None) # noqa: F405 From 31863345783f37b59c117213b70cf231afd84ac5 Mon Sep 17 00:00:00 2001 From: Jim Bosch Date: Thu, 21 Dec 2023 17:25:58 -0500 Subject: [PATCH 4/5] Hide pydantic docstrings from sphinx. This isn't pretty at all, but I have yet to find another way. --- python/lsst/pipe/base/_task_metadata.py | 22 ++++++++++ .../pipe/base/tests/mocks/_storage_class.py | 43 +++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/python/lsst/pipe/base/_task_metadata.py b/python/lsst/pipe/base/_task_metadata.py index 98aec0be2..66702a5ae 100644 --- a/python/lsst/pipe/base/_task_metadata.py +++ b/python/lsst/pipe/base/_task_metadata.py @@ -29,6 +29,7 @@ import itertools import numbers +import sys import warnings from collections.abc import Collection, Iterator, Mapping, Sequence from typing import Any, Protocol @@ -571,6 +572,27 @@ def _validate_value(self, value: Any) -> tuple[str, Any]: raise ValueError(f"TaskMetadata does not support values of type {value!r}.") + # Work around the fact that Sphinx chokes on Pydantic docstring formatting, + # when we inherit those docstrings in our public classes. + if "sphinx" in sys.modules: + + def copy(self, *args: Any, **kwargs: Any) -> Any: + """See `pydantic.BaseModel.copy`.""" + return super().copy(*args, **kwargs) + + def model_dump(self, *args: Any, **kwargs: Any) -> Any: + """See `pydantic.BaseModel.model_dump`.""" + return super().model_dump(*args, **kwargs) + + def model_copy(self, *args: Any, **kwargs: Any) -> Any: + """See `pydantic.BaseModel.model_copy`.""" + return super().model_copy(*args, **kwargs) + + @classmethod + def model_json_schema(cls, *args: Any, **kwargs: Any) -> Any: + """See `pydantic.BaseModel.model_json_schema`.""" + return super().model_json_schema(*args, **kwargs) + # Needed because a TaskMetadata can contain a TaskMetadata. TaskMetadata.model_rebuild() diff --git a/python/lsst/pipe/base/tests/mocks/_storage_class.py b/python/lsst/pipe/base/tests/mocks/_storage_class.py index 7901d2f96..49e173712 100644 --- a/python/lsst/pipe/base/tests/mocks/_storage_class.py +++ b/python/lsst/pipe/base/tests/mocks/_storage_class.py @@ -37,6 +37,7 @@ "is_mock_name", ) +import sys import uuid from collections.abc import Callable, Iterable, Mapping from typing import Any, cast @@ -202,6 +203,27 @@ def make_derived(self, **kwargs: Any) -> MockDataset: # it to try to optimize out the work above to make derived_ref. return self.model_copy(update=kwargs) + # Work around the fact that Sphinx chokes on Pydantic docstring formatting, + # when we inherit those docstrings in our public classes. + if "sphinx" in sys.modules: + + def copy(self, *args: Any, **kwargs: Any) -> Any: + """See `pydantic.BaseModel.copy`.""" + return super().copy(*args, **kwargs) + + def model_dump(self, *args: Any, **kwargs: Any) -> Any: + """See `pydantic.BaseModel.model_dump`.""" + return super().model_dump(*args, **kwargs) + + def model_copy(self, *args: Any, **kwargs: Any) -> Any: + """See `pydantic.BaseModel.model_copy`.""" + return super().model_copy(*args, **kwargs) + + @classmethod + def model_json_schema(cls, *args: Any, **kwargs: Any) -> Any: + """See `pydantic.BaseModel.model_json_schema`.""" + return super().model_json_schema(*args, **kwargs) + class MockDatasetQuantum(pydantic.BaseModel): """Description of the quantum that produced a mock dataset. @@ -222,6 +244,27 @@ class MockDatasetQuantum(pydantic.BaseModel): Keys are task-internal connection names, not dataset type names. """ + # Work around the fact that Sphinx chokes on Pydantic docstring formatting, + # when we inherit those docstrings in our public classes. + if "sphinx" in sys.modules: + + def copy(self, *args: Any, **kwargs: Any) -> Any: + """See `pydantic.BaseModel.copy`.""" + return super().copy(*args, **kwargs) + + def model_dump(self, *args: Any, **kwargs: Any) -> Any: + """See `pydantic.BaseModel.model_dump`.""" + return super().model_dump(*args, **kwargs) + + def model_copy(self, *args: Any, **kwargs: Any) -> Any: + """See `pydantic.BaseModel.model_copy`.""" + return super().model_copy(*args, **kwargs) + + @classmethod + def model_json_schema(cls, *args: Any, **kwargs: Any) -> Any: + """See `pydantic.BaseModel.model_json_schema`.""" + return super().model_json_schema(*args, **kwargs) + MockDataset.model_rebuild() From 473a15e98622e788896401c019603f3430d6e660 Mon Sep 17 00:00:00 2001 From: Jim Bosch Date: Wed, 3 Jan 2024 11:47:49 -0500 Subject: [PATCH 5/5] Add changelog entry. --- doc/changes/DM-42302.misc.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/changes/DM-42302.misc.md diff --git a/doc/changes/DM-42302.misc.md b/doc/changes/DM-42302.misc.md new file mode 100644 index 000000000..35cc96214 --- /dev/null +++ b/doc/changes/DM-42302.misc.md @@ -0,0 +1 @@ +Drop support for Pydantic 1.x.