diff --git a/.changes/unreleased/Dependencies-20231125-005006.yaml b/.changes/unreleased/Dependencies-20231125-005006.yaml new file mode 100644 index 00000000..bf87f9b0 --- /dev/null +++ b/.changes/unreleased/Dependencies-20231125-005006.yaml @@ -0,0 +1,6 @@ +kind: Dependencies +body: bumped pydantic to ~=2.0 +time: 2023-11-25T00:50:06.062891+01:00 +custom: + Author: emmanuel.sciara@gmail.com + PR: "217" diff --git a/dbt_semantic_interfaces/dataclass_serialization.py b/dbt_semantic_interfaces/dataclass_serialization.py index 44ada7b1..f85432f3 100644 --- a/dbt_semantic_interfaces/dataclass_serialization.py +++ b/dbt_semantic_interfaces/dataclass_serialization.py @@ -234,7 +234,7 @@ def pydantic_serialize(self, obj: SerializableDataclassT) -> str: # noqa: D return self._convert_dataclass_instance_to_pydantic_model( object_type=obj_class, obj=obj, - ).json() + ).model_dump_json() class DataClassDeserializer: @@ -299,7 +299,7 @@ def pydantic_deserialize( # noqa: D try: ClassAsPydantic = self._to_pydantic_type_converter.to_pydantic_type(dataclass_type) logger.debug(f"Serialized object for creation of {ClassAsPydantic} is {serialized_obj}") - pydantic_object = ClassAsPydantic.parse_raw(serialized_obj) + pydantic_object = ClassAsPydantic.model_validate_json(serialized_obj) return self._construct_dataclass_from_dataclass_like_object( dataclass_type=dataclass_type, obj=pydantic_object, @@ -320,13 +320,13 @@ def __init__(self) -> None: # noqa: D def to_pydantic_type(self, dataclass_type: Type[SerializableDataclass]) -> Type[BaseModel]: # noqa: D if dataclass_type not in self._dataclass_type_to_pydantic_type: - self._dataclass_type_to_pydantic_type[ + self._dataclass_type_to_pydantic_type[dataclass_type] = self._convert_dataclass_type_to_pydantic_type( dataclass_type - ] = DataClassTypeToPydanticTypeConverter._convert_dataclass_type_to_pydantic_type(dataclass_type) + ) return self._dataclass_type_to_pydantic_type[dataclass_type] - @staticmethod def _convert_dataclass_type_to_pydantic_type( + self, dataclass_type: Type, ) -> Type[BaseModel]: # noqa: D logger.debug(f"Converting {dataclass_type.__name__} to a pydantic class") @@ -339,7 +339,7 @@ def _convert_dataclass_type_to_pydantic_type( fields_for_pydantic_model: Dict[str, Tuple[Type, AnyValueType]] = {} logger.debug(f"Need to add: {pformat_big_objects(field_dict.keys())}") for field_name, field_definition in field_dict.items(): - field_definition = DataClassTypeToPydanticTypeConverter._convert_nested_fields(field_definition) + field_definition = self._convert_nested_fields(field_definition) fields_for_pydantic_model[field_name] = field_definition.as_pydantic_field_tuple() logger.debug(f"Adding {field_name} with type {field_definition.annotated_field_type}") @@ -352,8 +352,7 @@ def _convert_dataclass_type_to_pydantic_type( logger.debug(f"Finished converting {dataclass_type.__name__} to a pydantic class") return pydantic_model - @staticmethod - def _convert_nested_fields(field_definition: FieldDefinition) -> FieldDefinition: + def _convert_nested_fields(self, field_definition: FieldDefinition) -> FieldDefinition: """Recursively converts a given field definition into a fully serializable type specification. The initial set of FieldDefinitions sourced from the dataclass might contain arbitrarily @@ -365,7 +364,7 @@ def _convert_nested_fields(field_definition: FieldDefinition) -> FieldDefinition raise RuntimeError(f"Unsupported type: {field_definition.annotated_field_type}") elif _is_optional_type(field_definition.annotated_field_type): optional_field_type_parameter = _get_type_parameter_for_optional(field_definition.annotated_field_type) - converted_field_definition = DataClassTypeToPydanticTypeConverter._convert_nested_fields( + converted_field_definition = self._convert_nested_fields( FieldDefinition(field_type=optional_field_type_parameter) ) return FieldDefinition( # type: ignore[arg-type] @@ -376,7 +375,7 @@ def _convert_nested_fields(field_definition: FieldDefinition) -> FieldDefinition tuple_field_type_parameter = _get_type_parameter_for_sequence_like_tuple_type( field_definition.annotated_field_type ) - converted_field_definition = DataClassTypeToPydanticTypeConverter._convert_nested_fields( + converted_field_definition = self._convert_nested_fields( FieldDefinition(field_type=tuple_field_type_parameter) ) return FieldDefinition( @@ -385,9 +384,7 @@ def _convert_nested_fields(field_definition: FieldDefinition) -> FieldDefinition ) elif issubclass(field_definition.annotated_field_type, SerializableDataclass): return FieldDefinition( - field_type=DataClassTypeToPydanticTypeConverter._convert_dataclass_type_to_pydantic_type( - field_definition.annotated_field_type - ), + field_type=self.to_pydantic_type(field_definition.annotated_field_type), default_value=field_definition.default_value, ) else: diff --git a/dbt_semantic_interfaces/implementations/base.py b/dbt_semantic_interfaces/implementations/base.py index 31c9faeb..ae3887d2 100644 --- a/dbt_semantic_interfaces/implementations/base.py +++ b/dbt_semantic_interfaces/implementations/base.py @@ -3,9 +3,9 @@ import json import os from abc import ABC, abstractmethod -from typing import Any, Callable, ClassVar, Generator, Generic, Type, TypeVar +from typing import Any, ClassVar, Dict, Generic, Type, TypeVar -from pydantic import BaseModel, root_validator +from pydantic import BaseModel, ConfigDict, model_validator from dbt_semantic_interfaces.errors import ParsingException from dbt_semantic_interfaces.parsing.yaml_loader import ( @@ -21,20 +21,17 @@ class HashableBaseModel(BaseModel): """Extends BaseModel with a generic hash function.""" def __hash__(self) -> int: # noqa: D - return hash(json.dumps(self.json(sort_keys=True), sort_keys=True)) + return hash(json.dumps(self.model_dump_json(), sort_keys=True)) class FrozenBaseModel(HashableBaseModel): """Similar to HashableBaseModel but faux immutable.""" - class Config: - """Pydantic feature.""" - - allow_mutation = False + model_config = ConfigDict(frozen=True) def to_pretty_json(self) -> str: """Convert to a pretty JSON representation.""" - raw_json_str = self.json() + raw_json_str = self.model_dump_json() json_obj = json.loads(raw_json_str) return json.dumps(json_obj, indent=4) @@ -58,7 +55,7 @@ class ModelWithMetadataParsing(BaseModel): __METADATA_KEY__: ClassVar[str] = "metadata" - @root_validator(pre=True) + @model_validator(mode="before") @classmethod def extract_metadata_from_parsing_context(cls, values: PydanticParseableValueType) -> PydanticParseableValueType: """Takes info from parsing context and converts it to a Metadata model object. @@ -122,23 +119,20 @@ class PydanticCustomInputParser(ABC, Generic[ModelObjectT_co]): and validation of that model object itself. """ + @model_validator(mode="before") @classmethod - def __get_validators__( - cls: Type[PydanticCustomInputParser[ModelObjectT_co]], - ) -> Generator[Callable[[PydanticParseableValueType], PydanticCustomInputParser[ModelObjectT_co]], None, None]: - """Pydantic magic method for allowing parsing of arbitrary input on parse_obj invocation. + def _model_validator( + cls: Type[PydanticCustomInputParser[ModelObjectT_co]], input: PydanticParseableValueType + ) -> Dict[str, Any]: + """Pydantic magic method for allowing parsing of arbitrary input on validate_model invocation. This allows for parsing and validation prior to object initialization. Most classes implementing this interface in our model are doing so because the input value from user-supplied YAML will be a string representation rather than the structured object type. - """ - yield cls.__parse_with_custom_handling - @classmethod - def __parse_with_custom_handling( - cls: Type[PydanticCustomInputParser[ModelObjectT_co]], input: PydanticParseableValueType - ) -> PydanticCustomInputParser[ModelObjectT_co]: - """Core method for handling common valid - or easily validated - input types. + the previous and next docstrings were from two different methods, which have been combined here. + + Core method for handling common valid - or easily validated - input types. Pydantic objects can commonly appear as JSON object types (from, e.g., deserializing a Pydantic-serialized model) or direct instances of the model object class (from, e.g., initializing an object and passing it in @@ -152,9 +146,10 @@ def __parse_with_custom_handling( to the caller to be pre-validated, and so we do not bother guarding against that here. """ if isinstance(input, dict): - return cls(**input) # type: ignore - elif isinstance(input, cls): return input + elif isinstance(input, cls): + # TODO: find a better way to avoid mypy type ignore + return input.model_dump() # type: ignore[attr-defined] else: return cls._from_yaml_value(input) @@ -162,6 +157,6 @@ def __parse_with_custom_handling( @abstractmethod def _from_yaml_value( cls: Type[PydanticCustomInputParser[ModelObjectT_co]], input: PydanticParseableValueType - ) -> PydanticCustomInputParser[ModelObjectT_co]: + ) -> Dict[str, Any]: """Abstract method for providing object-specific parsing logic.""" raise NotImplementedError() diff --git a/dbt_semantic_interfaces/implementations/elements/dimension.py b/dbt_semantic_interfaces/implementations/elements/dimension.py index c3cfa2e4..3cec128b 100644 --- a/dbt_semantic_interfaces/implementations/elements/dimension.py +++ b/dbt_semantic_interfaces/implementations/elements/dimension.py @@ -41,12 +41,12 @@ class PydanticDimension(HashableBaseModel, ModelWithMetadataParsing): """Describes a dimension.""" name: str - description: Optional[str] + description: Optional[str] = None type: DimensionType is_partition: bool = False - type_params: Optional[PydanticDimensionTypeParams] + type_params: Optional[PydanticDimensionTypeParams] = None expr: Optional[str] = None - metadata: Optional[PydanticMetadata] + metadata: Optional[PydanticMetadata] = None label: Optional[str] = None @property diff --git a/dbt_semantic_interfaces/implementations/elements/entity.py b/dbt_semantic_interfaces/implementations/elements/entity.py index ac98e4db..b08fc9cb 100644 --- a/dbt_semantic_interfaces/implementations/elements/entity.py +++ b/dbt_semantic_interfaces/implementations/elements/entity.py @@ -15,9 +15,9 @@ class PydanticEntity(HashableBaseModel, ModelWithMetadataParsing): """Describes a entity.""" name: str - description: Optional[str] type: EntityType - role: Optional[str] + description: Optional[str] = None + role: Optional[str] = None expr: Optional[str] = None metadata: Optional[PydanticMetadata] = None label: Optional[str] = None diff --git a/dbt_semantic_interfaces/implementations/elements/measure.py b/dbt_semantic_interfaces/implementations/elements/measure.py index 3c7407cc..8a6d2619 100644 --- a/dbt_semantic_interfaces/implementations/elements/measure.py +++ b/dbt_semantic_interfaces/implementations/elements/measure.py @@ -37,11 +37,11 @@ class PydanticMeasure(HashableBaseModel, ModelWithMetadataParsing): name: str agg: AggregationType - description: Optional[str] - create_metric: Optional[bool] + description: Optional[str] = None + create_metric: Optional[bool] = None expr: Optional[str] = None - agg_params: Optional[PydanticMeasureAggregationParameters] - metadata: Optional[PydanticMetadata] + agg_params: Optional[PydanticMeasureAggregationParameters] = None + metadata: Optional[PydanticMetadata] = None non_additive_dimension: Optional[PydanticNonAdditiveDimensionParameters] = None agg_time_dimension: Optional[str] = None label: Optional[str] = None diff --git a/dbt_semantic_interfaces/implementations/export.py b/dbt_semantic_interfaces/implementations/export.py index 7a06d1b0..df3685c9 100644 --- a/dbt_semantic_interfaces/implementations/export.py +++ b/dbt_semantic_interfaces/implementations/export.py @@ -2,7 +2,7 @@ from typing import Optional -from pydantic import Field +from pydantic import ConfigDict, Field from typing_extensions import override from dbt_semantic_interfaces.implementations.base import HashableBaseModel @@ -21,8 +21,7 @@ class PydanticExportConfig(HashableBaseModel, ProtocolHint[ExportConfig]): enables parsing for both `schema` and `schema_name` when deserializing from JSON. """ - class Config: # noqa: D - allow_population_by_field_name = True + model_config = ConfigDict(populate_by_name=True) @override def _implements_protocol(self) -> ExportConfig: diff --git a/dbt_semantic_interfaces/implementations/filters/where_filter.py b/dbt_semantic_interfaces/implementations/filters/where_filter.py index 0be414e6..19b201e1 100644 --- a/dbt_semantic_interfaces/implementations/filters/where_filter.py +++ b/dbt_semantic_interfaces/implementations/filters/where_filter.py @@ -1,8 +1,8 @@ from __future__ import annotations -from typing import Callable, Generator, List, Tuple +from typing import Any, Dict, List, Tuple -from typing_extensions import Self +from pydantic import model_validator from dbt_semantic_interfaces.call_parameter_sets import ( FilterCallParameterSets, @@ -34,17 +34,14 @@ class PydanticWhereFilter(PydanticCustomInputParser, HashableBaseModel): where_sql_template: str @classmethod - def _from_yaml_value( - cls, - input: PydanticParseableValueType, - ) -> PydanticWhereFilter: + def _from_yaml_value(cls, input: PydanticParseableValueType) -> Dict[str, Any]: """Parses a WhereFilter from a string found in a user-provided model specification. User-provided constraint strings are SQL snippets conforming to the expectations of SQL WHERE clauses, and as such we parse them using our standard parse method below. """ if isinstance(input, str): - return PydanticWhereFilter(where_sql_template=input) + return {"where_sql_template": input} else: raise ValueError(f"Expected input to be of type string, but got type {type(input)} with value: {input}") @@ -62,17 +59,9 @@ class PydanticWhereFilterIntersection(HashableBaseModel): where_filters: List[PydanticWhereFilter] + @model_validator(mode="before") @classmethod - def __get_validators__(cls) -> Generator[Callable[[PydanticParseableValueType], Self], None, None]: - """Pydantic magic method for allowing handling of arbitrary input on parse_obj invocation. - - This class requires more subtle handling of input deserialized object types (dicts), and so it cannot - extend the common interface via _from_yaml_values. - """ - yield cls._convert_legacy_and_yaml_input - - @classmethod - def _convert_legacy_and_yaml_input(cls, input: PydanticParseableValueType) -> Self: + def _convert_legacy_and_yaml_input(cls, input: PydanticParseableValueType) -> Dict[str, Any]: """Specifies raw input conversion rules to ensure serialized semantic manifests will parse correctly. The original spec for where filters relied on a raw WhereFilter object, but this has now been updated to @@ -101,13 +90,13 @@ def _convert_legacy_and_yaml_input(cls, input: PydanticParseableValueType) -> Se is_legacy_where_filter = isinstance(input, str) or isinstance(input, PydanticWhereFilter) or has_legacy_keys if is_legacy_where_filter: - return cls(where_filters=[input]) + return {"where_filters": [input]} elif isinstance(input, list): - return cls(where_filters=input) + return {"where_filters": input} elif isinstance(input, dict): - return cls(**input) - elif isinstance(input, cls): return input + elif isinstance(input, cls): + return input.model_dump() else: raise ValueError( f"Expected input to be of type string, list, PydanticWhereFilter, PydanticWhereFilterIntersection, " diff --git a/dbt_semantic_interfaces/implementations/metric.py b/dbt_semantic_interfaces/implementations/metric.py index 366648a4..ca3ca5d0 100644 --- a/dbt_semantic_interfaces/implementations/metric.py +++ b/dbt_semantic_interfaces/implementations/metric.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import List, Optional, Sequence +from typing import Any, Dict, List, Optional, Sequence from pydantic import Field @@ -32,13 +32,13 @@ class PydanticMetricInputMeasure(PydanticCustomInputParser, HashableBaseModel): """ name: str - filter: Optional[PydanticWhereFilterIntersection] - alias: Optional[str] + filter: Optional[PydanticWhereFilterIntersection] = None + alias: Optional[str] = None join_to_timespine: bool = False fill_nulls_with: Optional[int] = None @classmethod - def _from_yaml_value(cls, input: PydanticParseableValueType) -> PydanticMetricInputMeasure: + def _from_yaml_value(cls, input: PydanticParseableValueType) -> Dict[str, Any]: """Parses a MetricInputMeasure from a string (name only) or object (struct spec) input. For user input cases, the original YAML spec for a PydanticMetric included measure(s) specified as string names @@ -46,7 +46,7 @@ def _from_yaml_value(cls, input: PydanticParseableValueType) -> PydanticMetricIn base name for this object. """ if isinstance(input, str): - return PydanticMetricInputMeasure(name=input) + return {"name": input} else: raise ValueError( f"MetricInputMeasure inputs from model configs are expected to be of either type string or " @@ -71,26 +71,22 @@ class PydanticMetricTimeWindow(PydanticCustomInputParser, HashableBaseModel): granularity: TimeGranularity @classmethod - def _from_yaml_value(cls, input: PydanticParseableValueType) -> PydanticMetricTimeWindow: + def _from_yaml_value(cls, input: PydanticParseableValueType) -> Dict[str, Any]: """Parses a MetricTimeWindow from a string input found in a user provided model specification. The MetricTimeWindow is always expected to be provided as a string in user-defined YAML configs. + + Output of the form: (