From ab13d5dc7d4e689e76565b34dba2a8cab0954775 Mon Sep 17 00:00:00 2001 From: Quigley Malcolm Date: Tue, 15 Aug 2023 12:39:14 -0700 Subject: [PATCH 01/20] Bump pydantic dep to `~=2.0` --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a892f39d..3aae8e0b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ classifiers = [ "Programming Language :: Python :: Implementation :: PyPy", ] dependencies = [ - "pydantic~=1.10", + "pydantic~=2.0", "jsonschema~=4.0", "PyYAML~=6.0", "more-itertools>=8.0,<11.0", From 3a4925cba3a7950737bc2666fdf7229b795cbd77 Mon Sep 17 00:00:00 2001 From: Quigley Malcolm Date: Tue, 15 Aug 2023 12:54:53 -0700 Subject: [PATCH 02/20] Explicitly set `Optional` pydantic object fields to default to `None` In pydantic 1.x `Optional` fields on pydantic objects automatically got defaulted to `None`. That is no longer the case in pydantic 2.x. --- .../implementations/elements/dimension.py | 6 +-- .../implementations/elements/entity.py | 4 +- .../implementations/elements/measure.py | 8 ++-- .../implementations/metric.py | 38 +++++++++---------- .../implementations/semantic_model.py | 10 ++--- .../implementations/semantic_version.py | 2 +- dbt_semantic_interfaces/parsing/objects.py | 2 +- .../validations/validator_helpers.py | 6 +-- 8 files changed, 38 insertions(+), 38 deletions(-) diff --git a/dbt_semantic_interfaces/implementations/elements/dimension.py b/dbt_semantic_interfaces/implementations/elements/dimension.py index c3cfa2e4..3cec128b 100644 --- a/dbt_semantic_interfaces/implementations/elements/dimension.py +++ b/dbt_semantic_interfaces/implementations/elements/dimension.py @@ -41,12 +41,12 @@ class PydanticDimension(HashableBaseModel, ModelWithMetadataParsing): """Describes a dimension.""" name: str - description: Optional[str] + description: Optional[str] = None type: DimensionType is_partition: bool = False - type_params: Optional[PydanticDimensionTypeParams] + type_params: Optional[PydanticDimensionTypeParams] = None expr: Optional[str] = None - metadata: Optional[PydanticMetadata] + metadata: Optional[PydanticMetadata] = None label: Optional[str] = None @property diff --git a/dbt_semantic_interfaces/implementations/elements/entity.py b/dbt_semantic_interfaces/implementations/elements/entity.py index ac98e4db..b08fc9cb 100644 --- a/dbt_semantic_interfaces/implementations/elements/entity.py +++ b/dbt_semantic_interfaces/implementations/elements/entity.py @@ -15,9 +15,9 @@ class PydanticEntity(HashableBaseModel, ModelWithMetadataParsing): """Describes a entity.""" name: str - description: Optional[str] type: EntityType - role: Optional[str] + description: Optional[str] = None + role: Optional[str] = None expr: Optional[str] = None metadata: Optional[PydanticMetadata] = None label: Optional[str] = None diff --git a/dbt_semantic_interfaces/implementations/elements/measure.py b/dbt_semantic_interfaces/implementations/elements/measure.py index 3c7407cc..8a6d2619 100644 --- a/dbt_semantic_interfaces/implementations/elements/measure.py +++ b/dbt_semantic_interfaces/implementations/elements/measure.py @@ -37,11 +37,11 @@ class PydanticMeasure(HashableBaseModel, ModelWithMetadataParsing): name: str agg: AggregationType - description: Optional[str] - create_metric: Optional[bool] + description: Optional[str] = None + create_metric: Optional[bool] = None expr: Optional[str] = None - agg_params: Optional[PydanticMeasureAggregationParameters] - metadata: Optional[PydanticMetadata] + agg_params: Optional[PydanticMeasureAggregationParameters] = None + metadata: Optional[PydanticMetadata] = None non_additive_dimension: Optional[PydanticNonAdditiveDimensionParameters] = None agg_time_dimension: Optional[str] = None label: Optional[str] = None diff --git a/dbt_semantic_interfaces/implementations/metric.py b/dbt_semantic_interfaces/implementations/metric.py index 366648a4..e4bbcb67 100644 --- a/dbt_semantic_interfaces/implementations/metric.py +++ b/dbt_semantic_interfaces/implementations/metric.py @@ -32,8 +32,8 @@ class PydanticMetricInputMeasure(PydanticCustomInputParser, HashableBaseModel): """ name: str - filter: Optional[PydanticWhereFilterIntersection] - alias: Optional[str] + filter: Optional[PydanticWhereFilterIntersection] = None + alias: Optional[str] = None join_to_timespine: bool = False fill_nulls_with: Optional[int] = None @@ -129,10 +129,10 @@ class PydanticMetricInput(HashableBaseModel): """Provides a pointer to a metric along with the additional properties used on that metric.""" name: str - filter: Optional[PydanticWhereFilterIntersection] - alias: Optional[str] - offset_window: Optional[PydanticMetricTimeWindow] - offset_to_grain: Optional[TimeGranularity] + filter: Optional[PydanticWhereFilterIntersection] = None + alias: Optional[str] = None + offset_window: Optional[PydanticMetricTimeWindow] = None + offset_to_grain: Optional[TimeGranularity] = None @property def as_reference(self) -> MetricReference: @@ -152,21 +152,21 @@ class PydanticConversionTypeParams(HashableBaseModel): conversion_measure: PydanticMetricInputMeasure entity: str calculation: ConversionCalculationType = ConversionCalculationType.CONVERSION_RATE - window: Optional[PydanticMetricTimeWindow] - constant_properties: Optional[List[PydanticConstantPropertyInput]] + window: Optional[PydanticMetricTimeWindow] = None + constant_properties: Optional[List[PydanticConstantPropertyInput]] = None class PydanticMetricTypeParams(HashableBaseModel): """Type params add additional context to certain metric types (the context depends on the metric type).""" - measure: Optional[PydanticMetricInputMeasure] - numerator: Optional[PydanticMetricInput] - denominator: Optional[PydanticMetricInput] - expr: Optional[str] - window: Optional[PydanticMetricTimeWindow] - grain_to_date: Optional[TimeGranularity] - metrics: Optional[List[PydanticMetricInput]] - conversion_type_params: Optional[PydanticConversionTypeParams] + measure: Optional[PydanticMetricInputMeasure] = None + numerator: Optional[PydanticMetricInput] = None + denominator: Optional[PydanticMetricInput] = None + expr: Optional[str] = None + window: Optional[PydanticMetricTimeWindow] = None + grain_to_date: Optional[TimeGranularity] = None + metrics: Optional[List[PydanticMetricInput]] = None + conversion_type_params: Optional[PydanticConversionTypeParams] = None input_measures: List[PydanticMetricInputMeasure] = Field(default_factory=list) @@ -175,11 +175,11 @@ class PydanticMetric(HashableBaseModel, ModelWithMetadataParsing): """Describes a metric.""" name: str - description: Optional[str] + description: Optional[str] = None type: MetricType type_params: PydanticMetricTypeParams - filter: Optional[PydanticWhereFilterIntersection] - metadata: Optional[PydanticMetadata] + filter: Optional[PydanticWhereFilterIntersection] = None + metadata: Optional[PydanticMetadata] = None label: Optional[str] = None @property diff --git a/dbt_semantic_interfaces/implementations/semantic_model.py b/dbt_semantic_interfaces/implementations/semantic_model.py index 45b90ae0..701ad1cd 100644 --- a/dbt_semantic_interfaces/implementations/semantic_model.py +++ b/dbt_semantic_interfaces/implementations/semantic_model.py @@ -73,7 +73,7 @@ class PydanticSemanticModelDefaults(HashableBaseModel, ProtocolHint[SemanticMode def _implements_protocol(self) -> SemanticModelDefaults: # noqa: D return self - agg_time_dimension: Optional[str] + agg_time_dimension: Optional[str] = None class PydanticSemanticModel(HashableBaseModel, ModelWithMetadataParsing, ProtocolHint[SemanticModel]): @@ -84,17 +84,17 @@ def _implements_protocol(self) -> SemanticModel: return self name: str - defaults: Optional[PydanticSemanticModelDefaults] - description: Optional[str] node_relation: NodeRelation + defaults: Optional[PydanticSemanticModelDefaults] = None + description: Optional[str] = None - primary_entity: Optional[str] + primary_entity: Optional[str] = None entities: Sequence[PydanticEntity] = [] measures: Sequence[PydanticMeasure] = [] dimensions: Sequence[PydanticDimension] = [] label: Optional[str] = None - metadata: Optional[PydanticMetadata] + metadata: Optional[PydanticMetadata] = None @property def entity_references(self) -> List[LinkableElementReference]: # noqa: D diff --git a/dbt_semantic_interfaces/implementations/semantic_version.py b/dbt_semantic_interfaces/implementations/semantic_version.py index 212cea61..f8a6d8a4 100644 --- a/dbt_semantic_interfaces/implementations/semantic_version.py +++ b/dbt_semantic_interfaces/implementations/semantic_version.py @@ -16,7 +16,7 @@ class PydanticSemanticVersion(PydanticCustomInputParser, HashableBaseModel): major_version: str minor_version: str - patch_version: Optional[str] + patch_version: Optional[str] = None @classmethod @override diff --git a/dbt_semantic_interfaces/parsing/objects.py b/dbt_semantic_interfaces/parsing/objects.py index adc8f424..3aebd083 100644 --- a/dbt_semantic_interfaces/parsing/objects.py +++ b/dbt_semantic_interfaces/parsing/objects.py @@ -16,7 +16,7 @@ class YamlConfigFile(HashableBaseModel): filepath: str contents: str - url: Optional[str] + url: Optional[str] = None class Version(HashableBaseModel): # noqa: D diff --git a/dbt_semantic_interfaces/validations/validator_helpers.py b/dbt_semantic_interfaces/validations/validator_helpers.py index 6ea60f48..f557292e 100644 --- a/dbt_semantic_interfaces/validations/validator_helpers.py +++ b/dbt_semantic_interfaces/validations/validator_helpers.py @@ -70,8 +70,8 @@ class SemanticModelElementType(Enum): class FileContext(BaseModel): """The base context class for validation issues.""" - file_name: Optional[str] - line_number: Optional[int] + file_name: Optional[str] = None + line_number: Optional[int] = None class Config: """Pydantic class configuration options.""" @@ -185,7 +185,7 @@ class ValidationIssue(ABC, BaseModel): message: str context: Optional[ValidationContext] = None - extra_detail: Optional[str] + extra_detail: Optional[str] = None @property @abstractmethod From 77d2d9debbece022e896f56e67d8c4af68b6d06e Mon Sep 17 00:00:00 2001 From: Quigley Malcolm Date: Tue, 15 Aug 2023 12:58:12 -0700 Subject: [PATCH 03/20] Migrate to `model_dump_json()` from deprecated `json()` --- dbt_semantic_interfaces/dataclass_serialization.py | 2 +- dbt_semantic_interfaces/implementations/base.py | 4 ++-- .../validations/semantic_manifest_validator.py | 2 +- tests/parsing/test_model_deserialization.py | 2 +- tests/validations/test_validator_helpers.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dbt_semantic_interfaces/dataclass_serialization.py b/dbt_semantic_interfaces/dataclass_serialization.py index 44ada7b1..3ab959e4 100644 --- a/dbt_semantic_interfaces/dataclass_serialization.py +++ b/dbt_semantic_interfaces/dataclass_serialization.py @@ -234,7 +234,7 @@ def pydantic_serialize(self, obj: SerializableDataclassT) -> str: # noqa: D return self._convert_dataclass_instance_to_pydantic_model( object_type=obj_class, obj=obj, - ).json() + ).model_dump_json() class DataClassDeserializer: diff --git a/dbt_semantic_interfaces/implementations/base.py b/dbt_semantic_interfaces/implementations/base.py index 31c9faeb..e043ae78 100644 --- a/dbt_semantic_interfaces/implementations/base.py +++ b/dbt_semantic_interfaces/implementations/base.py @@ -21,7 +21,7 @@ class HashableBaseModel(BaseModel): """Extends BaseModel with a generic hash function.""" def __hash__(self) -> int: # noqa: D - return hash(json.dumps(self.json(sort_keys=True), sort_keys=True)) + return hash(json.dumps(self.model_dump_json(), sort_keys=True)) class FrozenBaseModel(HashableBaseModel): @@ -34,7 +34,7 @@ class Config: def to_pretty_json(self) -> str: """Convert to a pretty JSON representation.""" - raw_json_str = self.json() + raw_json_str = self.model_dump_json() json_obj = json.loads(raw_json_str) return json.dumps(json_obj, indent=4) diff --git a/dbt_semantic_interfaces/validations/semantic_manifest_validator.py b/dbt_semantic_interfaces/validations/semantic_manifest_validator.py index 2093a9bc..520c8b64 100644 --- a/dbt_semantic_interfaces/validations/semantic_manifest_validator.py +++ b/dbt_semantic_interfaces/validations/semantic_manifest_validator.py @@ -59,7 +59,7 @@ def _validate_manifest_with_one_rule( """ return SemanticManifestValidationResults.from_issues_sequence( validation_rule.validate_manifest(semantic_manifest) - ).json() + ).model_dump_json() class SemanticManifestValidator(Generic[SemanticManifestT]): diff --git a/tests/parsing/test_model_deserialization.py b/tests/parsing/test_model_deserialization.py index bd122b6e..e6eaf0a8 100644 --- a/tests/parsing/test_model_deserialization.py +++ b/tests/parsing/test_model_deserialization.py @@ -9,6 +9,6 @@ def test_model_serialization_deserialization(simple_semantic_manifest: PydanticS This ensures any custom parsing operations internal to our Pydantic models are properly applied to not only user-provided YAML input, but also to internal parsing operations based on serialized model objects. """ - serialized_model = simple_semantic_manifest.json() + serialized_model = simple_semantic_manifest.model_dump_json() deserialized_model = simple_semantic_manifest.parse_raw(serialized_model) assert deserialized_model == simple_semantic_manifest diff --git a/tests/validations/test_validator_helpers.py b/tests/validations/test_validator_helpers.py index 056ceb1a..4ccbb68d 100644 --- a/tests/validations/test_validator_helpers.py +++ b/tests/validations/test_validator_helpers.py @@ -115,7 +115,7 @@ def test_jsonifying_and_reloading_model_validation_results_is_equal( # noqa: D set_context_types = set([issue.context.__class__ for issue in list_of_issues]) model_validation_issues = SemanticManifestValidationResults.from_issues_sequence(list_of_issues) - model_validation_issues_new = SemanticManifestValidationResults.parse_raw(model_validation_issues.json()) + model_validation_issues_new = SemanticManifestValidationResults.parse_raw(model_validation_issues.model_dump_json()) assert model_validation_issues_new == model_validation_issues assert model_validation_issues_new != SemanticManifestValidationResults( warnings=model_validation_issues.warnings, From 91e1af34f70d2da6415b57510990e3b98e57673e Mon Sep 17 00:00:00 2001 From: Quigley Malcolm Date: Tue, 15 Aug 2023 12:59:56 -0700 Subject: [PATCH 04/20] Migrate to `model_validate()` from deprecated `parse_obj()` --- dbt_semantic_interfaces/implementations/base.py | 2 +- dbt_semantic_interfaces/parsing/dir_to_model.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dbt_semantic_interfaces/implementations/base.py b/dbt_semantic_interfaces/implementations/base.py index e043ae78..e211df2c 100644 --- a/dbt_semantic_interfaces/implementations/base.py +++ b/dbt_semantic_interfaces/implementations/base.py @@ -126,7 +126,7 @@ class PydanticCustomInputParser(ABC, Generic[ModelObjectT_co]): def __get_validators__( cls: Type[PydanticCustomInputParser[ModelObjectT_co]], ) -> Generator[Callable[[PydanticParseableValueType], PydanticCustomInputParser[ModelObjectT_co]], None, None]: - """Pydantic magic method for allowing parsing of arbitrary input on parse_obj invocation. + """Pydantic magic method for allowing parsing of arbitrary input on validate_model invocation. This allows for parsing and validation prior to object initialization. Most classes implementing this interface in our model are doing so because the input value from user-supplied YAML will be a string diff --git a/dbt_semantic_interfaces/parsing/dir_to_model.py b/dbt_semantic_interfaces/parsing/dir_to_model.py index 418ca254..1cf7adc0 100644 --- a/dbt_semantic_interfaces/parsing/dir_to_model.py +++ b/dbt_semantic_interfaces/parsing/dir_to_model.py @@ -331,16 +331,16 @@ def parse_config_yaml( try: if document_type == METRIC_TYPE: metric_validator.validate(config_document[document_type]) - results.append(metric_class.parse_obj(object_cfg)) + results.append(metric_class.model_validate(object_cfg)) elif document_type == SEMANTIC_MODEL_TYPE: semantic_model_validator.validate(config_document[document_type]) - results.append(semantic_model_class.parse_obj(object_cfg)) + results.append(semantic_model_class.model_validate(object_cfg)) elif document_type == PROJECT_CONFIGURATION_TYPE: project_configuration_validator.validate(config_document[document_type]) - results.append(project_configuration_class.parse_obj(object_cfg)) + results.append(project_configuration_class.model_validate(object_cfg)) elif document_type == SAVED_QUERY_TYPE: saved_query_validator.validate(config_document[document_type]) - results.append(saved_query_class.parse_obj(object_cfg)) + results.append(saved_query_class.model_validate(object_cfg)) else: issues.append( ValidationError( @@ -358,7 +358,7 @@ def parse_config_yaml( extra_detail="".join(traceback.format_tb(e.__traceback__)), ) ) - # ParsingException: catches exceptions from *.parse_obj calls + # ParsingException: catches exceptions from *.model_validate calls # Exception: general exception for a given document. Basicially we # don't want an exception on one document to halt checking the rest # of the documents From 651c68c48cf5740d07ca7e4b1ee78eb45144841a Mon Sep 17 00:00:00 2001 From: Quigley Malcolm Date: Tue, 15 Aug 2023 13:08:17 -0700 Subject: [PATCH 05/20] Migrate to `model_validate_json()` from deprecated `parse_raw()` --- dbt_semantic_interfaces/dataclass_serialization.py | 2 +- .../validations/semantic_manifest_validator.py | 2 +- tests/parsing/test_model_deserialization.py | 2 +- tests/validations/test_validator_helpers.py | 4 +++- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/dbt_semantic_interfaces/dataclass_serialization.py b/dbt_semantic_interfaces/dataclass_serialization.py index 3ab959e4..01cda7a4 100644 --- a/dbt_semantic_interfaces/dataclass_serialization.py +++ b/dbt_semantic_interfaces/dataclass_serialization.py @@ -299,7 +299,7 @@ def pydantic_deserialize( # noqa: D try: ClassAsPydantic = self._to_pydantic_type_converter.to_pydantic_type(dataclass_type) logger.debug(f"Serialized object for creation of {ClassAsPydantic} is {serialized_obj}") - pydantic_object = ClassAsPydantic.parse_raw(serialized_obj) + pydantic_object = ClassAsPydantic.model_validate_json(serialized_obj) return self._construct_dataclass_from_dataclass_like_object( dataclass_type=dataclass_type, obj=pydantic_object, diff --git a/dbt_semantic_interfaces/validations/semantic_manifest_validator.py b/dbt_semantic_interfaces/validations/semantic_manifest_validator.py index 520c8b64..231de96a 100644 --- a/dbt_semantic_interfaces/validations/semantic_manifest_validator.py +++ b/dbt_semantic_interfaces/validations/semantic_manifest_validator.py @@ -140,7 +140,7 @@ def _validate_multi_process( # noqa: D ] for future in as_completed(futures): res = future.result() - result = SemanticManifestValidationResults.parse_raw(res) + result = SemanticManifestValidationResults.model_validate_json(res) results.append(result) return SemanticManifestValidationResults.merge(results) diff --git a/tests/parsing/test_model_deserialization.py b/tests/parsing/test_model_deserialization.py index e6eaf0a8..5635c99a 100644 --- a/tests/parsing/test_model_deserialization.py +++ b/tests/parsing/test_model_deserialization.py @@ -10,5 +10,5 @@ def test_model_serialization_deserialization(simple_semantic_manifest: PydanticS user-provided YAML input, but also to internal parsing operations based on serialized model objects. """ serialized_model = simple_semantic_manifest.model_dump_json() - deserialized_model = simple_semantic_manifest.parse_raw(serialized_model) + deserialized_model = simple_semantic_manifest.model_validate_json(serialized_model) assert deserialized_model == simple_semantic_manifest diff --git a/tests/validations/test_validator_helpers.py b/tests/validations/test_validator_helpers.py index 4ccbb68d..86b1fae6 100644 --- a/tests/validations/test_validator_helpers.py +++ b/tests/validations/test_validator_helpers.py @@ -115,7 +115,9 @@ def test_jsonifying_and_reloading_model_validation_results_is_equal( # noqa: D set_context_types = set([issue.context.__class__ for issue in list_of_issues]) model_validation_issues = SemanticManifestValidationResults.from_issues_sequence(list_of_issues) - model_validation_issues_new = SemanticManifestValidationResults.parse_raw(model_validation_issues.model_dump_json()) + model_validation_issues_new = SemanticManifestValidationResults.model_validate_json( + model_validation_issues.model_dump_json() + ) assert model_validation_issues_new == model_validation_issues assert model_validation_issues_new != SemanticManifestValidationResults( warnings=model_validation_issues.warnings, From 5180749a5352eed6f33c6624e95d0b8d997f2b2f Mon Sep 17 00:00:00 2001 From: Emmanuel Sciara Date: Fri, 24 Nov 2023 00:27:51 +0100 Subject: [PATCH 06/20] Bump pydantic dep to `~=2.5` --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3aae8e0b..64f16da6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ classifiers = [ "Programming Language :: Python :: Implementation :: PyPy", ] dependencies = [ - "pydantic~=2.0", + "pydantic~=2.5", "jsonschema~=4.0", "PyYAML~=6.0", "more-itertools>=8.0,<11.0", From 826bd37b19147e626c64013e357cde847333ee6f Mon Sep 17 00:00:00 2001 From: Emmanuel Sciara Date: Fri, 24 Nov 2023 00:36:07 +0100 Subject: [PATCH 07/20] Migrate to `ConfigDict` from deprecated `class Config` --- dbt_semantic_interfaces/implementations/base.py | 7 ++----- dbt_semantic_interfaces/implementations/export.py | 5 ++--- dbt_semantic_interfaces/validations/validator_helpers.py | 8 ++------ 3 files changed, 6 insertions(+), 14 deletions(-) diff --git a/dbt_semantic_interfaces/implementations/base.py b/dbt_semantic_interfaces/implementations/base.py index e211df2c..895caed9 100644 --- a/dbt_semantic_interfaces/implementations/base.py +++ b/dbt_semantic_interfaces/implementations/base.py @@ -5,7 +5,7 @@ from abc import ABC, abstractmethod from typing import Any, Callable, ClassVar, Generator, Generic, Type, TypeVar -from pydantic import BaseModel, root_validator +from pydantic import BaseModel, ConfigDict, root_validator from dbt_semantic_interfaces.errors import ParsingException from dbt_semantic_interfaces.parsing.yaml_loader import ( @@ -27,10 +27,7 @@ def __hash__(self) -> int: # noqa: D class FrozenBaseModel(HashableBaseModel): """Similar to HashableBaseModel but faux immutable.""" - class Config: - """Pydantic feature.""" - - allow_mutation = False + model_config = ConfigDict(frozen=True) def to_pretty_json(self) -> str: """Convert to a pretty JSON representation.""" diff --git a/dbt_semantic_interfaces/implementations/export.py b/dbt_semantic_interfaces/implementations/export.py index 7a06d1b0..df3685c9 100644 --- a/dbt_semantic_interfaces/implementations/export.py +++ b/dbt_semantic_interfaces/implementations/export.py @@ -2,7 +2,7 @@ from typing import Optional -from pydantic import Field +from pydantic import ConfigDict, Field from typing_extensions import override from dbt_semantic_interfaces.implementations.base import HashableBaseModel @@ -21,8 +21,7 @@ class PydanticExportConfig(HashableBaseModel, ProtocolHint[ExportConfig]): enables parsing for both `schema` and `schema_name` when deserializing from JSON. """ - class Config: # noqa: D - allow_population_by_field_name = True + model_config = ConfigDict(populate_by_name=True) @override def _implements_protocol(self) -> ExportConfig: diff --git a/dbt_semantic_interfaces/validations/validator_helpers.py b/dbt_semantic_interfaces/validations/validator_helpers.py index f557292e..bc86c804 100644 --- a/dbt_semantic_interfaces/validations/validator_helpers.py +++ b/dbt_semantic_interfaces/validations/validator_helpers.py @@ -20,7 +20,7 @@ ) import click -from pydantic import BaseModel, Extra +from pydantic import BaseModel, ConfigDict from dbt_semantic_interfaces.implementations.base import FrozenBaseModel from dbt_semantic_interfaces.protocols import Metadata, SemanticManifestT, SemanticModel @@ -72,11 +72,7 @@ class FileContext(BaseModel): file_name: Optional[str] = None line_number: Optional[int] = None - - class Config: - """Pydantic class configuration options.""" - - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") def context_str(self) -> str: """Human-readable stringified representation of the context.""" From ae3e72917c49eacc1ddb24c7ee0a532c821642fd Mon Sep 17 00:00:00 2001 From: Emmanuel Sciara Date: Fri, 24 Nov 2023 00:37:57 +0100 Subject: [PATCH 08/20] Migrate to `model_validator` from deprecated `root_validator` --- dbt_semantic_interfaces/implementations/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt_semantic_interfaces/implementations/base.py b/dbt_semantic_interfaces/implementations/base.py index 895caed9..5dfafdb6 100644 --- a/dbt_semantic_interfaces/implementations/base.py +++ b/dbt_semantic_interfaces/implementations/base.py @@ -5,7 +5,7 @@ from abc import ABC, abstractmethod from typing import Any, Callable, ClassVar, Generator, Generic, Type, TypeVar -from pydantic import BaseModel, ConfigDict, root_validator +from pydantic import BaseModel, ConfigDict, model_validator from dbt_semantic_interfaces.errors import ParsingException from dbt_semantic_interfaces.parsing.yaml_loader import ( @@ -55,7 +55,7 @@ class ModelWithMetadataParsing(BaseModel): __METADATA_KEY__: ClassVar[str] = "metadata" - @root_validator(pre=True) + @model_validator(mode="before") @classmethod def extract_metadata_from_parsing_context(cls, values: PydanticParseableValueType) -> PydanticParseableValueType: """Takes info from parsing context and converts it to a Metadata model object. From 1e7581c9dff28798bf5da8cad36f751af86b9cad Mon Sep 17 00:00:00 2001 From: Emmanuel Sciara Date: Fri, 24 Nov 2023 17:17:29 +0100 Subject: [PATCH 09/20] Migrate to `field_validator` and `model_validator` from deprecated `validator` --- .../implementations/project_configuration.py | 4 +-- .../implementations/semantic_model.py | 30 +++++++------------ 2 files changed, 12 insertions(+), 22 deletions(-) diff --git a/dbt_semantic_interfaces/implementations/project_configuration.py b/dbt_semantic_interfaces/implementations/project_configuration.py index 3e52fd91..dd8b94e0 100644 --- a/dbt_semantic_interfaces/implementations/project_configuration.py +++ b/dbt_semantic_interfaces/implementations/project_configuration.py @@ -3,7 +3,7 @@ from typing import List, Optional from importlib_metadata import version -from pydantic import validator +from pydantic import field_validator from typing_extensions import override from dbt_semantic_interfaces.implementations.base import ( @@ -33,7 +33,7 @@ def _implements_protocol(self) -> ProjectConfiguration: metadata: Optional[PydanticMetadata] = None dsi_package_version: PydanticSemanticVersion = UNKNOWN_VERSION_SENTINEL - @validator("dsi_package_version", always=True) + @field_validator("dsi_package_version") @classmethod def __create_default_dsi_package_version(cls, value: Optional[PydanticSemanticVersion]) -> PydanticSemanticVersion: """Returns the version of the dbt_semantic_interfaces package that generated this manifest.""" diff --git a/dbt_semantic_interfaces/implementations/semantic_model.py b/dbt_semantic_interfaces/implementations/semantic_model.py index 701ad1cd..cc2d6186 100644 --- a/dbt_semantic_interfaces/implementations/semantic_model.py +++ b/dbt_semantic_interfaces/implementations/semantic_model.py @@ -1,8 +1,8 @@ from __future__ import annotations -from typing import Any, List, Optional, Sequence +from typing import List, Optional, Sequence -from pydantic import validator +from pydantic import model_validator from typing_extensions import override from dbt_semantic_interfaces.implementations.base import ( @@ -35,25 +35,15 @@ class NodeRelation(HashableBaseModel): database: Optional[str] = None relation_name: str = "" - @validator("relation_name", always=True) - @classmethod - def __create_default_relation_name(cls, value: Any, values: Any) -> str: # type: ignore[misc] + @model_validator(mode="after") + def __create_default_relation_name(self) -> "NodeRelation": """Dynamically build the dot path for `relation_name`, if not specified.""" - if value: - # Only build the relation_name if it was not present in config. - return value - - alias, schema, database = values.get("alias"), values.get("schema_name"), values.get("database") - if alias is None or schema is None: - raise ValueError( - f"Failed to build relation_name because alias and/or schema was None. schema: {schema}, alias: {alias}" - ) - - if database is not None: - value = f"{database}.{schema}.{alias}" - else: - value = f"{schema}.{alias}" - return value + if not self.relation_name: + if self.database is not None: + self.relation_name = f"{self.database}.{self.schema_name}.{self.alias}" + else: + self.relation_name = f"{self.schema_name}.{self.alias}" + return self @staticmethod def from_string(sql_str: str) -> NodeRelation: # noqa: D From 6e5ca54c19ebf235af9c3f8de4bb0bab820c313b Mon Sep 17 00:00:00 2001 From: Emmanuel Sciara Date: Fri, 24 Nov 2023 22:27:03 +0100 Subject: [PATCH 10/20] fix test_nested_dataclass --- .../dataclass_serialization.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/dbt_semantic_interfaces/dataclass_serialization.py b/dbt_semantic_interfaces/dataclass_serialization.py index 01cda7a4..f85432f3 100644 --- a/dbt_semantic_interfaces/dataclass_serialization.py +++ b/dbt_semantic_interfaces/dataclass_serialization.py @@ -320,13 +320,13 @@ def __init__(self) -> None: # noqa: D def to_pydantic_type(self, dataclass_type: Type[SerializableDataclass]) -> Type[BaseModel]: # noqa: D if dataclass_type not in self._dataclass_type_to_pydantic_type: - self._dataclass_type_to_pydantic_type[ + self._dataclass_type_to_pydantic_type[dataclass_type] = self._convert_dataclass_type_to_pydantic_type( dataclass_type - ] = DataClassTypeToPydanticTypeConverter._convert_dataclass_type_to_pydantic_type(dataclass_type) + ) return self._dataclass_type_to_pydantic_type[dataclass_type] - @staticmethod def _convert_dataclass_type_to_pydantic_type( + self, dataclass_type: Type, ) -> Type[BaseModel]: # noqa: D logger.debug(f"Converting {dataclass_type.__name__} to a pydantic class") @@ -339,7 +339,7 @@ def _convert_dataclass_type_to_pydantic_type( fields_for_pydantic_model: Dict[str, Tuple[Type, AnyValueType]] = {} logger.debug(f"Need to add: {pformat_big_objects(field_dict.keys())}") for field_name, field_definition in field_dict.items(): - field_definition = DataClassTypeToPydanticTypeConverter._convert_nested_fields(field_definition) + field_definition = self._convert_nested_fields(field_definition) fields_for_pydantic_model[field_name] = field_definition.as_pydantic_field_tuple() logger.debug(f"Adding {field_name} with type {field_definition.annotated_field_type}") @@ -352,8 +352,7 @@ def _convert_dataclass_type_to_pydantic_type( logger.debug(f"Finished converting {dataclass_type.__name__} to a pydantic class") return pydantic_model - @staticmethod - def _convert_nested_fields(field_definition: FieldDefinition) -> FieldDefinition: + def _convert_nested_fields(self, field_definition: FieldDefinition) -> FieldDefinition: """Recursively converts a given field definition into a fully serializable type specification. The initial set of FieldDefinitions sourced from the dataclass might contain arbitrarily @@ -365,7 +364,7 @@ def _convert_nested_fields(field_definition: FieldDefinition) -> FieldDefinition raise RuntimeError(f"Unsupported type: {field_definition.annotated_field_type}") elif _is_optional_type(field_definition.annotated_field_type): optional_field_type_parameter = _get_type_parameter_for_optional(field_definition.annotated_field_type) - converted_field_definition = DataClassTypeToPydanticTypeConverter._convert_nested_fields( + converted_field_definition = self._convert_nested_fields( FieldDefinition(field_type=optional_field_type_parameter) ) return FieldDefinition( # type: ignore[arg-type] @@ -376,7 +375,7 @@ def _convert_nested_fields(field_definition: FieldDefinition) -> FieldDefinition tuple_field_type_parameter = _get_type_parameter_for_sequence_like_tuple_type( field_definition.annotated_field_type ) - converted_field_definition = DataClassTypeToPydanticTypeConverter._convert_nested_fields( + converted_field_definition = self._convert_nested_fields( FieldDefinition(field_type=tuple_field_type_parameter) ) return FieldDefinition( @@ -385,9 +384,7 @@ def _convert_nested_fields(field_definition: FieldDefinition) -> FieldDefinition ) elif issubclass(field_definition.annotated_field_type, SerializableDataclass): return FieldDefinition( - field_type=DataClassTypeToPydanticTypeConverter._convert_dataclass_type_to_pydantic_type( - field_definition.annotated_field_type - ), + field_type=self.to_pydantic_type(field_definition.annotated_field_type), default_value=field_definition.default_value, ) else: From 815da61a543c02dfc717e0ecdc8a41aca1d8a5ce Mon Sep 17 00:00:00 2001 From: Emmanuel Sciara Date: Fri, 24 Nov 2023 23:07:18 +0100 Subject: [PATCH 11/20] fix test_interfaces_version_matches --- .../implementations/project_configuration.py | 6 ++- .../implementations/semantic_version.py | 42 ++++++++----------- .../implementations/test_semantic_manifest.py | 2 +- 3 files changed, 22 insertions(+), 28 deletions(-) diff --git a/dbt_semantic_interfaces/implementations/project_configuration.py b/dbt_semantic_interfaces/implementations/project_configuration.py index dd8b94e0..b0a203e0 100644 --- a/dbt_semantic_interfaces/implementations/project_configuration.py +++ b/dbt_semantic_interfaces/implementations/project_configuration.py @@ -3,7 +3,7 @@ from typing import List, Optional from importlib_metadata import version -from pydantic import field_validator +from pydantic import ConfigDict, field_validator from typing_extensions import override from dbt_semantic_interfaces.implementations.base import ( @@ -29,6 +29,8 @@ class PydanticProjectConfiguration(HashableBaseModel, ModelWithMetadataParsing, def _implements_protocol(self) -> ProjectConfiguration: return self + model_config = ConfigDict(validate_default=True) + time_spine_table_configurations: List[PydanticTimeSpineTableConfiguration] metadata: Optional[PydanticMetadata] = None dsi_package_version: PydanticSemanticVersion = UNKNOWN_VERSION_SENTINEL @@ -39,4 +41,4 @@ def __create_default_dsi_package_version(cls, value: Optional[PydanticSemanticVe """Returns the version of the dbt_semantic_interfaces package that generated this manifest.""" if value is not None and value != UNKNOWN_VERSION_SENTINEL: return value - return PydanticSemanticVersion.create_from_string(version("dbt_semantic_interfaces")) + return PydanticSemanticVersion.model_validate(version("dbt_semantic_interfaces")) diff --git a/dbt_semantic_interfaces/implementations/semantic_version.py b/dbt_semantic_interfaces/implementations/semantic_version.py index f8a6d8a4..e19e63ea 100644 --- a/dbt_semantic_interfaces/implementations/semantic_version.py +++ b/dbt_semantic_interfaces/implementations/semantic_version.py @@ -1,44 +1,36 @@ from __future__ import annotations -from typing import Optional +from typing import Any, Dict, Optional -from typing_extensions import override +from pydantic import model_validator -from dbt_semantic_interfaces.implementations.base import ( - HashableBaseModel, - PydanticCustomInputParser, - PydanticParseableValueType, -) +from dbt_semantic_interfaces.implementations.base import HashableBaseModel -class PydanticSemanticVersion(PydanticCustomInputParser, HashableBaseModel): +class PydanticSemanticVersion(HashableBaseModel): """Pydantic implementation of SemanticVersion.""" major_version: str minor_version: str patch_version: Optional[str] = None + @model_validator(mode="before") @classmethod - @override - def _from_yaml_value(cls, input: PydanticParseableValueType) -> PydanticSemanticVersion: + def _create_from_string(cls, input: Any) -> Dict[str, Any]: # noqa: D if isinstance(input, str): - return PydanticSemanticVersion.create_from_string(input) - else: + version_str_split = input.split(".") + if len(version_str_split) < 2: + raise ValueError(f"Expected version string to be of the form x.y or x.y.z, but got {input}") + return { + "major_version": version_str_split[0], + "minor_version": version_str_split[1], + "patch_version": ".".join(version_str_split[2:]) if len(version_str_split) >= 3 else None, + } + elif not isinstance(input, dict): raise ValueError( - f"{cls.__name__} inputs from YAML files are expected to be of either type string or " - f"object (key/value pairs), but got type {type(input)} with value: {input}" + f"Expected input to be of type string or Dict[str, Any], but got type {type(input)} with value: {input}" ) - - @staticmethod - def create_from_string(version_str: str) -> PydanticSemanticVersion: # noqa: D - version_str_split = version_str.split(".") - if len(version_str_split) < 2: - raise ValueError(f"Expected version string to be of the form x.y or x.y.z, but got {version_str}") - return PydanticSemanticVersion( - major_version=version_str_split[0], - minor_version=version_str_split[1], - patch_version=".".join(version_str_split[2:]) if len(version_str_split) >= 3 else None, - ) + return input UNKNOWN_VERSION_SENTINEL = PydanticSemanticVersion(major_version="0", minor_version="0", patch_version="0") diff --git a/tests/implementations/test_semantic_manifest.py b/tests/implementations/test_semantic_manifest.py index b89e98c1..942979c3 100644 --- a/tests/implementations/test_semantic_manifest.py +++ b/tests/implementations/test_semantic_manifest.py @@ -19,6 +19,6 @@ def test_interfaces_version_matches() -> None: # get the actual installed version installed_version = version("dbt_semantic_interfaces") - assert semantic_manifest.project_configuration.dsi_package_version == PydanticSemanticVersion.create_from_string( + assert semantic_manifest.project_configuration.dsi_package_version == PydanticSemanticVersion.model_validate( installed_version ) From 84367f3393062c2250d7cb324154d6a80581c4fd Mon Sep 17 00:00:00 2001 From: Emmanuel Sciara Date: Fri, 24 Nov 2023 23:11:34 +0100 Subject: [PATCH 12/20] Migrate to `model_validate()` from deprecated `parse_obj()` in tests --- .../implementations/filters/where_filter.py | 2 +- tests/parsing/test_where_filter_parsing.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/dbt_semantic_interfaces/implementations/filters/where_filter.py b/dbt_semantic_interfaces/implementations/filters/where_filter.py index 0be414e6..a675f5c5 100644 --- a/dbt_semantic_interfaces/implementations/filters/where_filter.py +++ b/dbt_semantic_interfaces/implementations/filters/where_filter.py @@ -64,7 +64,7 @@ class PydanticWhereFilterIntersection(HashableBaseModel): @classmethod def __get_validators__(cls) -> Generator[Callable[[PydanticParseableValueType], Self], None, None]: - """Pydantic magic method for allowing handling of arbitrary input on parse_obj invocation. + """Pydantic magic method for allowing handling of arbitrary input on model_validate invocation. This class requires more subtle handling of input deserialized object types (dicts), and so it cannot extend the common interface via _from_yaml_values. diff --git a/tests/parsing/test_where_filter_parsing.py b/tests/parsing/test_where_filter_parsing.py index cf741f32..24335876 100644 --- a/tests/parsing/test_where_filter_parsing.py +++ b/tests/parsing/test_where_filter_parsing.py @@ -7,7 +7,7 @@ various conversion operations we will need to perform on semantic manifests defined out in the world. This module tests the various combinations we might encounter in the wild, with a particular focus -on inputs to parse_obj or parse_raw, as that is what the pydantic models will generally encounter. +on inputs to model_validate or parse_raw, as that is what the pydantic models will generally encounter. """ @@ -46,7 +46,7 @@ def test_partially_deserialized_object_string_parsing() -> None: """Tests parsing a where filter specified as a string within partially deserialized json object.""" obj = {"where_filter": __BOOLEAN_EXPRESSION__} - parsed_model = ModelWithWhereFilter.parse_obj(obj) + parsed_model = ModelWithWhereFilter.model_validate(obj) assert parsed_model.where_filter == PydanticWhereFilter(where_sql_template=__BOOLEAN_EXPRESSION__) @@ -55,7 +55,7 @@ def test_partially_deserialized_object_parsing() -> None: """Tests parsing a where filter that was serialized and then json decoded, but not fully parsed.""" obj = {"where_filter": {"where_sql_template": __BOOLEAN_EXPRESSION__}} - parsed_model = ModelWithWhereFilter.parse_obj(obj) + parsed_model = ModelWithWhereFilter.model_validate(obj) assert parsed_model.where_filter == PydanticWhereFilter(where_sql_template=__BOOLEAN_EXPRESSION__) @@ -68,7 +68,7 @@ def test_injected_object_parsing() -> None: """ obj = {"where_filter": PydanticWhereFilter(where_sql_template=__BOOLEAN_EXPRESSION__)} - parsed_model = ModelWithWhereFilter.parse_obj(obj) + parsed_model = ModelWithWhereFilter.model_validate(obj) assert parsed_model.where_filter == PydanticWhereFilter(where_sql_template=__BOOLEAN_EXPRESSION__) @@ -96,7 +96,7 @@ def test_conversion_from_partially_deserialized_where_filter_string() -> None: where_filters=[PydanticWhereFilter(where_sql_template=__BOOLEAN_EXPRESSION__)] ) - parsed_model = ModelWithWhereFilterIntersection.parse_obj(obj) + parsed_model = ModelWithWhereFilterIntersection.model_validate(obj) assert parsed_model.where_filter == expected_conversion_output @@ -108,7 +108,7 @@ def test_conversion_from_partially_deserialized_where_filter_object() -> None: where_filters=[PydanticWhereFilter(where_sql_template=__BOOLEAN_EXPRESSION__)] ) - parsed_model = ModelWithWhereFilterIntersection.parse_obj(obj) + parsed_model = ModelWithWhereFilterIntersection.model_validate(obj) assert parsed_model.where_filter == expected_conversion_output @@ -120,7 +120,7 @@ def test_conversion_from_injected_where_filter_object() -> None: where_filters=[PydanticWhereFilter(where_sql_template=__BOOLEAN_EXPRESSION__)] ) - parsed_model = ModelWithWhereFilterIntersection.parse_obj(obj) + parsed_model = ModelWithWhereFilterIntersection.model_validate(obj) assert parsed_model.where_filter == expected_conversion_output @@ -138,7 +138,7 @@ def test_where_filter_intersection_from_partially_deserialized_list_of_strings() ] ) - parsed_model = ModelWithWhereFilterIntersection.parse_obj(obj) + parsed_model = ModelWithWhereFilterIntersection.model_validate(obj) assert parsed_model.where_filter == expected_parsed_output From 8a918137b1a33221eccc3062cce0cfd0fb3d9e51 Mon Sep 17 00:00:00 2001 From: Emmanuel Sciara Date: Fri, 24 Nov 2023 23:14:42 +0100 Subject: [PATCH 13/20] Migrate to `model_dump_json()` from deprecated `json()` in tests --- tests/parsing/test_where_filter_parsing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parsing/test_where_filter_parsing.py b/tests/parsing/test_where_filter_parsing.py index 24335876..767647bf 100644 --- a/tests/parsing/test_where_filter_parsing.py +++ b/tests/parsing/test_where_filter_parsing.py @@ -80,7 +80,7 @@ def test_serialize_deserialize_operations() -> None: """ base_obj = ModelWithWhereFilter(where_filter=PydanticWhereFilter(where_sql_template=__BOOLEAN_EXPRESSION__)) - serialized = base_obj.json() + serialized = base_obj.model_dump_json() deserialized = ModelWithWhereFilter.parse_raw(serialized) assert deserialized == base_obj From 05a62d857783e1af352ce4d97eb0fc327aca2191 Mon Sep 17 00:00:00 2001 From: Emmanuel Sciara Date: Fri, 24 Nov 2023 23:17:23 +0100 Subject: [PATCH 14/20] Migrate to `model_validate_json()` from deprecated `parse_raw()` in tests --- tests/parsing/test_where_filter_parsing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parsing/test_where_filter_parsing.py b/tests/parsing/test_where_filter_parsing.py index 767647bf..b50838e0 100644 --- a/tests/parsing/test_where_filter_parsing.py +++ b/tests/parsing/test_where_filter_parsing.py @@ -81,7 +81,7 @@ def test_serialize_deserialize_operations() -> None: base_obj = ModelWithWhereFilter(where_filter=PydanticWhereFilter(where_sql_template=__BOOLEAN_EXPRESSION__)) serialized = base_obj.model_dump_json() - deserialized = ModelWithWhereFilter.parse_raw(serialized) + deserialized = ModelWithWhereFilter.model_validate_json(serialized) assert deserialized == base_obj From bd929084112746db4392fb5b0cc673e6c6a439a6 Mon Sep 17 00:00:00 2001 From: Emmanuel Sciara Date: Sat, 25 Nov 2023 00:30:05 +0100 Subject: [PATCH 15/20] Migrate to `model_validator` from deprecated `__get_validators__` --- .../implementations/base.py | 26 +++++++--------- .../implementations/filters/where_filter.py | 31 ++++++------------- .../implementations/metric.py | 30 ++++++++---------- .../validations/metrics.py | 2 +- tests/validations/test_metrics.py | 4 +-- 5 files changed, 38 insertions(+), 55 deletions(-) diff --git a/dbt_semantic_interfaces/implementations/base.py b/dbt_semantic_interfaces/implementations/base.py index 5dfafdb6..ae3887d2 100644 --- a/dbt_semantic_interfaces/implementations/base.py +++ b/dbt_semantic_interfaces/implementations/base.py @@ -3,7 +3,7 @@ import json import os from abc import ABC, abstractmethod -from typing import Any, Callable, ClassVar, Generator, Generic, Type, TypeVar +from typing import Any, ClassVar, Dict, Generic, Type, TypeVar from pydantic import BaseModel, ConfigDict, model_validator @@ -119,23 +119,20 @@ class PydanticCustomInputParser(ABC, Generic[ModelObjectT_co]): and validation of that model object itself. """ + @model_validator(mode="before") @classmethod - def __get_validators__( - cls: Type[PydanticCustomInputParser[ModelObjectT_co]], - ) -> Generator[Callable[[PydanticParseableValueType], PydanticCustomInputParser[ModelObjectT_co]], None, None]: + def _model_validator( + cls: Type[PydanticCustomInputParser[ModelObjectT_co]], input: PydanticParseableValueType + ) -> Dict[str, Any]: """Pydantic magic method for allowing parsing of arbitrary input on validate_model invocation. This allows for parsing and validation prior to object initialization. Most classes implementing this interface in our model are doing so because the input value from user-supplied YAML will be a string representation rather than the structured object type. - """ - yield cls.__parse_with_custom_handling - @classmethod - def __parse_with_custom_handling( - cls: Type[PydanticCustomInputParser[ModelObjectT_co]], input: PydanticParseableValueType - ) -> PydanticCustomInputParser[ModelObjectT_co]: - """Core method for handling common valid - or easily validated - input types. + the previous and next docstrings were from two different methods, which have been combined here. + + Core method for handling common valid - or easily validated - input types. Pydantic objects can commonly appear as JSON object types (from, e.g., deserializing a Pydantic-serialized model) or direct instances of the model object class (from, e.g., initializing an object and passing it in @@ -149,9 +146,10 @@ def __parse_with_custom_handling( to the caller to be pre-validated, and so we do not bother guarding against that here. """ if isinstance(input, dict): - return cls(**input) # type: ignore - elif isinstance(input, cls): return input + elif isinstance(input, cls): + # TODO: find a better way to avoid mypy type ignore + return input.model_dump() # type: ignore[attr-defined] else: return cls._from_yaml_value(input) @@ -159,6 +157,6 @@ def __parse_with_custom_handling( @abstractmethod def _from_yaml_value( cls: Type[PydanticCustomInputParser[ModelObjectT_co]], input: PydanticParseableValueType - ) -> PydanticCustomInputParser[ModelObjectT_co]: + ) -> Dict[str, Any]: """Abstract method for providing object-specific parsing logic.""" raise NotImplementedError() diff --git a/dbt_semantic_interfaces/implementations/filters/where_filter.py b/dbt_semantic_interfaces/implementations/filters/where_filter.py index a675f5c5..19b201e1 100644 --- a/dbt_semantic_interfaces/implementations/filters/where_filter.py +++ b/dbt_semantic_interfaces/implementations/filters/where_filter.py @@ -1,8 +1,8 @@ from __future__ import annotations -from typing import Callable, Generator, List, Tuple +from typing import Any, Dict, List, Tuple -from typing_extensions import Self +from pydantic import model_validator from dbt_semantic_interfaces.call_parameter_sets import ( FilterCallParameterSets, @@ -34,17 +34,14 @@ class PydanticWhereFilter(PydanticCustomInputParser, HashableBaseModel): where_sql_template: str @classmethod - def _from_yaml_value( - cls, - input: PydanticParseableValueType, - ) -> PydanticWhereFilter: + def _from_yaml_value(cls, input: PydanticParseableValueType) -> Dict[str, Any]: """Parses a WhereFilter from a string found in a user-provided model specification. User-provided constraint strings are SQL snippets conforming to the expectations of SQL WHERE clauses, and as such we parse them using our standard parse method below. """ if isinstance(input, str): - return PydanticWhereFilter(where_sql_template=input) + return {"where_sql_template": input} else: raise ValueError(f"Expected input to be of type string, but got type {type(input)} with value: {input}") @@ -62,17 +59,9 @@ class PydanticWhereFilterIntersection(HashableBaseModel): where_filters: List[PydanticWhereFilter] + @model_validator(mode="before") @classmethod - def __get_validators__(cls) -> Generator[Callable[[PydanticParseableValueType], Self], None, None]: - """Pydantic magic method for allowing handling of arbitrary input on model_validate invocation. - - This class requires more subtle handling of input deserialized object types (dicts), and so it cannot - extend the common interface via _from_yaml_values. - """ - yield cls._convert_legacy_and_yaml_input - - @classmethod - def _convert_legacy_and_yaml_input(cls, input: PydanticParseableValueType) -> Self: + def _convert_legacy_and_yaml_input(cls, input: PydanticParseableValueType) -> Dict[str, Any]: """Specifies raw input conversion rules to ensure serialized semantic manifests will parse correctly. The original spec for where filters relied on a raw WhereFilter object, but this has now been updated to @@ -101,13 +90,13 @@ def _convert_legacy_and_yaml_input(cls, input: PydanticParseableValueType) -> Se is_legacy_where_filter = isinstance(input, str) or isinstance(input, PydanticWhereFilter) or has_legacy_keys if is_legacy_where_filter: - return cls(where_filters=[input]) + return {"where_filters": [input]} elif isinstance(input, list): - return cls(where_filters=input) + return {"where_filters": input} elif isinstance(input, dict): - return cls(**input) - elif isinstance(input, cls): return input + elif isinstance(input, cls): + return input.model_dump() else: raise ValueError( f"Expected input to be of type string, list, PydanticWhereFilter, PydanticWhereFilterIntersection, " diff --git a/dbt_semantic_interfaces/implementations/metric.py b/dbt_semantic_interfaces/implementations/metric.py index e4bbcb67..ca3ca5d0 100644 --- a/dbt_semantic_interfaces/implementations/metric.py +++ b/dbt_semantic_interfaces/implementations/metric.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import List, Optional, Sequence +from typing import Any, Dict, List, Optional, Sequence from pydantic import Field @@ -38,7 +38,7 @@ class PydanticMetricInputMeasure(PydanticCustomInputParser, HashableBaseModel): fill_nulls_with: Optional[int] = None @classmethod - def _from_yaml_value(cls, input: PydanticParseableValueType) -> PydanticMetricInputMeasure: + def _from_yaml_value(cls, input: PydanticParseableValueType) -> Dict[str, Any]: """Parses a MetricInputMeasure from a string (name only) or object (struct spec) input. For user input cases, the original YAML spec for a PydanticMetric included measure(s) specified as string names @@ -46,7 +46,7 @@ def _from_yaml_value(cls, input: PydanticParseableValueType) -> PydanticMetricIn base name for this object. """ if isinstance(input, str): - return PydanticMetricInputMeasure(name=input) + return {"name": input} else: raise ValueError( f"MetricInputMeasure inputs from model configs are expected to be of either type string or " @@ -71,26 +71,22 @@ class PydanticMetricTimeWindow(PydanticCustomInputParser, HashableBaseModel): granularity: TimeGranularity @classmethod - def _from_yaml_value(cls, input: PydanticParseableValueType) -> PydanticMetricTimeWindow: + def _from_yaml_value(cls, input: PydanticParseableValueType) -> Dict[str, Any]: """Parses a MetricTimeWindow from a string input found in a user provided model specification. The MetricTimeWindow is always expected to be provided as a string in user-defined YAML configs. + + Output of the form: (