From 04623f1e469026370ca2135662aafd9d2e969413 Mon Sep 17 00:00:00 2001 From: JB Lovland Date: Tue, 9 Jan 2024 17:53:37 +0100 Subject: [PATCH 01/25] experimental: Build pydantic from schema --- foo.py | 84 +++++++ gen.sh | 20 ++ pyproject.toml | 1 + src/fmu/dataio/model.py | 501 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 606 insertions(+) create mode 100644 foo.py create mode 100755 gen.sh create mode 100644 src/fmu/dataio/model.py diff --git a/foo.py b/foo.py new file mode 100644 index 000000000..b98133664 --- /dev/null +++ b/foo.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +from pprint import pp +from typing import Literal +from uuid import UUID + +from pydantic import BaseModel, Field +from yaml import safe_load + + +class SMDAAttribute(BaseModel): + short_identifier: str | None = None + identifier: str | None = None + uuid: UUID = Field(title="...", description="....") + + +class SMDA(BaseModel): + country: list[SMDAAttribute] + discovery: list[SMDAAttribute] + field: list[SMDAAttribute] + coordinate_system: SMDAAttribute + stratigraphic_column: SMDAAttribute + + +class Masterdata(BaseModel): + smda: SMDA + + +class Asset(BaseModel): + name: str + + +class SSDL(BaseModel): + access_level: Literal["internal", "external"] + rep_include: bool + + +class Access(BaseModel): + asset: Asset + ssdl: SSDL + + +class Model(BaseModel): + name: str + revision: str = Field(pattern=r"^\d+\.\d+.\d+\.(dev|prod)$") + + +class StratigraphyAttribute(BaseModel): + stratigraphic: bool + name: str + alias: list[str] = [] + stratigraphic_alias: list[str] = [] + + +class RMS(BaseModel): + horizons: dict[str, list[str]] + zones: dict[str, list[str]] + + +class Root(BaseModel): + masterdata: Masterdata + access: Access + model: Model + stratigraphy: dict[str, StratigraphyAttribute] + global_: dict[str, str | float | int] = Field(alias="global") + rms: RMS + + +pp( + safe_load( + open( + "examples/s/d/nn/xcase/realization-1/iter-0/fmuconfig/output/global_variables.yml" + ) + ) +) + +m = Root.model_validate( + safe_load( + open( + "examples/s/d/nn/xcase/realization-1/iter-0/fmuconfig/output/global_variables.yml" + ) + ) +) +pp(m.rms) diff --git a/gen.sh b/gen.sh new file mode 100755 index 000000000..913e96366 --- /dev/null +++ b/gen.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +datamodel-codegen \ + --disable-timestamp \ + --enable-version-header \ + --enum-field-as-literal one \ + --field-constraints \ + --input $1 \ + --input-file-type "jsonschema" \ + --output-model-type pydantic_v2.BaseModel \ + --snake-case-field \ + --strict-nullable \ + --target-python-version 3.8 \ + --use-default-kwarg \ + --use-double-quotes \ + --use-schema-description \ + --use-standard-collections \ + --use-subclass-enum \ + --use-title-as-name \ + --use-union-operator diff --git a/pyproject.toml b/pyproject.toml index 7972c0faf..f00ef65df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ dependencies = [ "numpy", "pandas", "pyarrow", + "pydantic>=2.0.0", "PyYAML", "xtgeo>=2.16", ] diff --git a/src/fmu/dataio/model.py b/src/fmu/dataio/model.py new file mode 100644 index 000000000..1bdfb46b8 --- /dev/null +++ b/src/fmu/dataio/model.py @@ -0,0 +1,501 @@ +# generated by datamodel-codegen: +# filename: fmu_results_090.json +# version: 0.25.2 + +from __future__ import annotations + +from enum import Enum +from typing import Any, Literal + +from pydantic import BaseModel, Field, RootModel + + +class Generic(RootModel[Any]): + root: Any + + +class MetadataClass(str, Enum): + case = "case" + surface = "surface" + table = "table" + cpgrid = "cpgrid" + cpgrid_property = "cpgrid_property" + polygons = "polygons" + cube = "cube" + well = "well" + points = "points" + + +class Source(RootModel[Literal["fmu"]]): + root: Literal["fmu"] = Field(..., description="Data source (FMU)") + + +class FMUResultsMetadataVersion(RootModel[Literal["0.9.0"]]): + root: Literal["0.9.0"] = Field( + ..., examples=["1.2.3"], title="FMU results metadata version" + ) + + +class Property(BaseModel): + name: str | None = Field(default=None, examples=["MyPropertyName"]) + attribute: str | None = Field(default=None, examples=["MyAttributeName"]) + is_discrete: bool | None = Field(default=None, examples=[True, False]) + + +class VerticalDomain(str, Enum): + depth = "depth" + time = "time" + + +class DepthReference(Enum): + msl = "msl" + sb = "sb" + rkb = "rkb" + none_type_none = None + + +class GridModel(BaseModel): + name: str | None = Field(default=None, examples=["MyGrid"]) + + +class Yflip(int, Enum): + integer_0 = 0 + integer_1 = 1 + + +class Spec(BaseModel): + ncol: int | None = Field(default=None, examples=[281]) + nrow: int | None = Field(default=None, examples=[441]) + nlay: int | None = Field(default=None, examples=[333]) + xori: float | None = Field(default=None, examples=[461499.9997558594]) + yori: float | None = Field(default=None, examples=[461499.9997558594]) + xinc: float | None = Field(default=None, examples=[25]) + yflip: Yflip | None = Field(default=None, examples=[1]) + rotation: float | None = Field(default=None, examples=["30.00000000231"]) + undef: float | None = Field(default=None, examples=["1e+33"]) + npolys: int | None = Field( + default=None, + description="The number of individual polygons in the data object", + examples=[1], + ) + + +class Bbox(BaseModel): + xmin: float = Field(..., examples=[456012.5003497944]) + xmax: float = Field(..., examples=[467540.52762886323]) + ymin: float = Field(..., examples=[5926499.999511719]) + ymax: float = Field(..., examples=[5939492.128326312]) + zmin: float | None = Field(default=None, examples=[1244.039, None]) + zmax: float | None = Field(default=None, examples=[2302.683, None]) + + +class Line(BaseModel): + show: bool | None = Field(default=None, examples=[True]) + color: str | None = Field(default=None, examples=["black", "#000000"]) + + +class Points(BaseModel): + show: bool | None = Field(default=None, examples=[True]) + color: str | None = Field(default=None, examples=["black", "#000000"]) + + +class Contours(BaseModel): + show: bool | None = Field(default=None, examples=[True]) + color: str | None = Field( + default=None, + description="The color of the contour lines", + examples=["black", "#000000"], + ) + increment: float | None = Field( + default=None, + description="The contour increment in same values as the data", + examples=[20], + ) + + +class Fill(BaseModel): + show: bool | None = Field(default=None, examples=[True]) + color: str | None = Field(default=None, examples=["black", "#000000"]) + colormap: str | None = Field( + default=None, + description="named reference to a colormap", + examples=["gist_earth"], + ) + display_min: float | None = Field( + default=None, + description="The low-side boundary to use for fill color", + examples=[1000], + ) + display_max: float | None = Field( + default=None, + description="The high-side boundary to use for fill color", + examples=[1600], + ) + + +class Display(BaseModel): + name: str | None = Field(default=None, examples=["Top Volantis"]) + subtitle: str | None = Field(default=None, examples=["Some subtitle"]) + line: Line | None = None + points: Points | None = None + contours: Contours | None = None + fill: Fill | None = None + + +class Asset(BaseModel): + name: str = Field(..., examples=["Drogon"]) + + +class Access(BaseModel): + asset: Asset | None = None + + +class Workflow1(BaseModel): + reference: str | None = Field( + default=None, + description="Reference to the part of the FMU workflow that produced this", + ) + + +class Aggregation1(BaseModel): + operation: str = Field(..., description="The aggregation performed") + realization_ids: list[int] = Field( + ..., description="Array of realization ids included in this aggregation" + ) + parameters: dict[str, Any] | None = Field( + default=None, description="Parameters for this realization" + ) + id: str | None = Field( + default=None, + description="The ID of this aggregation", + examples=["15ce3b84-766f-4c93-9050-b154861f9100"], + ) + + +class File(BaseModel): + """ + Block describing the file as the data appear in FMU context + """ + + relative_path: str = Field( + ..., + description="The file path relative to RUNPATH", + examples=["share/results/maps/volantis_gp_base--depth.gri"], + ) + absolute_path: str | None = Field( + default=None, + description="The absolute file path", + examples=["/abs/path/share/results/maps/volantis_gp_base--depth.gri"], + ) + checksum_md5: str = Field( + ..., + description="md5 checksum of the file or bytestring", + examples=["kjhsdfvsdlfk23knerknvk23"], + ) + + +class Name(RootModel[str]): + root: str = Field( + ..., + description="Name of the surface. If stratigraphic, match the entry in the stratigraphic column", + examples=["VIKING GP. Top"], + ) + + +class Offset(RootModel[float]): + root: float = Field(..., examples=[11.2]) + + +class Stratigraphic(RootModel[bool]): + root: bool = Field( + ..., + description="True if surface represents an entity in the stratigraphic column", + examples=[True], + ) + + +class Aggregation(RootModel[Any]): + root: Any + + +class Case(RootModel[Any]): + root: Any + + +class Iteration(RootModel[Any]): + root: Any + + +class Model(RootModel[Any]): + root: Any + + +class Realization(RootModel[Any]): + root: Any + + +class Workflow(RootModel[Any]): + root: Any + + +class FieldDescription(RootModel[list[str]]): + root: list[str] + + +class Datetime(RootModel[str]): + root: str = Field(..., examples=["2020-10-28T14:28:02"]) + + +class User(BaseModel): + id: str = Field(..., examples=["peesv"], title="User ID") + + +class Uuid(RootModel[str]): + root: str = Field( + ..., + examples=["ad214d85-8a1d-19da-e053-c918a4889309"], + pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", + ) + + +class Fmu(BaseModel): + """ + The FMU block records properties that are specific to FMU + """ + + model: Model + case: Case + + +class Fmu1(BaseModel): + """ + The FMU block records properties that are specific to FMU + """ + + model: Model + case: Case + iteration: Iteration + workflow: Workflow + realization: Realization + + +class Fmu2(BaseModel): + """ + The FMU block records properties that are specific to FMU + """ + + model: Model + case: Case + iteration: Iteration + workflow: Workflow + aggregation: Aggregation + + +class FMUTimeObjectItem(BaseModel): + value: Datetime | None = None + label: str | None = Field(default=None, examples=["Label for time stamp"]) + + +class FMUTimeObject(RootModel[list[FMUTimeObjectItem]]): + """ + List of time stamps referring to simulated time + """ + + root: list[FMUTimeObjectItem] = Field( + ..., + description="List of time stamps referring to simulated time", + min_length=1, + title="FMU time object", + ) + + +class TracklogEvent(BaseModel): + datetime: Datetime | None = None + user: User | None = None + event: str | None = Field(default=None, examples=["created", "updated"]) + + +class Tracklog(RootModel[list[TracklogEvent]]): + root: list[TracklogEvent] + + +class Top(BaseModel): + name: Name | None = None + stratigraphic: Stratigraphic | None = None + offset: Offset | None = None + + +class Base(BaseModel): + name: Name | None = None + stratigraphic: Stratigraphic | None = None + offset: Offset | None = None + + +class TheDataBlock(BaseModel): + name: str = Field( + ..., + description="Name of the surface. If stratigraphic, match the entry in the stratigraphic column", + examples=["VIKING GP. Top"], + ) + stratigraphic: bool = Field( + ..., + description="True if surface represents an entity in the stratigraphic column", + examples=[True], + ) + alias: list[Name] | None = None + stratigraphic_alias: list[Name] | None = None + offset: float | None = Field(default=None, examples=[11.2]) + top: Top | None = None + base: Base | None = None + content: str = Field( + ..., description="The contents of this data object", examples=["depth"] + ) + tagname: str | None = Field( + default=None, + description="A semi-human readable tag for internal usage and uniqueness", + examples=["ds_extract_geogrid", "ds_post_strucmod"], + ) + properties: list[Property] | None = None + format: str = Field(..., examples=["irap_binary"]) + layout: str | None = Field(default=None, examples=["regular"]) + unit: str | None = Field(default=None, examples=["m"]) + vertical_domain: VerticalDomain | None = Field(default=None, examples=["depth"]) + depth_reference: DepthReference | None = Field(default=None, examples=["msl"]) + grid_model: GridModel | None = None + spec: Spec + bbox: Bbox | None = None + time: FMUTimeObject | None = None + is_prediction: bool = Field(..., examples=[True], title="Is prediction flag") + is_observation: bool = Field(..., examples=[True], title="Is observation flag") + description: FieldDescription | None = None + + +class CountryItem(BaseModel): + identifier: str = Field(..., examples=["Norway"]) + uuid: Uuid + + +class DiscoveryItem(BaseModel): + short_identifier: str = Field(..., examples=["SomeDiscovery"]) + uuid: Uuid + + +class FieldItem(BaseModel): + identifier: str = Field(..., examples=["OseFax"]) + uuid: Uuid + + +class CoordinateSystem(BaseModel): + identifier: str = Field(..., examples=["ST_WGS84_UTM37N_P32637"]) + uuid: Uuid + + +class StratigraphicColumn(BaseModel): + identifier: str = Field(..., examples=["DROGON_2020"]) + uuid: Uuid + + +class Smda(BaseModel): + country: list[CountryItem] + discovery: list[DiscoveryItem] + field: list[FieldItem] + coordinate_system: CoordinateSystem + stratigraphic_column: StratigraphicColumn + + +class Masterdata(BaseModel): + smda: Smda + + +class Model1(BaseModel): + name: str | None = Field(default=None, examples=["Drogon"]) + revision: str | None = Field(default=None, examples=["21.0.0.dev"]) + description: FieldDescription | None = Field( + default=None, description="This is a free text description of the model setup" + ) + + +class Case1(BaseModel): + name: str = Field(..., description="The case name", examples=["MyCaseName"]) + uuid: Uuid + user: User = Field(..., description="The user name used in ERT") + restart_from: str | None = Field( + default=None, + description="A reference to another case/iteration that this iteration was restarted from", + ) + description: FieldDescription | None = None + + +class Iteration1(BaseModel): + name: str = Field( + ..., + description="The convential name of this iteration, e.g. iter-0 or pred", + examples=["iter-0"], + ) + id: int = Field( + ..., + description="The internal identification of this iteration, e.g. the iteration number", + examples=[0], + ) + uuid: Uuid + + +class Realization1(BaseModel): + name: str = Field( + ..., + description="The convential name of this iteration, e.g. iter-0 or pred", + examples=["iter-0"], + ) + id: int = Field( + ..., + description="The unique number of this realization as used in FMU", + examples=[33], + ) + uuid: Uuid + parameters: dict[str, Any] | None = Field( + default=None, description="Parameters for this realization" + ) + jobs: dict[str, Any] | None = Field( + default=None, + description="Content directly taken from the ERT jobs.json file for this realization", + ) + + +class Fmu3(BaseModel): + model: Model1 | None = None + workflow: Workflow1 | None = None + case: Case1 | None = None + iteration: Iteration1 | None = None + realization: Realization1 | None = None + aggregation: Aggregation1 | None = None + + +class CaseModel(BaseModel): + source: Source + version: FMUResultsMetadataVersion + data: TheDataBlock | None = None + tracklog: Tracklog + access: Access + masterdata: Masterdata + class_: MetadataClass = Field(..., alias="class") + fmu: Fmu = Field( + ..., description="The FMU block records properties that are specific to FMU" + ) + + +class DataObject(BaseModel): + source: Source + version: FMUResultsMetadataVersion + data: TheDataBlock + tracklog: Tracklog + access: Access + masterdata: Masterdata + class_: MetadataClass = Field(..., alias="class") + fmu: Fmu1 | Fmu2 = Field( + ..., description="The FMU block records properties that are specific to FMU" + ) + file: File + + +class ModelModel(RootModel[CaseModel | DataObject]): + root: CaseModel | DataObject From efd9536c390e52f591b44e0f6a602171bb3aed5c Mon Sep 17 00:00:00 2001 From: JB Lovland Date: Mon, 15 Jan 2024 09:55:25 +0100 Subject: [PATCH 02/25] Yarr.. --- .../fmuconfig/output/global_variables.yml | 2 +- foo.py | 28 ++--- schema/docs/src/public/fmu_results_080.json | 23 +--- src/fmu/dataio/model.py | 111 ++++++++++++++++-- 4 files changed, 120 insertions(+), 44 deletions(-) diff --git a/examples/s/d/nn/xcase/realization-1/iter-0/fmuconfig/output/global_variables.yml b/examples/s/d/nn/xcase/realization-1/iter-0/fmuconfig/output/global_variables.yml index 91e801c69..5aecd2e6b 100644 --- a/examples/s/d/nn/xcase/realization-1/iter-0/fmuconfig/output/global_variables.yml +++ b/examples/s/d/nn/xcase/realization-1/iter-0/fmuconfig/output/global_variables.yml @@ -17,7 +17,7 @@ masterdata: uuid: ad214d85-dac7-19da-e053-c918a4889309 stratigraphic_column: identifier: DROGON_2020 - uuid: some-unique-id-to-be-provided-by-smda + uuid: ad214d85-dac7-19da-e053-c918a4889308 access: asset: name: Drogon diff --git a/foo.py b/foo.py index b98133664..314a09599 100644 --- a/foo.py +++ b/foo.py @@ -66,19 +66,15 @@ class Root(BaseModel): rms: RMS -pp( - safe_load( - open( - "examples/s/d/nn/xcase/realization-1/iter-0/fmuconfig/output/global_variables.yml" - ) - ) -) - -m = Root.model_validate( - safe_load( - open( - "examples/s/d/nn/xcase/realization-1/iter-0/fmuconfig/output/global_variables.yml" - ) - ) -) -pp(m.rms) +with open( + "examples/s/d/nn/xcase/realization-1/iter-0/fmuconfig/output/global_variables.yml" +) as f: + pp(safe_load(f)) + +print("-" * 100) + +with open( + "examples/s/d/nn/xcase/realization-1/iter-0/fmuconfig/output/global_variables.yml" +) as f: + m = Root.model_validate(safe_load(f)) + print(m.rms) diff --git a/schema/docs/src/public/fmu_results_080.json b/schema/docs/src/public/fmu_results_080.json index 5c137fcc6..2491378b1 100644 --- a/schema/docs/src/public/fmu_results_080.json +++ b/schema/docs/src/public/fmu_results_080.json @@ -129,10 +129,9 @@ "const": "fmu" }, "version": { - "$comment": "Must always be on root level.", + "$comment": "Must always be on root level. This is 0.8.0, so require version to be == 0.8.0", "title": "FMU results metadata version", "type": "string", - "$comment": "This is 0.8.0, so require version to be == 0.8.0", "enum": [ "0.8.0" ], @@ -215,8 +214,7 @@ } }, "offset": { - "$comment": "To be used if data represents an offset from the specified name", - "$comment": "e.g. a surface representing an offset from a known horizon", + "$comment": "To be used if data represents an offset from the specified name e.g. a surface representing an offset from a known horizon", "type": "number", "example": 11.2 }, @@ -1097,11 +1095,7 @@ "id": { "type": "string", "description": "The ID of this aggregation", - "$comment": "Used for tying together representations of the same entity", - "$comment": "Example: Statistics for various time steps of the same surface", - "$comment": "Not used in Drogon but widely used on e.g. Johan Sverdrup", - "$comment": "Used for enabling common display and joint visual settings", - "$comment": "A unique uuid is currently used, and included as example but any ID should be sufficient", + "$comment": "Used for tying together representations of the same entity Example: Statistics for various time steps of the same surface Not used in Drogon but widely used on e.g. Johan Sverdrup Used for enabling common display and joint visual settings. A unique uuid is currently used, and included as example but any ID should be sufficient", "examples": [ "15ce3b84-766f-4c93-9050-b154861f9100" ] @@ -1128,8 +1122,7 @@ }, "file": { "description": "Block describing the file as the data appear in FMU context", - "$comment": "While the file may seize to exist, the concept is still useful", - "$comment": "A lot of logic exists on top of the file name now, legacy but not likely to disappear soon", + "$comment": "While the file may seize to exist, the concept is still useful. A lot of logic exists on top of the file name now, legacy but not likely to disappear soon", "type": "object", "required": [ "relative_path", @@ -1167,10 +1160,7 @@ } } }, - "$comment": "DEFINITIONS END HERE", - "$comment": "oneOf below reflects different structures depending on type", - "$comment": "The fmu-block is different for cases and data objects,", - "$comment": "and the data block is only present for data objects.", + "$comment": "DEFINITIONS END HERE oneOf below reflects different structures depending on type The fmu-block is different for cases and data objects, and the data block is only present for data objects.", "required": [ "source", "version", @@ -1252,7 +1242,7 @@ }, "fmu": { "description": "The FMU block records properties that are specific to FMU", - "$comment": "This is the fmu block as it appears in data objects", + "$comment": "This is the fmu block as it appears in data objects implementation below allows realization or aggregation or none of them, never both", "required": [ "model", "case" @@ -1277,7 +1267,6 @@ "$ref": "#/definitions/fmu/aggregation" } }, - "$comment": "implementation below allows realization or aggregation or none of them, never both", "dependencies": { "aggregation": { "not": { diff --git a/src/fmu/dataio/model.py b/src/fmu/dataio/model.py index 1bdfb46b8..f1bc42f4e 100644 --- a/src/fmu/dataio/model.py +++ b/src/fmu/dataio/model.py @@ -5,9 +5,10 @@ from __future__ import annotations from enum import Enum -from typing import Any, Literal +from typing import Any, List, Literal, Union from pydantic import BaseModel, Field, RootModel +from typing_extensions import Annotated class Generic(RootModel[Any]): @@ -197,7 +198,10 @@ class File(BaseModel): class Name(RootModel[str]): root: str = Field( ..., - description="Name of the surface. If stratigraphic, match the entry in the stratigraphic column", + description=( + "Name of the surface. If stratigraphic, match " + "the entry in the stratigraphic column" + ), examples=["VIKING GP. Top"], ) @@ -238,7 +242,7 @@ class Workflow(RootModel[Any]): root: Any -class FieldDescription(RootModel[list[str]]): +class FieldDescription(RootModel[List[str]]): root: list[str] @@ -296,7 +300,7 @@ class FMUTimeObjectItem(BaseModel): label: str | None = Field(default=None, examples=["Label for time stamp"]) -class FMUTimeObject(RootModel[list[FMUTimeObjectItem]]): +class FMUTimeObject(RootModel[List[FMUTimeObjectItem]]): """ List of time stamps referring to simulated time """ @@ -315,7 +319,7 @@ class TracklogEvent(BaseModel): event: str | None = Field(default=None, examples=["created", "updated"]) -class Tracklog(RootModel[list[TracklogEvent]]): +class Tracklog(RootModel[List[TracklogEvent]]): root: list[TracklogEvent] @@ -334,7 +338,10 @@ class Base(BaseModel): class TheDataBlock(BaseModel): name: str = Field( ..., - description="Name of the surface. If stratigraphic, match the entry in the stratigraphic column", + description=( + "Name of the surface. If stratigraphic, match the entry " + "in the stratigraphic column" + ), examples=["VIKING GP. Top"], ) stratigraphic: bool = Field( @@ -421,7 +428,10 @@ class Case1(BaseModel): user: User = Field(..., description="The user name used in ERT") restart_from: str | None = Field( default=None, - description="A reference to another case/iteration that this iteration was restarted from", + description=( + "A reference to another case/iteration that this " + "iteration was restarted from" + ), ) description: FieldDescription | None = None @@ -434,7 +444,10 @@ class Iteration1(BaseModel): ) id: int = Field( ..., - description="The internal identification of this iteration, e.g. the iteration number", + description=( + "The internal identification of this " + "iteration, e.g. the iteration number" + ), examples=[0], ) uuid: Uuid @@ -457,7 +470,9 @@ class Realization1(BaseModel): ) jobs: dict[str, Any] | None = Field( default=None, - description="Content directly taken from the ERT jobs.json file for this realization", + description=( + "Content directly taken from the ERT jobs.json " "file for this realization" + ), ) @@ -497,5 +512,81 @@ class DataObject(BaseModel): file: File -class ModelModel(RootModel[CaseModel | DataObject]): +class ModelModel(RootModel[Union[CaseModel, DataObject]]): root: CaseModel | DataObject + + +### V3? + + +class GeologicalModel(BaseModel): + type: Literal["Structural", "Rock"] + + +class RockGeologicalModel(GeologicalModel): + type: Literal["Rock"] = "Rock" + + +class StructuralGeologicalModel(GeologicalModel): + type: Literal["Structural"] = "Structural" + + +class Shape(BaseModel): + ncol: int = Field(ge=0) + nrow: int = Field(ge=0) + nlay: int = Field(ge=0) + + +class Orientation(BaseModel): + x: float + y: float + z: float + + +class Grid(BaseModel): + coordinate_system: CoordinateSystem + orientation: Orientation + shape: Shape + undef: float | None + + +class Range(BaseModel): + start: float + stop: float + + +class BoundingBox(BaseModel): + x: Range + y: Range + z: Range + + +class Header(BaseModel): + asset: str + created_at: Datetime + created_by: str + version: int + + +class Payland(BaseModel): + type: Literal["fmu.everest", "fmu.ert"] + + +class FMUEverest(Payland): + type: Literal["fmu.everest"] = "fmu.everest" + + +class FMUErt(BaseModel): + type: Literal["fmu.ert"] = "fmu.ert" + model: Annotated[ + StructuralGeologicalModel | RockGeologicalModel, + Field(discriminator="type"), + ] + + +class Export(BaseModel): + header: Header + payload: Annotated[ + FMUEverest | FMUErt, + Field(discriminator="type"), + ] From 452d6fa6f67370ccc7a95991edc34578ec1fedff Mon Sep 17 00:00:00 2001 From: JB Lovland Date: Tue, 16 Jan 2024 16:17:43 +0100 Subject: [PATCH 03/25] Use json --- gen.sh | 3 +- .../definitions/0.9.0/schema/fmu_results.json | 1418 +++++++++++++++++ src/fmu/dataio/model.py | 592 ------- src/fmu/dataio/models/080model.py | 1332 ++++++++++++++++ src/fmu/dataio/models/090model.py | 1379 ++++++++++++++++ src/fmu/dataio/models/v3.py | 79 + 6 files changed, 4209 insertions(+), 594 deletions(-) create mode 100644 schema/definitions/0.9.0/schema/fmu_results.json delete mode 100644 src/fmu/dataio/model.py create mode 100644 src/fmu/dataio/models/080model.py create mode 100644 src/fmu/dataio/models/090model.py create mode 100644 src/fmu/dataio/models/v3.py diff --git a/gen.sh b/gen.sh index 913e96366..a8c618711 100755 --- a/gen.sh +++ b/gen.sh @@ -3,10 +3,9 @@ datamodel-codegen \ --disable-timestamp \ --enable-version-header \ - --enum-field-as-literal one \ --field-constraints \ --input $1 \ - --input-file-type "jsonschema" \ + --input-file-type "json" \ --output-model-type pydantic_v2.BaseModel \ --snake-case-field \ --strict-nullable \ diff --git a/schema/definitions/0.9.0/schema/fmu_results.json b/schema/definitions/0.9.0/schema/fmu_results.json new file mode 100644 index 000000000..36569bd02 --- /dev/null +++ b/schema/definitions/0.9.0/schema/fmu_results.json @@ -0,0 +1,1418 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "fmu_results.json", + "$contractual": [ + "class", + "source", + "version", + "tracklog", + "data.format", + "data.name", + "data.stratigraphic", + "data.alias", + "data.stratigraphic_alias", + "data.offset", + "data.content", + "data.tagname", + "data.vertical_domain", + "data.grid_model", + "data.bbox", + "data.time", + "data.is_prediction", + "data.is_observation", + "data.seismic.attribute", + "data.spec.columns", + "access", + "masterdata", + "fmu.model", + "fmu.workflow", + "fmu.case", + "fmu.iteration.name", + "fmu.iteration.uuid", + "fmu.realization.name", + "fmu.realization.id", + "fmu.realization.uuid", + "fmu.aggregation.operation", + "fmu.aggregation.realization_ids", + "fmu.context.stage", + "file.relative_path", + "file.checksum_md5", + "file.size_bytes" + ], + "$defs": { + "properties": { + "generic": { + "$comment": "GENERIC DEFINITIONS", + "datetime": { + "type": "string", + "examples": [ + "2020-10-28T14:28:02" + ] + }, + "uuid": { + "type": "string", + "pattern": "^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", + "examples": [ + "ad214d85-8a1d-19da-e053-c918a4889309" + ] + }, + "user": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "title": "User ID", + "type": "string", + "examples": [ + "peesv" + ] + } + } + }, + "_description": { + "$comment": "Underscore to keep JSON Schema validator functioning", + "type": "array", + "items": { + "type": "string", + "examples": [ + "Some description" + ] + } + } + }, + "fmu_time": { + "title": "FMU time object", + "description": "Time stamp for data object.", + "type": [ + "object", + "null" + ], + "properties": { + "value": { + "$ref": "#/$defs/properties/generic/datetime" + }, + "label": { + "type": "string", + "examples": [ + "base", + "monitor", + "mylabel" + ] + } + } + }, + "class": { + "title": "Metadata class", + "type": "string", + "examples": [ + "surface", + "table", + "points" + ], + "enum": [ + "case", + "surface", + "table", + "cpgrid", + "cpgrid_property", + "polygons", + "cube", + "well", + "points", + "dictionary" + ] + }, + "source": { + "$comment": "Fixed field, only valid value is fmu.", + "description": "Data source (FMU)", + "type": "string", + "const": "fmu" + }, + "version": { + "$comment": "Must always be on root level.", + "title": "FMU results metadata version", + "type": "string", + "$comment": "This is 0.9.0, so require version to be == 0.9.0", + "enum": [ + "0.9.0" + ], + "example": "1.2.3" + }, + "tracklog_event": { + "type": "object", + "properties": { + "datetime": { + "$ref": "#/$defs/properties/generic/datetime" + }, + "user": { + "$ref": "#/$defs/properties/generic/user" + }, + "event": { + "type": "string", + "examples": [ + "created", + "updated" + ] + } + } + }, + "tracklog": { + "type": "array", + "items": { + "$ref": "#/$defs/properties/tracklog_event" + } + }, + "data": { + "type": "object", + "title": "The data block", + "required": [ + "content", + "name", + "format", + "stratigraphic", + "is_prediction", + "is_observation" + ], + "dependencies": { + "top": { + "required": [ + "base" + ] + }, + "base": { + "required": [ + "top" + ] + } + }, + "properties": { + "name": { + "type": "string", + "description": "Name of the data object. If stratigraphic, match the entry in the stratigraphic column", + "examples": [ + "VIKING GP. Top" + ] + }, + "stratigraphic": { + "$comment": "Determines of validation against strat column should happen or not", + "type": "boolean", + "description": "True if data object represents an entity in the stratigraphic column", + "examples": [ + true + ] + }, + "alias": { + "type": "array", + "items": { + "$ref": "#/$defs/properties/data/properties/name" + } + }, + "stratigraphic_alias": { + "type": "array", + "items": { + "$ref": "#/$defs/properties/data/properties/name" + } + }, + "offset": { + "$comment": "To be used if data represents an offset from the specified name", + "$comment": "e.g. a surface representing an offset from a known horizon", + "type": "number", + "example": 11.2 + }, + "top": { + "$comment": "To be used if data object is an interval", + "properties": { + "name": { + "$ref": "#/$defs/properties/data/properties/name" + }, + "stratigraphic": { + "$ref": "#/$defs/properties/data/properties/stratigraphic" + }, + "offset": { + "$ref": "#/$defs/properties/data/properties/offset" + } + } + }, + "base": { + "$comment": "To be used if data object is an interval", + "properties": { + "name": { + "$ref": "#/$defs/properties/data/properties/name" + }, + "stratigraphic": { + "$ref": "#/$defs/properties/data/properties/stratigraphic" + }, + "offset": { + "$ref": "#/$defs/properties/data/properties/offset" + } + } + }, + "content": { + "type": "string", + "description": "The contents of this data object", + "examples": [ + "depth" + ] + }, + "tagname": { + "type": "string", + "description": "A semi-human readable tag for internal usage and uniqueness", + "examples": [ + "ds_extract_geogrid", + "ds_post_strucmod" + ] + }, + "properties": { + "$comment": "This is not the JSON Schema properties, this is FMU properties", + "description": "data.properties is an array of property objects, representing the properties present in the data object.", + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "examples": [ + "MyPropertyName" + ] + }, + "attribute": { + "type": "string", + "examples": [ + "MyAttributeName" + ] + }, + "is_discrete": { + "type": "boolean", + "examples": [ + true, + false + ] + } + } + } + }, + "format": { + "type": "string", + "examples": [ + "irap_binary" + ] + }, + "layout": { + "type": "string", + "examples": [ + "regular" + ] + }, + "unit": { + "type": [ + "string", + "null" + ], + "examples": [ + "m" + ] + }, + "vertical_domain": { + "type": "string", + "enum": [ + "depth", + "time" + ], + "examples": [ + "depth" + ] + }, + "depth_reference": { + "type": [ + "string", + "null" + ], + "examples": [ + "msl" + ], + "enum": [ + "msl", + "sb", + "rkb", + null + ] + }, + "undef_is_zero": { + "description": "Flag if undefined values are to be interpreted as zero", + "type": "boolean", + "examples": [ + "True" + ] + }, + "grid_model": { + "type": "object", + "properties": { + "name": { + "type": [ + "string", + "null" + ], + "examples": [ + "MyGrid" + ] + } + } + }, + "spec": { + "type": "object", + "properties": { + "ncol": { + "$comment": "https://json-schema.org/understanding-json-schema/reference/numeric.html", + "type": "integer", + "examples": [ + 281 + ] + }, + "nrow": { + "type": "integer", + "examples": [ + 441 + ] + }, + "nlay": { + "type": "integer", + "examples": [ + 333 + ] + }, + "xori": { + "type": "number", + "examples": [ + 461499.9997558594 + ] + }, + "yori": { + "type": "number", + "examples": [ + 5926500.224123242 + ] + }, + "xinc": { + "type": "number", + "examples": [ + 25.0 + ] + }, + "yflip": { + "type": "integer", + "$comment": "Orientation indicator of coordinate system", + "enum": [ + -1, + 1 + ], + "examples": [ + -1, + 1 + ] + }, + "rotation": { + "type": "number", + "examples": [ + "30.00000000231" + ] + }, + "undef": { + "type": "number", + "$comment": "How is the undefined value represented?", + "examples": [ + 1.0e+33 + ] + }, + "npolys": { + "description": "The number of individual polygons in the data object", + "type": "integer", + "examples": [ + 1 + ] + }, + "size": { + "description": "Size of data object.", + "type": "integer", + "examples": [ + 1, + 9999 + ] + }, + "columns": { + "description": "List of columns present in a table.", + "type": "array", + "items": { + "type": "string", + "examples": [ + "FOPT", + "STOIIP_OIL", + "COL_1" + ] + } + } + } + }, + "bbox": { + "type": "object", + "required": [ + "xmin", + "xmax", + "ymin", + "ymax" + ], + "properties": { + "xmin": { + "type": "number", + "examples": [ + 456012.5003497944 + ] + }, + "xmax": { + "type": "number", + "examples": [ + 467540.52762886323 + ] + }, + "ymin": { + "type": "number", + "examples": [ + 5926499.999511719 + ] + }, + "ymax": { + "type": "number", + "examples": [ + 5939492.128326312 + ] + }, + "zmin": { + "type": [ + "number", + "null" + ], + "examples": [ + 1244.039, + null + ] + }, + "zmax": { + "type": [ + "number", + "null" + ], + "examples": [ + 2302.683, + null + ] + } + } + }, + "time": { + "type": "object", + "properties": { + "t0": { + "$ref": "#/$defs/properties/fmu_time" + }, + "t1": { + "$ref": "#/$defs/properties/fmu_time" + } + } + }, + "is_prediction": { + "title": "Is prediction flag", + "$comment": "For flagging predictions, and separating them from non-predictions", + "type": "boolean", + "examples": [ + true + ] + }, + "is_observation": { + "title": "Is observation flag", + "$comment": "For flagging observations, and separating them from non-observations", + "type": "boolean", + "examples": [ + true + ] + }, + "fluid_contact": { + "type": "object", + "description": "Conditional field", + "properties": { + "contact": { + "type": "string", + "enum": [ + "owc", + "fwl", + "goc", + "fgl" + ], + "$comment": "Not sure if wise to enum this", + "examples": [ + "owc", + "fwl" + ] + }, + "truncated": { + "type": "boolean", + "$comment": "Is this contact truncated to the stratigraphy?", + "examples": [ + true + ] + } + } + }, + "field_outline": { + "description": "Conditional field", + "type": "object", + "required": [ + "contact" + ], + "properties": { + "contact": { + "type": "string" + } + } + }, + "field_region": { + "description": "Conditional field", + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "description": "The ID of the region", + "type": "integer" + } + } + }, + "description": { + "$ref": "#/$defs/properties/generic/_description" + }, + "seismic": { + "type": "object", + "description": "Conditional field", + "properties": { + "attribute": { + "type": "string", + "examples": [ + "amplitude_timeshifted" + ] + }, + "calculation": { + "type": "string", + "examples": [ + "mean" + ] + }, + "zrange": { + "type": "number", + "examples": [ + 12.0 + ] + }, + "filter_size": { + "type": "number", + "examples": [ + 1.0 + ] + }, + "scaling_factor": { + "type": "number", + "examples": [ + 1.0 + ] + }, + "stacking_offset": { + "type": "string", + "examples": [ + "0-15" + ] + } + } + } + } + }, + "display": { + "type": "object", + "properties": { + "name": { + "type": "string", + "examples": [ + "Top Volantis" + ] + }, + "subtitle": { + "type": "string", + "examples": [ + "Some subtitle" + ] + }, + "line": { + "type": "object", + "properties": { + "show": { + "type": "boolean", + "examples": [ + true + ] + }, + "color": { + "type": "string", + "examples": [ + "black", + "#000000" + ] + } + } + }, + "points": { + "type": "object", + "properties": { + "show": { + "type": "boolean", + "examples": [ + true + ] + }, + "color": { + "type": "string", + "examples": [ + "black", + "#000000" + ] + } + } + }, + "contours": { + "type": "object", + "properties": { + "show": { + "type": "boolean", + "examples": [ + true + ] + }, + "color": { + "description": "The color of the contour lines", + "type": "string", + "examples": [ + "black", + "#000000" + ] + }, + "increment": { + "description": "The contour increment in same values as the data", + "type": "number", + "examples": [ + 20.0 + ] + } + } + }, + "fill": { + "type": "object", + "properties": { + "show": { + "type": "boolean", + "examples": [ + true + ] + }, + "color": { + "type": "string", + "examples": [ + "black", + "#000000" + ] + }, + "colormap": { + "$comment": "colormap can be given alongside fill.color. Clients must choose.", + "description": "named reference to a colormap", + "type": "string", + "examples": [ + "gist_earth" + ] + }, + "display_min": { + "description": "The low-side boundary to use for fill color", + "type": "number", + "examples": [ + 1000.0 + ] + }, + "display_max": { + "description": "The high-side boundary to use for fill color", + "type": "number", + "examples": [ + 1600.0 + ] + } + } + } + } + }, + "access": { + "type": "object", + "properties": { + "asset": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "examples": [ + "Drogon" + ] + } + } + }, + "ssdl": { + "type": "object", + "$comment": "Required for data objects", + "required": [ + "access_level", + "rep_include" + ], + "properties": { + "access_level": { + "$comment": "'asset' is legacy, but will be allowed in a transition", + "type": "string", + "enum": [ + "internal", + "restricted", + "asset" + ] + }, + "rep_include": { + "$comment": "Should REP display this data?", + "type": "boolean", + "examples": [ + true, + false + ] + } + } + }, + "classification": { + "type": "string", + "enum": [ + "internal", + "restricted" + ], + "examples": [ + "internal", + "restricted" + ] + } + } + }, + "masterdata": { + "$comment": "Block holds references to master data", + "type": "object", + "required": [ + "smda" + ], + "properties": { + "smda": { + "type": "object", + "required": [ + "country", + "discovery", + "field", + "coordinate_system", + "stratigraphic_column" + ], + "properties": { + "country": { + "$comment": "Array of SMDA-valid countries. First entry is primary.", + "type": "array", + "items": { + "type": "object", + "required": [ + "identifier", + "uuid" + ], + "properties": { + "identifier": { + "type": "string", + "examples": [ + "Norway" + ] + }, + "uuid": { + "$ref": "#/$defs/properties/generic/uuid" + } + } + } + }, + "discovery": { + "$comment": "Array of SMDA-valid discoveries. First entry is primary.", + "type": "array", + "items": { + "type": "object", + "required": [ + "short_identifier", + "uuid" + ], + "properties": { + "short_identifier": { + "type": "string", + "examples": [ + "SomeDiscovery" + ] + }, + "uuid": { + "$ref": "#/$defs/properties/generic/uuid" + } + } + } + }, + "field": { + "$comment": "Array of SMDA-valid (oil & gas) fields. First entry is primary.", + "type": "array", + "items": { + "type": "object", + "required": [ + "identifier", + "uuid" + ], + "properties": { + "identifier": { + "type": "string", + "examples": [ + "OseFax" + ] + }, + "uuid": { + "$ref": "#/$defs/properties/generic/uuid" + } + } + } + }, + "coordinate_system": { + "$comment": "SMDA-valid coordinate system.", + "type": "object", + "required": [ + "identifier", + "uuid" + ], + "properties": { + "identifier": { + "type": "string", + "examples": [ + "ST_WGS84_UTM37N_P32637" + ] + }, + "uuid": { + "$ref": "#/$defs/properties/generic/uuid" + } + } + }, + "stratigraphic_column": { + "$comment": "SMDA-valid stratigraphic column.", + "type": "object", + "required": [ + "identifier", + "uuid" + ], + "properties": { + "identifier": { + "type": "string", + "examples": [ + "DROGON_2020" + ] + }, + "uuid": { + "$ref": "#/$defs/properties/generic/uuid" + } + } + } + } + } + } + }, + "fmu": { + "model": { + "type": "object", + "properties": { + "name": { + "type": "string", + "examples": [ + "Drogon" + ] + }, + "revision": { + "type": "string", + "examples": [ + "21.0.0.dev" + ] + }, + "description": { + "description": "This is a free text description of the model setup", + "$ref": "#/$defs/properties/generic/_description" + } + } + }, + "workflow": { + "type": "object", + "properties": { + "reference": { + "type": "string", + "description": "Reference to the part of the FMU workflow that produced this" + } + } + }, + "case": { + "type": "object", + "required": [ + "name", + "uuid", + "user" + ], + "properties": { + "name": { + "type": "string", + "description": "The case name", + "examples": [ + "MyCaseName" + ] + }, + "uuid": { + "$ref": "#/$defs/properties/generic/uuid" + }, + "user": { + "description": "The user name used in ERT", + "$ref": "#/$defs/properties/generic/user" + }, + "description": { + "$ref": "#/$defs/properties/generic/_description" + } + } + }, + "iteration": { + "type": "object", + "required": [ + "name", + "uuid" + ], + "properties": { + "name": { + "description": "The convential name of this iteration, e.g. iter-0 or pred", + "type": "string", + "examples": [ + "iter-0" + ] + }, + "id": { + "description": "The internal identification of this iteration, e.g. the iteration number", + "type": "integer", + "examples": [ + 0 + ] + }, + "uuid": { + "$ref": "#/$defs/properties/generic/uuid" + }, + "restart_from": { + "description": "A uuid reference to another iteration that this iteration was restarted from", + "$ref": "#/$defs/properties/generic/uuid" + } + } + }, + "realization": { + "$comment": "Some schema repetition between realization and aggregation", + "type": "object", + "required": [ + "name", + "id", + "uuid" + ], + "properties": { + "name": { + "description": "The convential name of this iteration, e.g. iter-0 or pred", + "type": "string", + "examples": [ + "iter-0" + ] + }, + "id": { + "type": "integer", + "description": "The unique number of this realization as used in FMU", + "examples": [ + 33 + ] + }, + "uuid": { + "$ref": "#/$defs/properties/generic/uuid" + }, + "parameters": { + "type": "object", + "description": "Parameters for this realization", + "items": { + "type": "object" + } + }, + "jobs": { + "type": "object", + "description": "Content directly taken from the ERT jobs.json file for this realization" + } + } + }, + "aggregation": { + "$comment": "Some schema repetition between realization and aggregation", + "type": "object", + "required": [ + "operation", + "realization_ids" + ], + "properties": { + "operation": { + "type": "string", + "description": "The aggregation performed" + }, + "realization_ids": { + "type": "array", + "description": "Array of realization ids included in this aggregation", + "items": { + "type": "integer", + "examples": [ + 0, + 1, + 2, + 3 + ] + } + }, + "parameters": { + "type": "object", + "description": "Parameters for this realization", + "items": { + "type": "object" + } + }, + "id": { + "type": "string", + "description": "The ID of this aggregation", + "$comment": "Used for tying together representations of the same entity", + "$comment": "Example: Statistics for various time steps of the same surface", + "$comment": "Not used in Drogon but widely used on e.g. Johan Sverdrup", + "$comment": "Used for enabling common display and joint visual settings", + "$comment": "A unique uuid is currently used, and included as example but any ID should be sufficient", + "examples": [ + "15ce3b84-766f-4c93-9050-b154861f9100" + ] + } + } + }, + "context": { + "description": "The internal FMU context in which this data object was produced", + "type": "object", + "required": [ + "stage" + ], + "properties": { + "stage": { + "type": "string", + "examples": [ + "case", + "iteration", + "realization" + ] + } + } + } + }, + "file": { + "description": "Block describing the file as the data appear in FMU context", + "$comment": "While the file may seize to exist, the concept is still useful", + "$comment": "A lot of logic exists on top of the file name now, legacy but not likely to disappear soon", + "type": "object", + "required": [ + "relative_path", + "checksum_md5" + ], + "properties": { + "relative_path": { + "type": "string", + "description": "The file path relative to RUNPATH", + "examples": [ + "share/results/maps/volantis_gp_base--depth.gri" + ] + }, + "absolute_path": { + "type": "string", + "description": "The absolute file path", + "examples": [ + "/abs/path/share/results/maps/volantis_gp_base--depth.gri" + ] + }, + "checksum_md5": { + "description": "md5 checksum of the file or bytestring", + "type": "string", + "examples": [ + "kjhsdfvsdlfk23knerknvk23" + ] + }, + "size_bytes": { + "description": "Size of file object in bytes", + "type": "integer", + "examples": [ + 123 + ] + } + } + } + }, + "subschemas": { + "case": { + "required": [ + "source", + "version", + "class", + "fmu", + "access", + "tracklog", + "masterdata" + ], + "properties": { + "source": { + "$ref": "#/$defs/properties/source" + }, + "version": { + "$ref": "#/$defs/properties/version" + }, + "class": { + "$ref": "#/$defs/properties/class" + }, + "tracklog": { + "$ref": "#/$defs/properties/tracklog" + }, + "data": { + "$ref": "#/$defs/properties/data" + }, + "access": { + "$ref": "#/$defs/properties/access", + "required": [ + "asset" + ] + }, + "masterdata": { + "$ref": "#/$defs/properties/masterdata" + }, + "fmu": { + "description": "The FMU block records properties that are specific to FMU", + "$comment": "This is the fmu block as it appears in data objects", + "type": "object", + "required": [ + "model", + "case" + ], + "properties": { + "model": { + "$ref": "#/$defs/properties/fmu/model" + }, + "case": { + "$ref": "#/$defs/properties/fmu/case" + } + } + } + } + }, + "data_object": { + "properties": { + "fmu": { + "description": "The FMU block records properties that are specific to FMU", + "$comment": "This is the fmu block as it appears in data objects", + "required": [ + "model", + "case" + ], + "properties": { + "model": { + "$ref": "#/$defs/properties/fmu/model" + }, + "case": { + "$ref": "#/$defs/properties/fmu/case" + }, + "iteration": { + "$ref": "#/$defs/properties/fmu/iteration" + }, + "realization": { + "$ref": "#/$defs/properties/fmu/realization" + }, + "workflow": { + "$ref": "#/$defs/properties/fmu/workflow" + }, + "aggregation": { + "$ref": "#/$defs/properties/fmu/aggregation" + } + }, + "$comment": "implementation below allows realization or aggregation or none of them, never both", + "dependencies": { + "aggregation": { + "not": { + "required": [ + "realization" + ] + } + }, + "realization": { + "not": { + "required": [ + "aggregation" + ] + } + } + } + }, + "file": { + "$ref": "#/$defs/properties/file" + }, + "data": { + "$comment": "Conditionals", + "allOf": [ + { + "$ref": "#/$defs/properties/data" + }, + { + "$comment": "When content == field_outline, require data.field_outline", + "if": { + "properties": { + "content": { + "const": "field_outline" + } + } + }, + "then": { + "required": [ + "field_outline" + ] + } + }, + { + "$comment": "When content == field_region, require data.field_region", + "if": { + "properties": { + "content": { + "const": "field_region" + } + } + }, + "then": { + "required": [ + "field_region" + ] + } + }, + { + "$comment": "When content == fluid_contact, require data.fluid_contact", + "if": { + "properties": { + "content": { + "const": "fluid_contact" + } + } + }, + "then": { + "required": [ + "fluid_contact" + ] + } + }, + { + "$comment": "When content == seismic, require data.seismic", + "if": { + "properties": { + "content": { + "const": "seismic" + } + } + }, + "then": { + "required": [ + "seismic" + ] + } + } + ] + }, + "access": { + "$ref": "#/$defs/properties/access", + "required": [ + "asset", + "ssdl" + ] + }, + "source": { + "$ref": "#/$defs/properties/source" + }, + "version": { + "$ref": "#/$defs/properties/version" + }, + "class": { + "$ref": "#/$defs/properties/class" + }, + "tracklog": { + "$ref": "#/$defs/properties/tracklog" + }, + "masterdata": { + "$ref": "#/$defs/properties/masterdata" + } + }, + "if": { + "properties": { + "class": { + "enum": [ + "table", + "surface" + ] + } + } + }, + "then": { + "properties": { + "data": { + "required": [ + "spec" + ] + } + } + }, + "required": [ + "source", + "version", + "class", + "fmu", + "access", + "tracklog", + "masterdata", + "data", + "file" + ] + } + } + }, + "$comment": "=== MAIN SCHEMA ENTRY POINT ===", + "if": { + "properties": { + "class": { + "const": "case" + } + } + }, + "then": { + "title": "Case object", + "description": "Validation of a case object.", + "$ref": "#/$defs/subschemas/case" + }, + "else": { + "title": "Data object", + "description": "Validation of a non-case object (data object).", + "$ref": "#/$defs/subschemas/data_object" + } +} \ No newline at end of file diff --git a/src/fmu/dataio/model.py b/src/fmu/dataio/model.py deleted file mode 100644 index f1bc42f4e..000000000 --- a/src/fmu/dataio/model.py +++ /dev/null @@ -1,592 +0,0 @@ -# generated by datamodel-codegen: -# filename: fmu_results_090.json -# version: 0.25.2 - -from __future__ import annotations - -from enum import Enum -from typing import Any, List, Literal, Union - -from pydantic import BaseModel, Field, RootModel -from typing_extensions import Annotated - - -class Generic(RootModel[Any]): - root: Any - - -class MetadataClass(str, Enum): - case = "case" - surface = "surface" - table = "table" - cpgrid = "cpgrid" - cpgrid_property = "cpgrid_property" - polygons = "polygons" - cube = "cube" - well = "well" - points = "points" - - -class Source(RootModel[Literal["fmu"]]): - root: Literal["fmu"] = Field(..., description="Data source (FMU)") - - -class FMUResultsMetadataVersion(RootModel[Literal["0.9.0"]]): - root: Literal["0.9.0"] = Field( - ..., examples=["1.2.3"], title="FMU results metadata version" - ) - - -class Property(BaseModel): - name: str | None = Field(default=None, examples=["MyPropertyName"]) - attribute: str | None = Field(default=None, examples=["MyAttributeName"]) - is_discrete: bool | None = Field(default=None, examples=[True, False]) - - -class VerticalDomain(str, Enum): - depth = "depth" - time = "time" - - -class DepthReference(Enum): - msl = "msl" - sb = "sb" - rkb = "rkb" - none_type_none = None - - -class GridModel(BaseModel): - name: str | None = Field(default=None, examples=["MyGrid"]) - - -class Yflip(int, Enum): - integer_0 = 0 - integer_1 = 1 - - -class Spec(BaseModel): - ncol: int | None = Field(default=None, examples=[281]) - nrow: int | None = Field(default=None, examples=[441]) - nlay: int | None = Field(default=None, examples=[333]) - xori: float | None = Field(default=None, examples=[461499.9997558594]) - yori: float | None = Field(default=None, examples=[461499.9997558594]) - xinc: float | None = Field(default=None, examples=[25]) - yflip: Yflip | None = Field(default=None, examples=[1]) - rotation: float | None = Field(default=None, examples=["30.00000000231"]) - undef: float | None = Field(default=None, examples=["1e+33"]) - npolys: int | None = Field( - default=None, - description="The number of individual polygons in the data object", - examples=[1], - ) - - -class Bbox(BaseModel): - xmin: float = Field(..., examples=[456012.5003497944]) - xmax: float = Field(..., examples=[467540.52762886323]) - ymin: float = Field(..., examples=[5926499.999511719]) - ymax: float = Field(..., examples=[5939492.128326312]) - zmin: float | None = Field(default=None, examples=[1244.039, None]) - zmax: float | None = Field(default=None, examples=[2302.683, None]) - - -class Line(BaseModel): - show: bool | None = Field(default=None, examples=[True]) - color: str | None = Field(default=None, examples=["black", "#000000"]) - - -class Points(BaseModel): - show: bool | None = Field(default=None, examples=[True]) - color: str | None = Field(default=None, examples=["black", "#000000"]) - - -class Contours(BaseModel): - show: bool | None = Field(default=None, examples=[True]) - color: str | None = Field( - default=None, - description="The color of the contour lines", - examples=["black", "#000000"], - ) - increment: float | None = Field( - default=None, - description="The contour increment in same values as the data", - examples=[20], - ) - - -class Fill(BaseModel): - show: bool | None = Field(default=None, examples=[True]) - color: str | None = Field(default=None, examples=["black", "#000000"]) - colormap: str | None = Field( - default=None, - description="named reference to a colormap", - examples=["gist_earth"], - ) - display_min: float | None = Field( - default=None, - description="The low-side boundary to use for fill color", - examples=[1000], - ) - display_max: float | None = Field( - default=None, - description="The high-side boundary to use for fill color", - examples=[1600], - ) - - -class Display(BaseModel): - name: str | None = Field(default=None, examples=["Top Volantis"]) - subtitle: str | None = Field(default=None, examples=["Some subtitle"]) - line: Line | None = None - points: Points | None = None - contours: Contours | None = None - fill: Fill | None = None - - -class Asset(BaseModel): - name: str = Field(..., examples=["Drogon"]) - - -class Access(BaseModel): - asset: Asset | None = None - - -class Workflow1(BaseModel): - reference: str | None = Field( - default=None, - description="Reference to the part of the FMU workflow that produced this", - ) - - -class Aggregation1(BaseModel): - operation: str = Field(..., description="The aggregation performed") - realization_ids: list[int] = Field( - ..., description="Array of realization ids included in this aggregation" - ) - parameters: dict[str, Any] | None = Field( - default=None, description="Parameters for this realization" - ) - id: str | None = Field( - default=None, - description="The ID of this aggregation", - examples=["15ce3b84-766f-4c93-9050-b154861f9100"], - ) - - -class File(BaseModel): - """ - Block describing the file as the data appear in FMU context - """ - - relative_path: str = Field( - ..., - description="The file path relative to RUNPATH", - examples=["share/results/maps/volantis_gp_base--depth.gri"], - ) - absolute_path: str | None = Field( - default=None, - description="The absolute file path", - examples=["/abs/path/share/results/maps/volantis_gp_base--depth.gri"], - ) - checksum_md5: str = Field( - ..., - description="md5 checksum of the file or bytestring", - examples=["kjhsdfvsdlfk23knerknvk23"], - ) - - -class Name(RootModel[str]): - root: str = Field( - ..., - description=( - "Name of the surface. If stratigraphic, match " - "the entry in the stratigraphic column" - ), - examples=["VIKING GP. Top"], - ) - - -class Offset(RootModel[float]): - root: float = Field(..., examples=[11.2]) - - -class Stratigraphic(RootModel[bool]): - root: bool = Field( - ..., - description="True if surface represents an entity in the stratigraphic column", - examples=[True], - ) - - -class Aggregation(RootModel[Any]): - root: Any - - -class Case(RootModel[Any]): - root: Any - - -class Iteration(RootModel[Any]): - root: Any - - -class Model(RootModel[Any]): - root: Any - - -class Realization(RootModel[Any]): - root: Any - - -class Workflow(RootModel[Any]): - root: Any - - -class FieldDescription(RootModel[List[str]]): - root: list[str] - - -class Datetime(RootModel[str]): - root: str = Field(..., examples=["2020-10-28T14:28:02"]) - - -class User(BaseModel): - id: str = Field(..., examples=["peesv"], title="User ID") - - -class Uuid(RootModel[str]): - root: str = Field( - ..., - examples=["ad214d85-8a1d-19da-e053-c918a4889309"], - pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", - ) - - -class Fmu(BaseModel): - """ - The FMU block records properties that are specific to FMU - """ - - model: Model - case: Case - - -class Fmu1(BaseModel): - """ - The FMU block records properties that are specific to FMU - """ - - model: Model - case: Case - iteration: Iteration - workflow: Workflow - realization: Realization - - -class Fmu2(BaseModel): - """ - The FMU block records properties that are specific to FMU - """ - - model: Model - case: Case - iteration: Iteration - workflow: Workflow - aggregation: Aggregation - - -class FMUTimeObjectItem(BaseModel): - value: Datetime | None = None - label: str | None = Field(default=None, examples=["Label for time stamp"]) - - -class FMUTimeObject(RootModel[List[FMUTimeObjectItem]]): - """ - List of time stamps referring to simulated time - """ - - root: list[FMUTimeObjectItem] = Field( - ..., - description="List of time stamps referring to simulated time", - min_length=1, - title="FMU time object", - ) - - -class TracklogEvent(BaseModel): - datetime: Datetime | None = None - user: User | None = None - event: str | None = Field(default=None, examples=["created", "updated"]) - - -class Tracklog(RootModel[List[TracklogEvent]]): - root: list[TracklogEvent] - - -class Top(BaseModel): - name: Name | None = None - stratigraphic: Stratigraphic | None = None - offset: Offset | None = None - - -class Base(BaseModel): - name: Name | None = None - stratigraphic: Stratigraphic | None = None - offset: Offset | None = None - - -class TheDataBlock(BaseModel): - name: str = Field( - ..., - description=( - "Name of the surface. If stratigraphic, match the entry " - "in the stratigraphic column" - ), - examples=["VIKING GP. Top"], - ) - stratigraphic: bool = Field( - ..., - description="True if surface represents an entity in the stratigraphic column", - examples=[True], - ) - alias: list[Name] | None = None - stratigraphic_alias: list[Name] | None = None - offset: float | None = Field(default=None, examples=[11.2]) - top: Top | None = None - base: Base | None = None - content: str = Field( - ..., description="The contents of this data object", examples=["depth"] - ) - tagname: str | None = Field( - default=None, - description="A semi-human readable tag for internal usage and uniqueness", - examples=["ds_extract_geogrid", "ds_post_strucmod"], - ) - properties: list[Property] | None = None - format: str = Field(..., examples=["irap_binary"]) - layout: str | None = Field(default=None, examples=["regular"]) - unit: str | None = Field(default=None, examples=["m"]) - vertical_domain: VerticalDomain | None = Field(default=None, examples=["depth"]) - depth_reference: DepthReference | None = Field(default=None, examples=["msl"]) - grid_model: GridModel | None = None - spec: Spec - bbox: Bbox | None = None - time: FMUTimeObject | None = None - is_prediction: bool = Field(..., examples=[True], title="Is prediction flag") - is_observation: bool = Field(..., examples=[True], title="Is observation flag") - description: FieldDescription | None = None - - -class CountryItem(BaseModel): - identifier: str = Field(..., examples=["Norway"]) - uuid: Uuid - - -class DiscoveryItem(BaseModel): - short_identifier: str = Field(..., examples=["SomeDiscovery"]) - uuid: Uuid - - -class FieldItem(BaseModel): - identifier: str = Field(..., examples=["OseFax"]) - uuid: Uuid - - -class CoordinateSystem(BaseModel): - identifier: str = Field(..., examples=["ST_WGS84_UTM37N_P32637"]) - uuid: Uuid - - -class StratigraphicColumn(BaseModel): - identifier: str = Field(..., examples=["DROGON_2020"]) - uuid: Uuid - - -class Smda(BaseModel): - country: list[CountryItem] - discovery: list[DiscoveryItem] - field: list[FieldItem] - coordinate_system: CoordinateSystem - stratigraphic_column: StratigraphicColumn - - -class Masterdata(BaseModel): - smda: Smda - - -class Model1(BaseModel): - name: str | None = Field(default=None, examples=["Drogon"]) - revision: str | None = Field(default=None, examples=["21.0.0.dev"]) - description: FieldDescription | None = Field( - default=None, description="This is a free text description of the model setup" - ) - - -class Case1(BaseModel): - name: str = Field(..., description="The case name", examples=["MyCaseName"]) - uuid: Uuid - user: User = Field(..., description="The user name used in ERT") - restart_from: str | None = Field( - default=None, - description=( - "A reference to another case/iteration that this " - "iteration was restarted from" - ), - ) - description: FieldDescription | None = None - - -class Iteration1(BaseModel): - name: str = Field( - ..., - description="The convential name of this iteration, e.g. iter-0 or pred", - examples=["iter-0"], - ) - id: int = Field( - ..., - description=( - "The internal identification of this " - "iteration, e.g. the iteration number" - ), - examples=[0], - ) - uuid: Uuid - - -class Realization1(BaseModel): - name: str = Field( - ..., - description="The convential name of this iteration, e.g. iter-0 or pred", - examples=["iter-0"], - ) - id: int = Field( - ..., - description="The unique number of this realization as used in FMU", - examples=[33], - ) - uuid: Uuid - parameters: dict[str, Any] | None = Field( - default=None, description="Parameters for this realization" - ) - jobs: dict[str, Any] | None = Field( - default=None, - description=( - "Content directly taken from the ERT jobs.json " "file for this realization" - ), - ) - - -class Fmu3(BaseModel): - model: Model1 | None = None - workflow: Workflow1 | None = None - case: Case1 | None = None - iteration: Iteration1 | None = None - realization: Realization1 | None = None - aggregation: Aggregation1 | None = None - - -class CaseModel(BaseModel): - source: Source - version: FMUResultsMetadataVersion - data: TheDataBlock | None = None - tracklog: Tracklog - access: Access - masterdata: Masterdata - class_: MetadataClass = Field(..., alias="class") - fmu: Fmu = Field( - ..., description="The FMU block records properties that are specific to FMU" - ) - - -class DataObject(BaseModel): - source: Source - version: FMUResultsMetadataVersion - data: TheDataBlock - tracklog: Tracklog - access: Access - masterdata: Masterdata - class_: MetadataClass = Field(..., alias="class") - fmu: Fmu1 | Fmu2 = Field( - ..., description="The FMU block records properties that are specific to FMU" - ) - file: File - - -class ModelModel(RootModel[Union[CaseModel, DataObject]]): - root: CaseModel | DataObject - - -### V3? - - -class GeologicalModel(BaseModel): - type: Literal["Structural", "Rock"] - - -class RockGeologicalModel(GeologicalModel): - type: Literal["Rock"] = "Rock" - - -class StructuralGeologicalModel(GeologicalModel): - type: Literal["Structural"] = "Structural" - - -class Shape(BaseModel): - ncol: int = Field(ge=0) - nrow: int = Field(ge=0) - nlay: int = Field(ge=0) - - -class Orientation(BaseModel): - x: float - y: float - z: float - - -class Grid(BaseModel): - coordinate_system: CoordinateSystem - orientation: Orientation - shape: Shape - undef: float | None - - -class Range(BaseModel): - start: float - stop: float - - -class BoundingBox(BaseModel): - x: Range - y: Range - z: Range - - -class Header(BaseModel): - asset: str - created_at: Datetime - created_by: str - version: int - - -class Payland(BaseModel): - type: Literal["fmu.everest", "fmu.ert"] - - -class FMUEverest(Payland): - type: Literal["fmu.everest"] = "fmu.everest" - - -class FMUErt(BaseModel): - type: Literal["fmu.ert"] = "fmu.ert" - model: Annotated[ - StructuralGeologicalModel | RockGeologicalModel, - Field(discriminator="type"), - ] - - -class Export(BaseModel): - header: Header - payload: Annotated[ - FMUEverest | FMUErt, - Field(discriminator="type"), - ] diff --git a/src/fmu/dataio/models/080model.py b/src/fmu/dataio/models/080model.py new file mode 100644 index 000000000..d06b61230 --- /dev/null +++ b/src/fmu/dataio/models/080model.py @@ -0,0 +1,1332 @@ +# generated by datamodel-codegen: +# filename: fmu_results.json +# version: 0.25.2 + +from __future__ import annotations + +from pydantic import BaseModel, Field + + +class Datetime(BaseModel): + type: str + examples: list[str] + + +class Uuid(BaseModel): + type: str + pattern: str + examples: list[str] + + +class Id(BaseModel): + title: str + type: str + examples: list[str] + + +class Properties(BaseModel): + id: Id + + +class User(BaseModel): + type: str + required: list[str] + properties: Properties + + +class Items(BaseModel): + type: str + examples: list[str] + + +class FieldDescription(BaseModel): + field_comment: str = Field(..., alias="$comment") + type: str + items: Items + + +class Generic(BaseModel): + field_comment: str = Field(..., alias="$comment") + datetime: Datetime + uuid: Uuid + user: User + field_description: FieldDescription = Field(..., alias="_description") + + +class Value(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Label(BaseModel): + type: str + examples: list[str] + + +class Properties1(BaseModel): + value: Value + label: Label + + +class FmuTime(BaseModel): + title: str + description: str + type: list[str] + properties: Properties1 + + +class Class(BaseModel): + title: str + type: str + examples: list[str] + enum: list[str] + + +class Source(BaseModel): + field_comment: str = Field(..., alias="$comment") + description: str + type: str + const: str + + +class Version(BaseModel): + field_comment: str = Field(..., alias="$comment") + title: str + type: str + enum: list[str] + example: str + + +class Datetime1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class User1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Event(BaseModel): + type: str + examples: list[str] + + +class Properties2(BaseModel): + datetime: Datetime1 + user: User1 + event: Event + + +class TracklogEvent(BaseModel): + type: str + properties: Properties2 + + +class Items1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Tracklog(BaseModel): + type: str + items: Items1 + + +class Top(BaseModel): + required: list[str] + + +class Base(BaseModel): + required: list[str] + + +class Dependencies(BaseModel): + top: Top + base: Base + + +class Name(BaseModel): + type: str + description: str + examples: list[str] + + +class Stratigraphic(BaseModel): + field_comment: str = Field(..., alias="$comment") + type: str + description: str + examples: list[bool] + + +class Alias(BaseModel): + type: str + items: Items1 + + +class StratigraphicAlias(BaseModel): + type: str + items: Items1 + + +class Offset(BaseModel): + field_comment: str = Field(..., alias="$comment") + type: str + example: float + + +class Name1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Stratigraphic1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Offset1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Properties4(BaseModel): + name: Name1 + stratigraphic: Stratigraphic1 + offset: Offset1 + + +class Top1(BaseModel): + field_comment: str = Field(..., alias="$comment") + properties: Properties4 + + +class Properties5(BaseModel): + name: Name1 + stratigraphic: Stratigraphic1 + offset: Offset1 + + +class Base1(BaseModel): + field_comment: str = Field(..., alias="$comment") + properties: Properties5 + + +class Content(BaseModel): + type: str + description: str + enum: list[str] + examples: list[str] + + +class Tagname(BaseModel): + type: str + description: str + examples: list[str] + + +class Name3(BaseModel): + type: str + examples: list[str] + + +class Attribute(BaseModel): + type: str + examples: list[str] + + +class IsDiscrete(BaseModel): + type: str + examples: list[bool] + + +class Properties7(BaseModel): + name: Name3 + attribute: Attribute + is_discrete: IsDiscrete + + +class Items4(BaseModel): + type: str + properties: Properties7 + + +class Properties6(BaseModel): + field_comment: str = Field(..., alias="$comment") + description: str + type: str + items: Items4 + + +class Format(BaseModel): + type: str + examples: list[str] + + +class Layout(BaseModel): + type: str + examples: list[str] + + +class Unit(BaseModel): + type: list[str] + examples: list[str] + + +class VerticalDomain(BaseModel): + type: str + enum: list[str] + examples: list[str] + + +class DepthReference(BaseModel): + type: list[str] + examples: list[str] + enum: list[str | None] + + +class UndefIsZero(BaseModel): + description: str + type: str + examples: list[str] + + +class Name4(BaseModel): + type: list[str] + examples: list[str] + + +class Properties8(BaseModel): + name: Name4 + + +class GridModel(BaseModel): + type: str + properties: Properties8 + + +class Ncol(BaseModel): + field_comment: str = Field(..., alias="$comment") + type: str + examples: list[int] + + +class Nrow(BaseModel): + type: str + examples: list[int] + + +class Nlay(BaseModel): + type: str + examples: list[int] + + +class Xori(BaseModel): + type: str + examples: list[float] + + +class Yori(BaseModel): + type: str + examples: list[float] + + +class Xinc(BaseModel): + type: str + examples: list[float] + + +class Yflip(BaseModel): + type: str + field_comment: str = Field(..., alias="$comment") + enum: list[int] + examples: list[int] + + +class Rotation(BaseModel): + type: str + examples: list[str] + + +class Undef(BaseModel): + type: str + field_comment: str = Field(..., alias="$comment") + examples: list[float] + + +class Npolys(BaseModel): + description: str + type: str + examples: list[int] + + +class Size(BaseModel): + description: str + type: str + examples: list[int] + + +class Items5(BaseModel): + type: str + examples: list[str] + + +class Columns(BaseModel): + description: str + type: str + items: Items5 + + +class Properties9(BaseModel): + ncol: Ncol + nrow: Nrow + nlay: Nlay + xori: Xori + yori: Yori + xinc: Xinc + yflip: Yflip + rotation: Rotation + undef: Undef + npolys: Npolys + size: Size + columns: Columns + + +class Spec(BaseModel): + type: str + properties: Properties9 + + +class Xmin(BaseModel): + type: str + examples: list[float] + + +class Xmax(BaseModel): + type: str + examples: list[float] + + +class Ymin(BaseModel): + type: str + examples: list[float] + + +class Ymax(BaseModel): + type: str + examples: list[float] + + +class Zmin(BaseModel): + type: list[str] + examples: list[float | None] + + +class Zmax(BaseModel): + type: list[str] + examples: list[float | None] + + +class Properties10(BaseModel): + xmin: Xmin + xmax: Xmax + ymin: Ymin + ymax: Ymax + zmin: Zmin + zmax: Zmax + + +class Bbox(BaseModel): + type: str + required: list[str] + properties: Properties10 + + +class T0(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class T1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Properties11(BaseModel): + t0: T0 + t1: T1 + + +class Time(BaseModel): + type: str + properties: Properties11 + + +class IsPrediction(BaseModel): + title: str + field_comment: str = Field(..., alias="$comment") + type: str + examples: list[bool] + + +class IsObservation(BaseModel): + title: str + field_comment: str = Field(..., alias="$comment") + type: str + examples: list[bool] + + +class Contact(BaseModel): + type: str + enum: list[str] + field_comment: str = Field(..., alias="$comment") + examples: list[str] + + +class Truncated(BaseModel): + type: str + field_comment: str = Field(..., alias="$comment") + examples: list[bool] + + +class Properties12(BaseModel): + contact: Contact + truncated: Truncated + + +class FluidContact(BaseModel): + type: str + description: str + properties: Properties12 + + +class Contact1(BaseModel): + type: str + + +class Properties13(BaseModel): + contact: Contact1 + + +class FieldOutline(BaseModel): + description: str + type: str + required: list[str] + properties: Properties13 + + +class Id1(BaseModel): + description: str + type: str + + +class Properties14(BaseModel): + id: Id1 + + +class FieldRegion(BaseModel): + description: str + type: str + required: list[str] + properties: Properties14 + + +class Description(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Calculation(BaseModel): + type: str + examples: list[str] + + +class Zrange(BaseModel): + type: str + examples: list[float] + + +class FilterSize(BaseModel): + type: str + examples: list[float] + + +class ScalingFactor(BaseModel): + type: str + examples: list[float] + + +class StackingOffset(BaseModel): + type: str + examples: list[str] + + +class Properties15(BaseModel): + attribute: Attribute + calculation: Calculation + zrange: Zrange + filter_size: FilterSize + scaling_factor: ScalingFactor + stacking_offset: StackingOffset + + +class Seismic(BaseModel): + type: str + description: str + properties: Properties15 + + +class Properties3(BaseModel): + name: Name + stratigraphic: Stratigraphic + alias: Alias + stratigraphic_alias: StratigraphicAlias + offset: Offset + top: Top1 + base: Base1 + content: Content + tagname: Tagname + properties: Properties6 + format: Format + layout: Layout + unit: Unit + vertical_domain: VerticalDomain + depth_reference: DepthReference + undef_is_zero: UndefIsZero + grid_model: GridModel + spec: Spec + bbox: Bbox + time: Time + is_prediction: IsPrediction + is_observation: IsObservation + fluid_contact: FluidContact + field_outline: FieldOutline + field_region: FieldRegion + description: Description + seismic: Seismic + + +class Data(BaseModel): + type: str + title: str + required: list[str] + dependencies: Dependencies + properties: Properties3 + + +class Name5(BaseModel): + type: str + examples: list[str] + + +class Subtitle(BaseModel): + type: str + examples: list[str] + + +class Show(BaseModel): + type: str + examples: list[bool] + + +class Color(BaseModel): + type: str + examples: list[str] + + +class Properties17(BaseModel): + show: Show + color: Color + + +class Line(BaseModel): + type: str + properties: Properties17 + + +class Properties18(BaseModel): + show: Show + color: Color + + +class Points(BaseModel): + type: str + properties: Properties18 + + +class Color2(BaseModel): + description: str + type: str + examples: list[str] + + +class Increment(BaseModel): + description: str + type: str + examples: list[float] + + +class Properties19(BaseModel): + show: Show + color: Color2 + increment: Increment + + +class Contours(BaseModel): + type: str + properties: Properties19 + + +class Color3(BaseModel): + type: str + examples: list[str] + + +class Colormap(BaseModel): + field_comment: str = Field(..., alias="$comment") + description: str + type: str + examples: list[str] + + +class DisplayMin(BaseModel): + description: str + type: str + examples: list[float] + + +class DisplayMax(BaseModel): + description: str + type: str + examples: list[float] + + +class Properties20(BaseModel): + show: Show + color: Color3 + colormap: Colormap + display_min: DisplayMin + display_max: DisplayMax + + +class Fill(BaseModel): + type: str + properties: Properties20 + + +class Properties16(BaseModel): + name: Name5 + subtitle: Subtitle + line: Line + points: Points + contours: Contours + fill: Fill + + +class Display(BaseModel): + type: str + properties: Properties16 + + +class Properties22(BaseModel): + name: Name5 + + +class Asset(BaseModel): + type: str + required: list[str] + properties: Properties22 + + +class AccessLevel(BaseModel): + field_comment: str = Field(..., alias="$comment") + type: str + enum: list[str] + + +class RepInclude(BaseModel): + field_comment: str = Field(..., alias="$comment") + type: str + examples: list[bool] + + +class Properties23(BaseModel): + access_level: AccessLevel + rep_include: RepInclude + + +class Ssdl(BaseModel): + type: str + field_comment: str = Field(..., alias="$comment") + required: list[str] + properties: Properties23 + + +class Classification(BaseModel): + type: str + enum: list[str] + examples: list[str] + + +class Properties21(BaseModel): + asset: Asset + ssdl: Ssdl + classification: Classification + + +class Access(BaseModel): + type: str + properties: Properties21 + + +class Identifier(BaseModel): + type: str + examples: list[str] + + +class Uuid1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Properties26(BaseModel): + identifier: Identifier + uuid: Uuid1 + + +class Items6(BaseModel): + type: str + required: list[str] + properties: Properties26 + + +class Country(BaseModel): + field_comment: str = Field(..., alias="$comment") + type: str + items: Items6 + + +class ShortIdentifier(BaseModel): + type: str + examples: list[str] + + +class Properties27(BaseModel): + short_identifier: ShortIdentifier + uuid: Uuid1 + + +class Items7(BaseModel): + type: str + required: list[str] + properties: Properties27 + + +class Discovery(BaseModel): + field_comment: str = Field(..., alias="$comment") + type: str + items: Items7 + + +class Properties28(BaseModel): + identifier: Identifier + uuid: Uuid1 + + +class Items8(BaseModel): + type: str + required: list[str] + properties: Properties28 + + +class FieldModel(BaseModel): + field_comment: str = Field(..., alias="$comment") + type: str + items: Items8 + + +class Properties29(BaseModel): + identifier: Identifier + uuid: Uuid1 + + +class CoordinateSystem(BaseModel): + field_comment: str = Field(..., alias="$comment") + type: str + required: list[str] + properties: Properties29 + + +class Properties30(BaseModel): + identifier: Identifier + uuid: Uuid1 + + +class StratigraphicColumn(BaseModel): + field_comment: str = Field(..., alias="$comment") + type: str + required: list[str] + properties: Properties30 + + +class Properties25(BaseModel): + country: Country + discovery: Discovery + field: FieldModel + coordinate_system: CoordinateSystem + stratigraphic_column: StratigraphicColumn + + +class Smda(BaseModel): + type: str + required: list[str] + properties: Properties25 + + +class Properties24(BaseModel): + smda: Smda + + +class Masterdata(BaseModel): + field_comment: str = Field(..., alias="$comment") + type: str + required: list[str] + properties: Properties24 + + +class Revision(BaseModel): + type: str + examples: list[str] + + +class Description1(BaseModel): + description: str + field_ref: str = Field(..., alias="$ref") + + +class Properties31(BaseModel): + name: Name5 + revision: Revision + description: Description1 + + +class Model1(BaseModel): + type: str + properties: Properties31 + + +class Reference(BaseModel): + type: str + description: str + + +class Properties32(BaseModel): + reference: Reference + + +class Workflow(BaseModel): + type: str + properties: Properties32 + + +class Name8(BaseModel): + type: str + description: str + examples: list[str] + + +class User2(BaseModel): + description: str + field_ref: str = Field(..., alias="$ref") + + +class Description2(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Properties33(BaseModel): + name: Name8 + uuid: Uuid1 + user: User2 + description: Description2 + + +class Case(BaseModel): + type: str + required: list[str] + properties: Properties33 + + +class Name9(BaseModel): + description: str + type: str + examples: list[str] + + +class Id2(BaseModel): + description: str + type: str + examples: list[int] + + +class RestartFrom(BaseModel): + description: str + field_ref: str = Field(..., alias="$ref") + + +class Properties34(BaseModel): + name: Name9 + id: Id2 + uuid: Uuid1 + restart_from: RestartFrom + + +class Iteration(BaseModel): + type: str + required: list[str] + properties: Properties34 + + +class Id3(BaseModel): + type: str + description: str + examples: list[int] + + +class Items9(BaseModel): + type: str + + +class Parameters(BaseModel): + type: str + description: str + items: Items9 + + +class Jobs(BaseModel): + type: str + description: str + + +class Properties35(BaseModel): + name: Name9 + id: Id3 + uuid: Uuid1 + parameters: Parameters + jobs: Jobs + + +class Realization(BaseModel): + field_comment: str = Field(..., alias="$comment") + type: str + required: list[str] + properties: Properties35 + + +class Operation(BaseModel): + type: str + description: str + + +class Items10(BaseModel): + type: str + examples: list[int] + + +class RealizationIds(BaseModel): + type: str + description: str + items: Items10 + + +class Items11(BaseModel): + type: str + + +class Parameters1(BaseModel): + type: str + description: str + items: Items11 + + +class Id4(BaseModel): + type: str + description: str + field_comment: str = Field(..., alias="$comment") + examples: list[str] + + +class Properties36(BaseModel): + operation: Operation + realization_ids: RealizationIds + parameters: Parameters1 + id: Id4 + + +class Aggregation(BaseModel): + field_comment: str = Field(..., alias="$comment") + type: str + required: list[str] + properties: Properties36 + + +class Stage(BaseModel): + type: str + examples: list[str] + + +class Properties37(BaseModel): + stage: Stage + + +class Context(BaseModel): + description: str + type: str + required: list[str] + properties: Properties37 + + +class Fmu(BaseModel): + model: Model1 + workflow: Workflow + case: Case + iteration: Iteration + realization: Realization + aggregation: Aggregation + context: Context + + +class RelativePath(BaseModel): + type: str + description: str + examples: list[str] + + +class AbsolutePath(BaseModel): + type: str + description: str + examples: list[str] + + +class ChecksumMd5(BaseModel): + description: str + type: str + examples: list[str] + + +class SizeBytes(BaseModel): + description: str + type: str + examples: list[int] + + +class Properties38(BaseModel): + relative_path: RelativePath + absolute_path: AbsolutePath + checksum_md5: ChecksumMd5 + size_bytes: SizeBytes + + +class File(BaseModel): + description: str + field_comment: str = Field(..., alias="$comment") + type: str + required: list[str] + properties: Properties38 + + +class Definitions(BaseModel): + generic: Generic + fmu_time: FmuTime + class_: Class = Field(..., alias="class") + source: Source + version: Version + tracklog_event: TracklogEvent + tracklog: Tracklog + data: Data + display: Display + access: Access + masterdata: Masterdata + fmu: Fmu + file: File + + +class Source1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Version1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Data1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Tracklog1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Access1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Masterdata1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Class1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Properties39(BaseModel): + source: Source1 + version: Version1 + data: Data1 + tracklog: Tracklog1 + access: Access1 + masterdata: Masterdata1 + class_: Class1 = Field(..., alias="class") + + +class Not(BaseModel): + enum: list[str] + + +class Class2(BaseModel): + enum: list[str] | None = None + not_: Not | None = Field(default=None, alias="not") + + +class Model2(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Case1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Iteration1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Realization1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Workflow1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Aggregation1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Properties41(BaseModel): + model: Model2 + case: Case1 + iteration: Iteration1 | None = None + realization: Realization1 | None = None + workflow: Workflow1 | None = None + aggregation: Aggregation1 | None = None + + +class Not1(BaseModel): + required: list[str] + + +class Aggregation2(BaseModel): + not_: Not1 = Field(..., alias="not") + + +class Realization2(BaseModel): + not_: Not1 = Field(..., alias="not") + + +class Dependencies1(BaseModel): + aggregation: Aggregation2 + realization: Realization2 + + +class Fmu1(BaseModel): + description: str + field_comment: str = Field(..., alias="$comment") + type: str | None = None + required: list[str] + properties: Properties41 + dependencies: Dependencies1 | None = None + + +class Access2(BaseModel): + required: list[str] + + +class File1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Content1(BaseModel): + const: str + + +class Properties42(BaseModel): + content: Content1 + + +class If(BaseModel): + properties: Properties42 + + +class Then(BaseModel): + required: list[str] + + +class AllOfItem(BaseModel): + field_comment: str = Field(..., alias="$comment") + if_: If = Field(..., alias="if") + then: Then + + +class Data2(BaseModel): + field_comment: str = Field(..., alias="$comment") + all_of: list[AllOfItem] = Field(..., alias="allOf") + + +class Properties40(BaseModel): + class_: Class2 = Field(..., alias="class") + fmu: Fmu1 + access: Access2 + file: File1 | None = None + data: Data2 | None = None + + +class Class3(BaseModel): + enum: list[str] + + +class Properties43(BaseModel): + class_: Class3 = Field(..., alias="class") + + +class If1(BaseModel): + properties: Properties43 + + +class Data3(BaseModel): + required: list[str] + + +class Properties44(BaseModel): + data: Data3 + + +class Then1(BaseModel): + properties: Properties44 + + +class OneOfItem(BaseModel): + field_comment: str = Field(..., alias="$comment") + properties: Properties40 + required: list[str] | None = None + if_: If1 | None = Field(default=None, alias="if") + then: Then1 | None = None + + +class Model(BaseModel): + field_schema: str = Field(..., alias="$schema") + field_id: str = Field(..., alias="$id") + field_contractual: list[str] = Field(..., alias="$contractual") + definitions: Definitions + field_comment: str = Field(..., alias="$comment") + required: list[str] + properties: Properties39 + one_of: list[OneOfItem] = Field(..., alias="oneOf") diff --git a/src/fmu/dataio/models/090model.py b/src/fmu/dataio/models/090model.py new file mode 100644 index 000000000..49227a438 --- /dev/null +++ b/src/fmu/dataio/models/090model.py @@ -0,0 +1,1379 @@ +# generated by datamodel-codegen: +# filename: fmu_results.json +# version: 0.25.2 + +from __future__ import annotations + +from pydantic import BaseModel, Field + + +class Datetime(BaseModel): + type: str + examples: list[str] + + +class Uuid(BaseModel): + type: str + pattern: str + examples: list[str] + + +class Id(BaseModel): + title: str + type: str + examples: list[str] + + +class Properties1(BaseModel): + id: Id + + +class User(BaseModel): + type: str + required: list[str] + properties: Properties1 + + +class Items(BaseModel): + type: str + examples: list[str] + + +class FieldDescription(BaseModel): + field_comment: str = Field(..., alias="$comment") + type: str + items: Items + + +class Generic(BaseModel): + field_comment: str = Field(..., alias="$comment") + datetime: Datetime + uuid: Uuid + user: User + field_description: FieldDescription = Field(..., alias="_description") + + +class Value(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Label(BaseModel): + type: str + examples: list[str] + + +class Properties2(BaseModel): + value: Value + label: Label + + +class FmuTime(BaseModel): + title: str + description: str + type: list[str] + properties: Properties2 + + +class Class(BaseModel): + title: str + type: str + examples: list[str] + enum: list[str] + + +class Source(BaseModel): + field_comment: str = Field(..., alias="$comment") + description: str + type: str + const: str + + +class Version(BaseModel): + field_comment: str = Field(..., alias="$comment") + title: str + type: str + enum: list[str] + example: str + + +class Datetime1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class User1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Event(BaseModel): + type: str + examples: list[str] + + +class Properties3(BaseModel): + datetime: Datetime1 + user: User1 + event: Event + + +class TracklogEvent(BaseModel): + type: str + properties: Properties3 + + +class Items1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Tracklog(BaseModel): + type: str + items: Items1 + + +class Top(BaseModel): + required: list[str] + + +class Base(BaseModel): + required: list[str] + + +class Dependencies(BaseModel): + top: Top + base: Base + + +class Name(BaseModel): + type: str + description: str + examples: list[str] + + +class Stratigraphic(BaseModel): + field_comment: str = Field(..., alias="$comment") + type: str + description: str + examples: list[bool] + + +class Alias(BaseModel): + type: str + items: Items1 + + +class StratigraphicAlias(BaseModel): + type: str + items: Items1 + + +class Offset(BaseModel): + field_comment: str = Field(..., alias="$comment") + type: str + example: float + + +class Name1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Stratigraphic1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Offset1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Properties5(BaseModel): + name: Name1 + stratigraphic: Stratigraphic1 + offset: Offset1 + + +class Top1(BaseModel): + field_comment: str = Field(..., alias="$comment") + properties: Properties5 + + +class Properties6(BaseModel): + name: Name1 + stratigraphic: Stratigraphic1 + offset: Offset1 + + +class Base1(BaseModel): + field_comment: str = Field(..., alias="$comment") + properties: Properties6 + + +class Content(BaseModel): + type: str + description: str + examples: list[str] + + +class Tagname(BaseModel): + type: str + description: str + examples: list[str] + + +class Name3(BaseModel): + type: str + examples: list[str] + + +class Attribute(BaseModel): + type: str + examples: list[str] + + +class IsDiscrete(BaseModel): + type: str + examples: list[bool] + + +class Properties8(BaseModel): + name: Name3 + attribute: Attribute + is_discrete: IsDiscrete + + +class Items4(BaseModel): + type: str + properties: Properties8 + + +class Properties7(BaseModel): + field_comment: str = Field(..., alias="$comment") + description: str + type: str + items: Items4 + + +class Format(BaseModel): + type: str + examples: list[str] + + +class Layout(BaseModel): + type: str + examples: list[str] + + +class Unit(BaseModel): + type: list[str] + examples: list[str] + + +class VerticalDomain(BaseModel): + type: str + enum: list[str] + examples: list[str] + + +class DepthReference(BaseModel): + type: list[str] + examples: list[str] + enum: list[str | None] + + +class UndefIsZero(BaseModel): + description: str + type: str + examples: list[str] + + +class Name4(BaseModel): + type: list[str] + examples: list[str] + + +class Properties9(BaseModel): + name: Name4 + + +class GridModel(BaseModel): + type: str + properties: Properties9 + + +class Ncol(BaseModel): + field_comment: str = Field(..., alias="$comment") + type: str + examples: list[int] + + +class Nrow(BaseModel): + type: str + examples: list[int] + + +class Nlay(BaseModel): + type: str + examples: list[int] + + +class Xori(BaseModel): + type: str + examples: list[float] + + +class Yori(BaseModel): + type: str + examples: list[float] + + +class Xinc(BaseModel): + type: str + examples: list[float] + + +class Yflip(BaseModel): + type: str + field_comment: str = Field(..., alias="$comment") + enum: list[int] + examples: list[int] + + +class Rotation(BaseModel): + type: str + examples: list[str] + + +class Undef(BaseModel): + type: str + field_comment: str = Field(..., alias="$comment") + examples: list[float] + + +class Npolys(BaseModel): + description: str + type: str + examples: list[int] + + +class Size(BaseModel): + description: str + type: str + examples: list[int] + + +class Items5(BaseModel): + type: str + examples: list[str] + + +class Columns(BaseModel): + description: str + type: str + items: Items5 + + +class Properties10(BaseModel): + ncol: Ncol + nrow: Nrow + nlay: Nlay + xori: Xori + yori: Yori + xinc: Xinc + yflip: Yflip + rotation: Rotation + undef: Undef + npolys: Npolys + size: Size + columns: Columns + + +class Spec(BaseModel): + type: str + properties: Properties10 + + +class Xmin(BaseModel): + type: str + examples: list[float] + + +class Xmax(BaseModel): + type: str + examples: list[float] + + +class Ymin(BaseModel): + type: str + examples: list[float] + + +class Ymax(BaseModel): + type: str + examples: list[float] + + +class Zmin(BaseModel): + type: list[str] + examples: list[float | None] + + +class Zmax(BaseModel): + type: list[str] + examples: list[float | None] + + +class Properties11(BaseModel): + xmin: Xmin + xmax: Xmax + ymin: Ymin + ymax: Ymax + zmin: Zmin + zmax: Zmax + + +class Bbox(BaseModel): + type: str + required: list[str] + properties: Properties11 + + +class T0(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class T1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Properties12(BaseModel): + t0: T0 + t1: T1 + + +class Time(BaseModel): + type: str + properties: Properties12 + + +class IsPrediction(BaseModel): + title: str + field_comment: str = Field(..., alias="$comment") + type: str + examples: list[bool] + + +class IsObservation(BaseModel): + title: str + field_comment: str = Field(..., alias="$comment") + type: str + examples: list[bool] + + +class Contact(BaseModel): + type: str + enum: list[str] + field_comment: str = Field(..., alias="$comment") + examples: list[str] + + +class Truncated(BaseModel): + type: str + field_comment: str = Field(..., alias="$comment") + examples: list[bool] + + +class Properties13(BaseModel): + contact: Contact + truncated: Truncated + + +class FluidContact(BaseModel): + type: str + description: str + properties: Properties13 + + +class Contact1(BaseModel): + type: str + + +class Properties14(BaseModel): + contact: Contact1 + + +class FieldOutline(BaseModel): + description: str + type: str + required: list[str] + properties: Properties14 + + +class Id1(BaseModel): + description: str + type: str + + +class Properties15(BaseModel): + id: Id1 + + +class FieldRegion(BaseModel): + description: str + type: str + required: list[str] + properties: Properties15 + + +class Description(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Calculation(BaseModel): + type: str + examples: list[str] + + +class Zrange(BaseModel): + type: str + examples: list[float] + + +class FilterSize(BaseModel): + type: str + examples: list[float] + + +class ScalingFactor(BaseModel): + type: str + examples: list[float] + + +class StackingOffset(BaseModel): + type: str + examples: list[str] + + +class Properties16(BaseModel): + attribute: Attribute + calculation: Calculation + zrange: Zrange + filter_size: FilterSize + scaling_factor: ScalingFactor + stacking_offset: StackingOffset + + +class Seismic(BaseModel): + type: str + description: str + properties: Properties16 + + +class Properties4(BaseModel): + name: Name + stratigraphic: Stratigraphic + alias: Alias + stratigraphic_alias: StratigraphicAlias + offset: Offset + top: Top1 + base: Base1 + content: Content + tagname: Tagname + properties: Properties7 + format: Format + layout: Layout + unit: Unit + vertical_domain: VerticalDomain + depth_reference: DepthReference + undef_is_zero: UndefIsZero + grid_model: GridModel + spec: Spec + bbox: Bbox + time: Time + is_prediction: IsPrediction + is_observation: IsObservation + fluid_contact: FluidContact + field_outline: FieldOutline + field_region: FieldRegion + description: Description + seismic: Seismic + + +class Data(BaseModel): + type: str + title: str + required: list[str] + dependencies: Dependencies + properties: Properties4 + + +class Name5(BaseModel): + type: str + examples: list[str] + + +class Subtitle(BaseModel): + type: str + examples: list[str] + + +class Show(BaseModel): + type: str + examples: list[bool] + + +class Color(BaseModel): + type: str + examples: list[str] + + +class Properties18(BaseModel): + show: Show + color: Color + + +class Line(BaseModel): + type: str + properties: Properties18 + + +class Properties19(BaseModel): + show: Show + color: Color + + +class Points(BaseModel): + type: str + properties: Properties19 + + +class Color2(BaseModel): + description: str + type: str + examples: list[str] + + +class Increment(BaseModel): + description: str + type: str + examples: list[float] + + +class Properties20(BaseModel): + show: Show + color: Color2 + increment: Increment + + +class Contours(BaseModel): + type: str + properties: Properties20 + + +class Color3(BaseModel): + type: str + examples: list[str] + + +class Colormap(BaseModel): + field_comment: str = Field(..., alias="$comment") + description: str + type: str + examples: list[str] + + +class DisplayMin(BaseModel): + description: str + type: str + examples: list[float] + + +class DisplayMax(BaseModel): + description: str + type: str + examples: list[float] + + +class Properties21(BaseModel): + show: Show + color: Color3 + colormap: Colormap + display_min: DisplayMin + display_max: DisplayMax + + +class Fill(BaseModel): + type: str + properties: Properties21 + + +class Properties17(BaseModel): + name: Name5 + subtitle: Subtitle + line: Line + points: Points + contours: Contours + fill: Fill + + +class Display(BaseModel): + type: str + properties: Properties17 + + +class Properties23(BaseModel): + name: Name5 + + +class Asset(BaseModel): + type: str + required: list[str] + properties: Properties23 + + +class AccessLevel(BaseModel): + field_comment: str = Field(..., alias="$comment") + type: str + enum: list[str] + + +class RepInclude(BaseModel): + field_comment: str = Field(..., alias="$comment") + type: str + examples: list[bool] + + +class Properties24(BaseModel): + access_level: AccessLevel + rep_include: RepInclude + + +class Ssdl(BaseModel): + type: str + field_comment: str = Field(..., alias="$comment") + required: list[str] + properties: Properties24 + + +class Classification(BaseModel): + type: str + enum: list[str] + examples: list[str] + + +class Properties22(BaseModel): + asset: Asset + ssdl: Ssdl + classification: Classification + + +class Access(BaseModel): + type: str + properties: Properties22 + + +class Identifier(BaseModel): + type: str + examples: list[str] + + +class Uuid1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Properties27(BaseModel): + identifier: Identifier + uuid: Uuid1 + + +class Items6(BaseModel): + type: str + required: list[str] + properties: Properties27 + + +class Country(BaseModel): + field_comment: str = Field(..., alias="$comment") + type: str + items: Items6 + + +class ShortIdentifier(BaseModel): + type: str + examples: list[str] + + +class Properties28(BaseModel): + short_identifier: ShortIdentifier + uuid: Uuid1 + + +class Items7(BaseModel): + type: str + required: list[str] + properties: Properties28 + + +class Discovery(BaseModel): + field_comment: str = Field(..., alias="$comment") + type: str + items: Items7 + + +class Properties29(BaseModel): + identifier: Identifier + uuid: Uuid1 + + +class Items8(BaseModel): + type: str + required: list[str] + properties: Properties29 + + +class FieldModel(BaseModel): + field_comment: str = Field(..., alias="$comment") + type: str + items: Items8 + + +class Properties30(BaseModel): + identifier: Identifier + uuid: Uuid1 + + +class CoordinateSystem(BaseModel): + field_comment: str = Field(..., alias="$comment") + type: str + required: list[str] + properties: Properties30 + + +class Properties31(BaseModel): + identifier: Identifier + uuid: Uuid1 + + +class StratigraphicColumn(BaseModel): + field_comment: str = Field(..., alias="$comment") + type: str + required: list[str] + properties: Properties31 + + +class Properties26(BaseModel): + country: Country + discovery: Discovery + field: FieldModel + coordinate_system: CoordinateSystem + stratigraphic_column: StratigraphicColumn + + +class Smda(BaseModel): + type: str + required: list[str] + properties: Properties26 + + +class Properties25(BaseModel): + smda: Smda + + +class Masterdata(BaseModel): + field_comment: str = Field(..., alias="$comment") + type: str + required: list[str] + properties: Properties25 + + +class Revision(BaseModel): + type: str + examples: list[str] + + +class Description1(BaseModel): + description: str + field_ref: str = Field(..., alias="$ref") + + +class Properties32(BaseModel): + name: Name5 + revision: Revision + description: Description1 + + +class Model1(BaseModel): + type: str + properties: Properties32 + + +class Reference(BaseModel): + type: str + description: str + + +class Properties33(BaseModel): + reference: Reference + + +class Workflow(BaseModel): + type: str + properties: Properties33 + + +class Name8(BaseModel): + type: str + description: str + examples: list[str] + + +class User2(BaseModel): + description: str + field_ref: str = Field(..., alias="$ref") + + +class Description2(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Properties34(BaseModel): + name: Name8 + uuid: Uuid1 + user: User2 + description: Description2 + + +class Case(BaseModel): + type: str + required: list[str] + properties: Properties34 + + +class Name9(BaseModel): + description: str + type: str + examples: list[str] + + +class Id2(BaseModel): + description: str + type: str + examples: list[int] + + +class RestartFrom(BaseModel): + description: str + field_ref: str = Field(..., alias="$ref") + + +class Properties35(BaseModel): + name: Name9 + id: Id2 + uuid: Uuid1 + restart_from: RestartFrom + + +class Iteration(BaseModel): + type: str + required: list[str] + properties: Properties35 + + +class Id3(BaseModel): + type: str + description: str + examples: list[int] + + +class Items9(BaseModel): + type: str + + +class Parameters(BaseModel): + type: str + description: str + items: Items9 + + +class Jobs(BaseModel): + type: str + description: str + + +class Properties36(BaseModel): + name: Name9 + id: Id3 + uuid: Uuid1 + parameters: Parameters + jobs: Jobs + + +class Realization(BaseModel): + field_comment: str = Field(..., alias="$comment") + type: str + required: list[str] + properties: Properties36 + + +class Operation(BaseModel): + type: str + description: str + + +class Items10(BaseModel): + type: str + examples: list[int] + + +class RealizationIds(BaseModel): + type: str + description: str + items: Items10 + + +class Items11(BaseModel): + type: str + + +class Parameters1(BaseModel): + type: str + description: str + items: Items11 + + +class Id4(BaseModel): + type: str + description: str + field_comment: str = Field(..., alias="$comment") + examples: list[str] + + +class Properties37(BaseModel): + operation: Operation + realization_ids: RealizationIds + parameters: Parameters1 + id: Id4 + + +class Aggregation(BaseModel): + field_comment: str = Field(..., alias="$comment") + type: str + required: list[str] + properties: Properties37 + + +class Stage(BaseModel): + type: str + examples: list[str] + + +class Properties38(BaseModel): + stage: Stage + + +class Context(BaseModel): + description: str + type: str + required: list[str] + properties: Properties38 + + +class Fmu(BaseModel): + model: Model1 + workflow: Workflow + case: Case + iteration: Iteration + realization: Realization + aggregation: Aggregation + context: Context + + +class RelativePath(BaseModel): + type: str + description: str + examples: list[str] + + +class AbsolutePath(BaseModel): + type: str + description: str + examples: list[str] + + +class ChecksumMd5(BaseModel): + description: str + type: str + examples: list[str] + + +class SizeBytes(BaseModel): + description: str + type: str + examples: list[int] + + +class Properties39(BaseModel): + relative_path: RelativePath + absolute_path: AbsolutePath + checksum_md5: ChecksumMd5 + size_bytes: SizeBytes + + +class File(BaseModel): + description: str + field_comment: str = Field(..., alias="$comment") + type: str + required: list[str] + properties: Properties39 + + +class Properties(BaseModel): + generic: Generic + fmu_time: FmuTime + class_: Class = Field(..., alias="class") + source: Source + version: Version + tracklog_event: TracklogEvent + tracklog: Tracklog + data: Data + display: Display + access: Access + masterdata: Masterdata + fmu: Fmu + file: File + + +class Source1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Version1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Class1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Tracklog1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Data1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Access1(BaseModel): + field_ref: str = Field(..., alias="$ref") + required: list[str] + + +class Masterdata1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Model2(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Case2(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Properties41(BaseModel): + model: Model2 + case: Case2 + + +class Fmu1(BaseModel): + description: str + field_comment: str = Field(..., alias="$comment") + type: str + required: list[str] + properties: Properties41 + + +class Properties40(BaseModel): + source: Source1 + version: Version1 + class_: Class1 = Field(..., alias="class") + tracklog: Tracklog1 + data: Data1 + access: Access1 + masterdata: Masterdata1 + fmu: Fmu1 + + +class Case1(BaseModel): + required: list[str] + properties: Properties40 + + +class Case3(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Iteration1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Realization1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Workflow1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Aggregation1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Properties43(BaseModel): + model: Model2 + case: Case3 + iteration: Iteration1 + realization: Realization1 + workflow: Workflow1 + aggregation: Aggregation1 + + +class Not(BaseModel): + required: list[str] + + +class Aggregation2(BaseModel): + not_: Not = Field(..., alias="not") + + +class Realization2(BaseModel): + not_: Not = Field(..., alias="not") + + +class Dependencies1(BaseModel): + aggregation: Aggregation2 + realization: Realization2 + + +class Fmu2(BaseModel): + description: str + field_comment: str = Field(..., alias="$comment") + required: list[str] + properties: Properties43 + dependencies: Dependencies1 + + +class File1(BaseModel): + field_ref: str = Field(..., alias="$ref") + + +class Content1(BaseModel): + const: str + + +class Properties44(BaseModel): + content: Content1 + + +class If(BaseModel): + properties: Properties44 + + +class Then(BaseModel): + required: list[str] + + +class AllOfItem(BaseModel): + field_ref: str | None = Field(default=None, alias="$ref") + field_comment: str | None = Field(default=None, alias="$comment") + if_: If | None = Field(default=None, alias="if") + then: Then | None = None + + +class Data2(BaseModel): + field_comment: str = Field(..., alias="$comment") + all_of: list[AllOfItem] = Field(..., alias="allOf") + + +class Properties42(BaseModel): + fmu: Fmu2 + file: File1 + data: Data2 + access: Access1 + source: Source1 + version: Version1 + class_: Class1 = Field(..., alias="class") + tracklog: Tracklog1 + masterdata: Masterdata1 + + +class Class3(BaseModel): + enum: list[str] + + +class Properties45(BaseModel): + class_: Class3 = Field(..., alias="class") + + +class If1(BaseModel): + properties: Properties45 + + +class Data3(BaseModel): + required: list[str] + + +class Properties46(BaseModel): + data: Data3 + + +class Then1(BaseModel): + properties: Properties46 + + +class DataObject(BaseModel): + properties: Properties42 + if_: If1 = Field(..., alias="if") + then: Then1 + required: list[str] + + +class Subschemas(BaseModel): + case: Case1 + data_object: DataObject + + +class FieldDefs(BaseModel): + properties: Properties + subschemas: Subschemas + + +class Class4(BaseModel): + const: str + + +class Properties47(BaseModel): + class_: Class4 = Field(..., alias="class") + + +class If2(BaseModel): + properties: Properties47 + + +class Then2(BaseModel): + title: str + description: str + field_ref: str = Field(..., alias="$ref") + + +class Else(BaseModel): + title: str + description: str + field_ref: str = Field(..., alias="$ref") + + +class Model(BaseModel): + field_schema: str = Field(..., alias="$schema") + field_id: str = Field(..., alias="$id") + field_contractual: list[str] = Field(..., alias="$contractual") + field_defs: FieldDefs = Field(..., alias="$defs") + field_comment: str = Field(..., alias="$comment") + if_: If2 = Field(..., alias="if") + then: Then2 + else_: Else = Field(..., alias="else") diff --git a/src/fmu/dataio/models/v3.py b/src/fmu/dataio/models/v3.py new file mode 100644 index 000000000..4a9873be1 --- /dev/null +++ b/src/fmu/dataio/models/v3.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +from datetime import datetime +from typing import Literal + +from pydantic import BaseModel, Field +from typing_extensions import Annotated + + +class GeologicalModel(BaseModel): + type: Literal["Structural", "Rock"] + + +class RockGeologicalModel(GeologicalModel): + type: Literal["Rock"] = "Rock" + + +class StructuralGeologicalModel(GeologicalModel): + type: Literal["Structural"] = "Structural" + + +class Shape(BaseModel): + ncol: int = Field(ge=0) + nrow: int = Field(ge=0) + nlay: int = Field(ge=0) + + +class Orientation(BaseModel): + x: float + y: float + z: float + + +class Grid(BaseModel): + orientation: Orientation + shape: Shape + undef: float | None + + +class Range(BaseModel): + start: float + stop: float + + +class BoundingBox(BaseModel): + x: Range + y: Range + z: Range + + +class Header(BaseModel): + asset: str + created_at: datetime + created_by: str + version: int + + +class Payland(BaseModel): + type: Literal["fmu.everest", "fmu.ert"] + + +class FMUEverest(Payland): + type: Literal["fmu.everest"] = "fmu.everest" + + +class FMUErt(BaseModel): + type: Literal["fmu.ert"] = "fmu.ert" + model: Annotated[ + StructuralGeologicalModel | RockGeologicalModel, + Field(discriminator="type"), + ] + + +class Export(BaseModel): + header: Header + payload: Annotated[ + FMUEverest | FMUErt, + Field(discriminator="type"), + ] From 3f45ae6c4da7fe3eb26bff0f3433eeef4630a008 Mon Sep 17 00:00:00 2001 From: JB Lovland Date: Wed, 17 Jan 2024 14:01:06 +0100 Subject: [PATCH 04/25] Rename model to meta --- gen.sh | 9 +- mypy.ini | 1 + src/fmu/dataio/models/080model.py | 1332 ---------------------------- src/fmu/dataio/models/090model.py | 1379 ----------------------------- src/fmu/dataio/models/eninja.md | 16 + src/fmu/dataio/models/meta.py | 575 ++++++++++++ src/fmu/dataio/models/v3.py | 79 -- 7 files changed, 598 insertions(+), 2793 deletions(-) delete mode 100644 src/fmu/dataio/models/080model.py delete mode 100644 src/fmu/dataio/models/090model.py create mode 100644 src/fmu/dataio/models/eninja.md create mode 100644 src/fmu/dataio/models/meta.py delete mode 100644 src/fmu/dataio/models/v3.py diff --git a/gen.sh b/gen.sh index a8c618711..757815e26 100755 --- a/gen.sh +++ b/gen.sh @@ -1,19 +1,22 @@ #!/usr/bin/env bash datamodel-codegen \ + --collapse-root-models \ --disable-timestamp \ --enable-version-header \ + --enum-field-as-litera all \ --field-constraints \ --input $1 \ - --input-file-type "json" \ + --input-file-type jsonschema \ + --output src/fmu/dataio/models/meta.py \ --output-model-type pydantic_v2.BaseModel \ --snake-case-field \ --strict-nullable \ + --strip-default-none \ --target-python-version 3.8 \ --use-default-kwarg \ --use-double-quotes \ --use-schema-description \ --use-standard-collections \ --use-subclass-enum \ - --use-title-as-name \ - --use-union-operator + --use-title-as-name diff --git a/mypy.ini b/mypy.ini index 0fcd3fff5..2a56bf1a7 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,4 +1,5 @@ [mypy] +plugins = pydantic.mypy disallow_untyped_defs = True exclude = ^((tests|docs|examples|build)/|conftest.py?) extra_checks = True diff --git a/src/fmu/dataio/models/080model.py b/src/fmu/dataio/models/080model.py deleted file mode 100644 index d06b61230..000000000 --- a/src/fmu/dataio/models/080model.py +++ /dev/null @@ -1,1332 +0,0 @@ -# generated by datamodel-codegen: -# filename: fmu_results.json -# version: 0.25.2 - -from __future__ import annotations - -from pydantic import BaseModel, Field - - -class Datetime(BaseModel): - type: str - examples: list[str] - - -class Uuid(BaseModel): - type: str - pattern: str - examples: list[str] - - -class Id(BaseModel): - title: str - type: str - examples: list[str] - - -class Properties(BaseModel): - id: Id - - -class User(BaseModel): - type: str - required: list[str] - properties: Properties - - -class Items(BaseModel): - type: str - examples: list[str] - - -class FieldDescription(BaseModel): - field_comment: str = Field(..., alias="$comment") - type: str - items: Items - - -class Generic(BaseModel): - field_comment: str = Field(..., alias="$comment") - datetime: Datetime - uuid: Uuid - user: User - field_description: FieldDescription = Field(..., alias="_description") - - -class Value(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Label(BaseModel): - type: str - examples: list[str] - - -class Properties1(BaseModel): - value: Value - label: Label - - -class FmuTime(BaseModel): - title: str - description: str - type: list[str] - properties: Properties1 - - -class Class(BaseModel): - title: str - type: str - examples: list[str] - enum: list[str] - - -class Source(BaseModel): - field_comment: str = Field(..., alias="$comment") - description: str - type: str - const: str - - -class Version(BaseModel): - field_comment: str = Field(..., alias="$comment") - title: str - type: str - enum: list[str] - example: str - - -class Datetime1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class User1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Event(BaseModel): - type: str - examples: list[str] - - -class Properties2(BaseModel): - datetime: Datetime1 - user: User1 - event: Event - - -class TracklogEvent(BaseModel): - type: str - properties: Properties2 - - -class Items1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Tracklog(BaseModel): - type: str - items: Items1 - - -class Top(BaseModel): - required: list[str] - - -class Base(BaseModel): - required: list[str] - - -class Dependencies(BaseModel): - top: Top - base: Base - - -class Name(BaseModel): - type: str - description: str - examples: list[str] - - -class Stratigraphic(BaseModel): - field_comment: str = Field(..., alias="$comment") - type: str - description: str - examples: list[bool] - - -class Alias(BaseModel): - type: str - items: Items1 - - -class StratigraphicAlias(BaseModel): - type: str - items: Items1 - - -class Offset(BaseModel): - field_comment: str = Field(..., alias="$comment") - type: str - example: float - - -class Name1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Stratigraphic1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Offset1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Properties4(BaseModel): - name: Name1 - stratigraphic: Stratigraphic1 - offset: Offset1 - - -class Top1(BaseModel): - field_comment: str = Field(..., alias="$comment") - properties: Properties4 - - -class Properties5(BaseModel): - name: Name1 - stratigraphic: Stratigraphic1 - offset: Offset1 - - -class Base1(BaseModel): - field_comment: str = Field(..., alias="$comment") - properties: Properties5 - - -class Content(BaseModel): - type: str - description: str - enum: list[str] - examples: list[str] - - -class Tagname(BaseModel): - type: str - description: str - examples: list[str] - - -class Name3(BaseModel): - type: str - examples: list[str] - - -class Attribute(BaseModel): - type: str - examples: list[str] - - -class IsDiscrete(BaseModel): - type: str - examples: list[bool] - - -class Properties7(BaseModel): - name: Name3 - attribute: Attribute - is_discrete: IsDiscrete - - -class Items4(BaseModel): - type: str - properties: Properties7 - - -class Properties6(BaseModel): - field_comment: str = Field(..., alias="$comment") - description: str - type: str - items: Items4 - - -class Format(BaseModel): - type: str - examples: list[str] - - -class Layout(BaseModel): - type: str - examples: list[str] - - -class Unit(BaseModel): - type: list[str] - examples: list[str] - - -class VerticalDomain(BaseModel): - type: str - enum: list[str] - examples: list[str] - - -class DepthReference(BaseModel): - type: list[str] - examples: list[str] - enum: list[str | None] - - -class UndefIsZero(BaseModel): - description: str - type: str - examples: list[str] - - -class Name4(BaseModel): - type: list[str] - examples: list[str] - - -class Properties8(BaseModel): - name: Name4 - - -class GridModel(BaseModel): - type: str - properties: Properties8 - - -class Ncol(BaseModel): - field_comment: str = Field(..., alias="$comment") - type: str - examples: list[int] - - -class Nrow(BaseModel): - type: str - examples: list[int] - - -class Nlay(BaseModel): - type: str - examples: list[int] - - -class Xori(BaseModel): - type: str - examples: list[float] - - -class Yori(BaseModel): - type: str - examples: list[float] - - -class Xinc(BaseModel): - type: str - examples: list[float] - - -class Yflip(BaseModel): - type: str - field_comment: str = Field(..., alias="$comment") - enum: list[int] - examples: list[int] - - -class Rotation(BaseModel): - type: str - examples: list[str] - - -class Undef(BaseModel): - type: str - field_comment: str = Field(..., alias="$comment") - examples: list[float] - - -class Npolys(BaseModel): - description: str - type: str - examples: list[int] - - -class Size(BaseModel): - description: str - type: str - examples: list[int] - - -class Items5(BaseModel): - type: str - examples: list[str] - - -class Columns(BaseModel): - description: str - type: str - items: Items5 - - -class Properties9(BaseModel): - ncol: Ncol - nrow: Nrow - nlay: Nlay - xori: Xori - yori: Yori - xinc: Xinc - yflip: Yflip - rotation: Rotation - undef: Undef - npolys: Npolys - size: Size - columns: Columns - - -class Spec(BaseModel): - type: str - properties: Properties9 - - -class Xmin(BaseModel): - type: str - examples: list[float] - - -class Xmax(BaseModel): - type: str - examples: list[float] - - -class Ymin(BaseModel): - type: str - examples: list[float] - - -class Ymax(BaseModel): - type: str - examples: list[float] - - -class Zmin(BaseModel): - type: list[str] - examples: list[float | None] - - -class Zmax(BaseModel): - type: list[str] - examples: list[float | None] - - -class Properties10(BaseModel): - xmin: Xmin - xmax: Xmax - ymin: Ymin - ymax: Ymax - zmin: Zmin - zmax: Zmax - - -class Bbox(BaseModel): - type: str - required: list[str] - properties: Properties10 - - -class T0(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class T1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Properties11(BaseModel): - t0: T0 - t1: T1 - - -class Time(BaseModel): - type: str - properties: Properties11 - - -class IsPrediction(BaseModel): - title: str - field_comment: str = Field(..., alias="$comment") - type: str - examples: list[bool] - - -class IsObservation(BaseModel): - title: str - field_comment: str = Field(..., alias="$comment") - type: str - examples: list[bool] - - -class Contact(BaseModel): - type: str - enum: list[str] - field_comment: str = Field(..., alias="$comment") - examples: list[str] - - -class Truncated(BaseModel): - type: str - field_comment: str = Field(..., alias="$comment") - examples: list[bool] - - -class Properties12(BaseModel): - contact: Contact - truncated: Truncated - - -class FluidContact(BaseModel): - type: str - description: str - properties: Properties12 - - -class Contact1(BaseModel): - type: str - - -class Properties13(BaseModel): - contact: Contact1 - - -class FieldOutline(BaseModel): - description: str - type: str - required: list[str] - properties: Properties13 - - -class Id1(BaseModel): - description: str - type: str - - -class Properties14(BaseModel): - id: Id1 - - -class FieldRegion(BaseModel): - description: str - type: str - required: list[str] - properties: Properties14 - - -class Description(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Calculation(BaseModel): - type: str - examples: list[str] - - -class Zrange(BaseModel): - type: str - examples: list[float] - - -class FilterSize(BaseModel): - type: str - examples: list[float] - - -class ScalingFactor(BaseModel): - type: str - examples: list[float] - - -class StackingOffset(BaseModel): - type: str - examples: list[str] - - -class Properties15(BaseModel): - attribute: Attribute - calculation: Calculation - zrange: Zrange - filter_size: FilterSize - scaling_factor: ScalingFactor - stacking_offset: StackingOffset - - -class Seismic(BaseModel): - type: str - description: str - properties: Properties15 - - -class Properties3(BaseModel): - name: Name - stratigraphic: Stratigraphic - alias: Alias - stratigraphic_alias: StratigraphicAlias - offset: Offset - top: Top1 - base: Base1 - content: Content - tagname: Tagname - properties: Properties6 - format: Format - layout: Layout - unit: Unit - vertical_domain: VerticalDomain - depth_reference: DepthReference - undef_is_zero: UndefIsZero - grid_model: GridModel - spec: Spec - bbox: Bbox - time: Time - is_prediction: IsPrediction - is_observation: IsObservation - fluid_contact: FluidContact - field_outline: FieldOutline - field_region: FieldRegion - description: Description - seismic: Seismic - - -class Data(BaseModel): - type: str - title: str - required: list[str] - dependencies: Dependencies - properties: Properties3 - - -class Name5(BaseModel): - type: str - examples: list[str] - - -class Subtitle(BaseModel): - type: str - examples: list[str] - - -class Show(BaseModel): - type: str - examples: list[bool] - - -class Color(BaseModel): - type: str - examples: list[str] - - -class Properties17(BaseModel): - show: Show - color: Color - - -class Line(BaseModel): - type: str - properties: Properties17 - - -class Properties18(BaseModel): - show: Show - color: Color - - -class Points(BaseModel): - type: str - properties: Properties18 - - -class Color2(BaseModel): - description: str - type: str - examples: list[str] - - -class Increment(BaseModel): - description: str - type: str - examples: list[float] - - -class Properties19(BaseModel): - show: Show - color: Color2 - increment: Increment - - -class Contours(BaseModel): - type: str - properties: Properties19 - - -class Color3(BaseModel): - type: str - examples: list[str] - - -class Colormap(BaseModel): - field_comment: str = Field(..., alias="$comment") - description: str - type: str - examples: list[str] - - -class DisplayMin(BaseModel): - description: str - type: str - examples: list[float] - - -class DisplayMax(BaseModel): - description: str - type: str - examples: list[float] - - -class Properties20(BaseModel): - show: Show - color: Color3 - colormap: Colormap - display_min: DisplayMin - display_max: DisplayMax - - -class Fill(BaseModel): - type: str - properties: Properties20 - - -class Properties16(BaseModel): - name: Name5 - subtitle: Subtitle - line: Line - points: Points - contours: Contours - fill: Fill - - -class Display(BaseModel): - type: str - properties: Properties16 - - -class Properties22(BaseModel): - name: Name5 - - -class Asset(BaseModel): - type: str - required: list[str] - properties: Properties22 - - -class AccessLevel(BaseModel): - field_comment: str = Field(..., alias="$comment") - type: str - enum: list[str] - - -class RepInclude(BaseModel): - field_comment: str = Field(..., alias="$comment") - type: str - examples: list[bool] - - -class Properties23(BaseModel): - access_level: AccessLevel - rep_include: RepInclude - - -class Ssdl(BaseModel): - type: str - field_comment: str = Field(..., alias="$comment") - required: list[str] - properties: Properties23 - - -class Classification(BaseModel): - type: str - enum: list[str] - examples: list[str] - - -class Properties21(BaseModel): - asset: Asset - ssdl: Ssdl - classification: Classification - - -class Access(BaseModel): - type: str - properties: Properties21 - - -class Identifier(BaseModel): - type: str - examples: list[str] - - -class Uuid1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Properties26(BaseModel): - identifier: Identifier - uuid: Uuid1 - - -class Items6(BaseModel): - type: str - required: list[str] - properties: Properties26 - - -class Country(BaseModel): - field_comment: str = Field(..., alias="$comment") - type: str - items: Items6 - - -class ShortIdentifier(BaseModel): - type: str - examples: list[str] - - -class Properties27(BaseModel): - short_identifier: ShortIdentifier - uuid: Uuid1 - - -class Items7(BaseModel): - type: str - required: list[str] - properties: Properties27 - - -class Discovery(BaseModel): - field_comment: str = Field(..., alias="$comment") - type: str - items: Items7 - - -class Properties28(BaseModel): - identifier: Identifier - uuid: Uuid1 - - -class Items8(BaseModel): - type: str - required: list[str] - properties: Properties28 - - -class FieldModel(BaseModel): - field_comment: str = Field(..., alias="$comment") - type: str - items: Items8 - - -class Properties29(BaseModel): - identifier: Identifier - uuid: Uuid1 - - -class CoordinateSystem(BaseModel): - field_comment: str = Field(..., alias="$comment") - type: str - required: list[str] - properties: Properties29 - - -class Properties30(BaseModel): - identifier: Identifier - uuid: Uuid1 - - -class StratigraphicColumn(BaseModel): - field_comment: str = Field(..., alias="$comment") - type: str - required: list[str] - properties: Properties30 - - -class Properties25(BaseModel): - country: Country - discovery: Discovery - field: FieldModel - coordinate_system: CoordinateSystem - stratigraphic_column: StratigraphicColumn - - -class Smda(BaseModel): - type: str - required: list[str] - properties: Properties25 - - -class Properties24(BaseModel): - smda: Smda - - -class Masterdata(BaseModel): - field_comment: str = Field(..., alias="$comment") - type: str - required: list[str] - properties: Properties24 - - -class Revision(BaseModel): - type: str - examples: list[str] - - -class Description1(BaseModel): - description: str - field_ref: str = Field(..., alias="$ref") - - -class Properties31(BaseModel): - name: Name5 - revision: Revision - description: Description1 - - -class Model1(BaseModel): - type: str - properties: Properties31 - - -class Reference(BaseModel): - type: str - description: str - - -class Properties32(BaseModel): - reference: Reference - - -class Workflow(BaseModel): - type: str - properties: Properties32 - - -class Name8(BaseModel): - type: str - description: str - examples: list[str] - - -class User2(BaseModel): - description: str - field_ref: str = Field(..., alias="$ref") - - -class Description2(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Properties33(BaseModel): - name: Name8 - uuid: Uuid1 - user: User2 - description: Description2 - - -class Case(BaseModel): - type: str - required: list[str] - properties: Properties33 - - -class Name9(BaseModel): - description: str - type: str - examples: list[str] - - -class Id2(BaseModel): - description: str - type: str - examples: list[int] - - -class RestartFrom(BaseModel): - description: str - field_ref: str = Field(..., alias="$ref") - - -class Properties34(BaseModel): - name: Name9 - id: Id2 - uuid: Uuid1 - restart_from: RestartFrom - - -class Iteration(BaseModel): - type: str - required: list[str] - properties: Properties34 - - -class Id3(BaseModel): - type: str - description: str - examples: list[int] - - -class Items9(BaseModel): - type: str - - -class Parameters(BaseModel): - type: str - description: str - items: Items9 - - -class Jobs(BaseModel): - type: str - description: str - - -class Properties35(BaseModel): - name: Name9 - id: Id3 - uuid: Uuid1 - parameters: Parameters - jobs: Jobs - - -class Realization(BaseModel): - field_comment: str = Field(..., alias="$comment") - type: str - required: list[str] - properties: Properties35 - - -class Operation(BaseModel): - type: str - description: str - - -class Items10(BaseModel): - type: str - examples: list[int] - - -class RealizationIds(BaseModel): - type: str - description: str - items: Items10 - - -class Items11(BaseModel): - type: str - - -class Parameters1(BaseModel): - type: str - description: str - items: Items11 - - -class Id4(BaseModel): - type: str - description: str - field_comment: str = Field(..., alias="$comment") - examples: list[str] - - -class Properties36(BaseModel): - operation: Operation - realization_ids: RealizationIds - parameters: Parameters1 - id: Id4 - - -class Aggregation(BaseModel): - field_comment: str = Field(..., alias="$comment") - type: str - required: list[str] - properties: Properties36 - - -class Stage(BaseModel): - type: str - examples: list[str] - - -class Properties37(BaseModel): - stage: Stage - - -class Context(BaseModel): - description: str - type: str - required: list[str] - properties: Properties37 - - -class Fmu(BaseModel): - model: Model1 - workflow: Workflow - case: Case - iteration: Iteration - realization: Realization - aggregation: Aggregation - context: Context - - -class RelativePath(BaseModel): - type: str - description: str - examples: list[str] - - -class AbsolutePath(BaseModel): - type: str - description: str - examples: list[str] - - -class ChecksumMd5(BaseModel): - description: str - type: str - examples: list[str] - - -class SizeBytes(BaseModel): - description: str - type: str - examples: list[int] - - -class Properties38(BaseModel): - relative_path: RelativePath - absolute_path: AbsolutePath - checksum_md5: ChecksumMd5 - size_bytes: SizeBytes - - -class File(BaseModel): - description: str - field_comment: str = Field(..., alias="$comment") - type: str - required: list[str] - properties: Properties38 - - -class Definitions(BaseModel): - generic: Generic - fmu_time: FmuTime - class_: Class = Field(..., alias="class") - source: Source - version: Version - tracklog_event: TracklogEvent - tracklog: Tracklog - data: Data - display: Display - access: Access - masterdata: Masterdata - fmu: Fmu - file: File - - -class Source1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Version1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Data1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Tracklog1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Access1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Masterdata1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Class1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Properties39(BaseModel): - source: Source1 - version: Version1 - data: Data1 - tracklog: Tracklog1 - access: Access1 - masterdata: Masterdata1 - class_: Class1 = Field(..., alias="class") - - -class Not(BaseModel): - enum: list[str] - - -class Class2(BaseModel): - enum: list[str] | None = None - not_: Not | None = Field(default=None, alias="not") - - -class Model2(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Case1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Iteration1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Realization1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Workflow1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Aggregation1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Properties41(BaseModel): - model: Model2 - case: Case1 - iteration: Iteration1 | None = None - realization: Realization1 | None = None - workflow: Workflow1 | None = None - aggregation: Aggregation1 | None = None - - -class Not1(BaseModel): - required: list[str] - - -class Aggregation2(BaseModel): - not_: Not1 = Field(..., alias="not") - - -class Realization2(BaseModel): - not_: Not1 = Field(..., alias="not") - - -class Dependencies1(BaseModel): - aggregation: Aggregation2 - realization: Realization2 - - -class Fmu1(BaseModel): - description: str - field_comment: str = Field(..., alias="$comment") - type: str | None = None - required: list[str] - properties: Properties41 - dependencies: Dependencies1 | None = None - - -class Access2(BaseModel): - required: list[str] - - -class File1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Content1(BaseModel): - const: str - - -class Properties42(BaseModel): - content: Content1 - - -class If(BaseModel): - properties: Properties42 - - -class Then(BaseModel): - required: list[str] - - -class AllOfItem(BaseModel): - field_comment: str = Field(..., alias="$comment") - if_: If = Field(..., alias="if") - then: Then - - -class Data2(BaseModel): - field_comment: str = Field(..., alias="$comment") - all_of: list[AllOfItem] = Field(..., alias="allOf") - - -class Properties40(BaseModel): - class_: Class2 = Field(..., alias="class") - fmu: Fmu1 - access: Access2 - file: File1 | None = None - data: Data2 | None = None - - -class Class3(BaseModel): - enum: list[str] - - -class Properties43(BaseModel): - class_: Class3 = Field(..., alias="class") - - -class If1(BaseModel): - properties: Properties43 - - -class Data3(BaseModel): - required: list[str] - - -class Properties44(BaseModel): - data: Data3 - - -class Then1(BaseModel): - properties: Properties44 - - -class OneOfItem(BaseModel): - field_comment: str = Field(..., alias="$comment") - properties: Properties40 - required: list[str] | None = None - if_: If1 | None = Field(default=None, alias="if") - then: Then1 | None = None - - -class Model(BaseModel): - field_schema: str = Field(..., alias="$schema") - field_id: str = Field(..., alias="$id") - field_contractual: list[str] = Field(..., alias="$contractual") - definitions: Definitions - field_comment: str = Field(..., alias="$comment") - required: list[str] - properties: Properties39 - one_of: list[OneOfItem] = Field(..., alias="oneOf") diff --git a/src/fmu/dataio/models/090model.py b/src/fmu/dataio/models/090model.py deleted file mode 100644 index 49227a438..000000000 --- a/src/fmu/dataio/models/090model.py +++ /dev/null @@ -1,1379 +0,0 @@ -# generated by datamodel-codegen: -# filename: fmu_results.json -# version: 0.25.2 - -from __future__ import annotations - -from pydantic import BaseModel, Field - - -class Datetime(BaseModel): - type: str - examples: list[str] - - -class Uuid(BaseModel): - type: str - pattern: str - examples: list[str] - - -class Id(BaseModel): - title: str - type: str - examples: list[str] - - -class Properties1(BaseModel): - id: Id - - -class User(BaseModel): - type: str - required: list[str] - properties: Properties1 - - -class Items(BaseModel): - type: str - examples: list[str] - - -class FieldDescription(BaseModel): - field_comment: str = Field(..., alias="$comment") - type: str - items: Items - - -class Generic(BaseModel): - field_comment: str = Field(..., alias="$comment") - datetime: Datetime - uuid: Uuid - user: User - field_description: FieldDescription = Field(..., alias="_description") - - -class Value(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Label(BaseModel): - type: str - examples: list[str] - - -class Properties2(BaseModel): - value: Value - label: Label - - -class FmuTime(BaseModel): - title: str - description: str - type: list[str] - properties: Properties2 - - -class Class(BaseModel): - title: str - type: str - examples: list[str] - enum: list[str] - - -class Source(BaseModel): - field_comment: str = Field(..., alias="$comment") - description: str - type: str - const: str - - -class Version(BaseModel): - field_comment: str = Field(..., alias="$comment") - title: str - type: str - enum: list[str] - example: str - - -class Datetime1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class User1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Event(BaseModel): - type: str - examples: list[str] - - -class Properties3(BaseModel): - datetime: Datetime1 - user: User1 - event: Event - - -class TracklogEvent(BaseModel): - type: str - properties: Properties3 - - -class Items1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Tracklog(BaseModel): - type: str - items: Items1 - - -class Top(BaseModel): - required: list[str] - - -class Base(BaseModel): - required: list[str] - - -class Dependencies(BaseModel): - top: Top - base: Base - - -class Name(BaseModel): - type: str - description: str - examples: list[str] - - -class Stratigraphic(BaseModel): - field_comment: str = Field(..., alias="$comment") - type: str - description: str - examples: list[bool] - - -class Alias(BaseModel): - type: str - items: Items1 - - -class StratigraphicAlias(BaseModel): - type: str - items: Items1 - - -class Offset(BaseModel): - field_comment: str = Field(..., alias="$comment") - type: str - example: float - - -class Name1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Stratigraphic1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Offset1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Properties5(BaseModel): - name: Name1 - stratigraphic: Stratigraphic1 - offset: Offset1 - - -class Top1(BaseModel): - field_comment: str = Field(..., alias="$comment") - properties: Properties5 - - -class Properties6(BaseModel): - name: Name1 - stratigraphic: Stratigraphic1 - offset: Offset1 - - -class Base1(BaseModel): - field_comment: str = Field(..., alias="$comment") - properties: Properties6 - - -class Content(BaseModel): - type: str - description: str - examples: list[str] - - -class Tagname(BaseModel): - type: str - description: str - examples: list[str] - - -class Name3(BaseModel): - type: str - examples: list[str] - - -class Attribute(BaseModel): - type: str - examples: list[str] - - -class IsDiscrete(BaseModel): - type: str - examples: list[bool] - - -class Properties8(BaseModel): - name: Name3 - attribute: Attribute - is_discrete: IsDiscrete - - -class Items4(BaseModel): - type: str - properties: Properties8 - - -class Properties7(BaseModel): - field_comment: str = Field(..., alias="$comment") - description: str - type: str - items: Items4 - - -class Format(BaseModel): - type: str - examples: list[str] - - -class Layout(BaseModel): - type: str - examples: list[str] - - -class Unit(BaseModel): - type: list[str] - examples: list[str] - - -class VerticalDomain(BaseModel): - type: str - enum: list[str] - examples: list[str] - - -class DepthReference(BaseModel): - type: list[str] - examples: list[str] - enum: list[str | None] - - -class UndefIsZero(BaseModel): - description: str - type: str - examples: list[str] - - -class Name4(BaseModel): - type: list[str] - examples: list[str] - - -class Properties9(BaseModel): - name: Name4 - - -class GridModel(BaseModel): - type: str - properties: Properties9 - - -class Ncol(BaseModel): - field_comment: str = Field(..., alias="$comment") - type: str - examples: list[int] - - -class Nrow(BaseModel): - type: str - examples: list[int] - - -class Nlay(BaseModel): - type: str - examples: list[int] - - -class Xori(BaseModel): - type: str - examples: list[float] - - -class Yori(BaseModel): - type: str - examples: list[float] - - -class Xinc(BaseModel): - type: str - examples: list[float] - - -class Yflip(BaseModel): - type: str - field_comment: str = Field(..., alias="$comment") - enum: list[int] - examples: list[int] - - -class Rotation(BaseModel): - type: str - examples: list[str] - - -class Undef(BaseModel): - type: str - field_comment: str = Field(..., alias="$comment") - examples: list[float] - - -class Npolys(BaseModel): - description: str - type: str - examples: list[int] - - -class Size(BaseModel): - description: str - type: str - examples: list[int] - - -class Items5(BaseModel): - type: str - examples: list[str] - - -class Columns(BaseModel): - description: str - type: str - items: Items5 - - -class Properties10(BaseModel): - ncol: Ncol - nrow: Nrow - nlay: Nlay - xori: Xori - yori: Yori - xinc: Xinc - yflip: Yflip - rotation: Rotation - undef: Undef - npolys: Npolys - size: Size - columns: Columns - - -class Spec(BaseModel): - type: str - properties: Properties10 - - -class Xmin(BaseModel): - type: str - examples: list[float] - - -class Xmax(BaseModel): - type: str - examples: list[float] - - -class Ymin(BaseModel): - type: str - examples: list[float] - - -class Ymax(BaseModel): - type: str - examples: list[float] - - -class Zmin(BaseModel): - type: list[str] - examples: list[float | None] - - -class Zmax(BaseModel): - type: list[str] - examples: list[float | None] - - -class Properties11(BaseModel): - xmin: Xmin - xmax: Xmax - ymin: Ymin - ymax: Ymax - zmin: Zmin - zmax: Zmax - - -class Bbox(BaseModel): - type: str - required: list[str] - properties: Properties11 - - -class T0(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class T1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Properties12(BaseModel): - t0: T0 - t1: T1 - - -class Time(BaseModel): - type: str - properties: Properties12 - - -class IsPrediction(BaseModel): - title: str - field_comment: str = Field(..., alias="$comment") - type: str - examples: list[bool] - - -class IsObservation(BaseModel): - title: str - field_comment: str = Field(..., alias="$comment") - type: str - examples: list[bool] - - -class Contact(BaseModel): - type: str - enum: list[str] - field_comment: str = Field(..., alias="$comment") - examples: list[str] - - -class Truncated(BaseModel): - type: str - field_comment: str = Field(..., alias="$comment") - examples: list[bool] - - -class Properties13(BaseModel): - contact: Contact - truncated: Truncated - - -class FluidContact(BaseModel): - type: str - description: str - properties: Properties13 - - -class Contact1(BaseModel): - type: str - - -class Properties14(BaseModel): - contact: Contact1 - - -class FieldOutline(BaseModel): - description: str - type: str - required: list[str] - properties: Properties14 - - -class Id1(BaseModel): - description: str - type: str - - -class Properties15(BaseModel): - id: Id1 - - -class FieldRegion(BaseModel): - description: str - type: str - required: list[str] - properties: Properties15 - - -class Description(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Calculation(BaseModel): - type: str - examples: list[str] - - -class Zrange(BaseModel): - type: str - examples: list[float] - - -class FilterSize(BaseModel): - type: str - examples: list[float] - - -class ScalingFactor(BaseModel): - type: str - examples: list[float] - - -class StackingOffset(BaseModel): - type: str - examples: list[str] - - -class Properties16(BaseModel): - attribute: Attribute - calculation: Calculation - zrange: Zrange - filter_size: FilterSize - scaling_factor: ScalingFactor - stacking_offset: StackingOffset - - -class Seismic(BaseModel): - type: str - description: str - properties: Properties16 - - -class Properties4(BaseModel): - name: Name - stratigraphic: Stratigraphic - alias: Alias - stratigraphic_alias: StratigraphicAlias - offset: Offset - top: Top1 - base: Base1 - content: Content - tagname: Tagname - properties: Properties7 - format: Format - layout: Layout - unit: Unit - vertical_domain: VerticalDomain - depth_reference: DepthReference - undef_is_zero: UndefIsZero - grid_model: GridModel - spec: Spec - bbox: Bbox - time: Time - is_prediction: IsPrediction - is_observation: IsObservation - fluid_contact: FluidContact - field_outline: FieldOutline - field_region: FieldRegion - description: Description - seismic: Seismic - - -class Data(BaseModel): - type: str - title: str - required: list[str] - dependencies: Dependencies - properties: Properties4 - - -class Name5(BaseModel): - type: str - examples: list[str] - - -class Subtitle(BaseModel): - type: str - examples: list[str] - - -class Show(BaseModel): - type: str - examples: list[bool] - - -class Color(BaseModel): - type: str - examples: list[str] - - -class Properties18(BaseModel): - show: Show - color: Color - - -class Line(BaseModel): - type: str - properties: Properties18 - - -class Properties19(BaseModel): - show: Show - color: Color - - -class Points(BaseModel): - type: str - properties: Properties19 - - -class Color2(BaseModel): - description: str - type: str - examples: list[str] - - -class Increment(BaseModel): - description: str - type: str - examples: list[float] - - -class Properties20(BaseModel): - show: Show - color: Color2 - increment: Increment - - -class Contours(BaseModel): - type: str - properties: Properties20 - - -class Color3(BaseModel): - type: str - examples: list[str] - - -class Colormap(BaseModel): - field_comment: str = Field(..., alias="$comment") - description: str - type: str - examples: list[str] - - -class DisplayMin(BaseModel): - description: str - type: str - examples: list[float] - - -class DisplayMax(BaseModel): - description: str - type: str - examples: list[float] - - -class Properties21(BaseModel): - show: Show - color: Color3 - colormap: Colormap - display_min: DisplayMin - display_max: DisplayMax - - -class Fill(BaseModel): - type: str - properties: Properties21 - - -class Properties17(BaseModel): - name: Name5 - subtitle: Subtitle - line: Line - points: Points - contours: Contours - fill: Fill - - -class Display(BaseModel): - type: str - properties: Properties17 - - -class Properties23(BaseModel): - name: Name5 - - -class Asset(BaseModel): - type: str - required: list[str] - properties: Properties23 - - -class AccessLevel(BaseModel): - field_comment: str = Field(..., alias="$comment") - type: str - enum: list[str] - - -class RepInclude(BaseModel): - field_comment: str = Field(..., alias="$comment") - type: str - examples: list[bool] - - -class Properties24(BaseModel): - access_level: AccessLevel - rep_include: RepInclude - - -class Ssdl(BaseModel): - type: str - field_comment: str = Field(..., alias="$comment") - required: list[str] - properties: Properties24 - - -class Classification(BaseModel): - type: str - enum: list[str] - examples: list[str] - - -class Properties22(BaseModel): - asset: Asset - ssdl: Ssdl - classification: Classification - - -class Access(BaseModel): - type: str - properties: Properties22 - - -class Identifier(BaseModel): - type: str - examples: list[str] - - -class Uuid1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Properties27(BaseModel): - identifier: Identifier - uuid: Uuid1 - - -class Items6(BaseModel): - type: str - required: list[str] - properties: Properties27 - - -class Country(BaseModel): - field_comment: str = Field(..., alias="$comment") - type: str - items: Items6 - - -class ShortIdentifier(BaseModel): - type: str - examples: list[str] - - -class Properties28(BaseModel): - short_identifier: ShortIdentifier - uuid: Uuid1 - - -class Items7(BaseModel): - type: str - required: list[str] - properties: Properties28 - - -class Discovery(BaseModel): - field_comment: str = Field(..., alias="$comment") - type: str - items: Items7 - - -class Properties29(BaseModel): - identifier: Identifier - uuid: Uuid1 - - -class Items8(BaseModel): - type: str - required: list[str] - properties: Properties29 - - -class FieldModel(BaseModel): - field_comment: str = Field(..., alias="$comment") - type: str - items: Items8 - - -class Properties30(BaseModel): - identifier: Identifier - uuid: Uuid1 - - -class CoordinateSystem(BaseModel): - field_comment: str = Field(..., alias="$comment") - type: str - required: list[str] - properties: Properties30 - - -class Properties31(BaseModel): - identifier: Identifier - uuid: Uuid1 - - -class StratigraphicColumn(BaseModel): - field_comment: str = Field(..., alias="$comment") - type: str - required: list[str] - properties: Properties31 - - -class Properties26(BaseModel): - country: Country - discovery: Discovery - field: FieldModel - coordinate_system: CoordinateSystem - stratigraphic_column: StratigraphicColumn - - -class Smda(BaseModel): - type: str - required: list[str] - properties: Properties26 - - -class Properties25(BaseModel): - smda: Smda - - -class Masterdata(BaseModel): - field_comment: str = Field(..., alias="$comment") - type: str - required: list[str] - properties: Properties25 - - -class Revision(BaseModel): - type: str - examples: list[str] - - -class Description1(BaseModel): - description: str - field_ref: str = Field(..., alias="$ref") - - -class Properties32(BaseModel): - name: Name5 - revision: Revision - description: Description1 - - -class Model1(BaseModel): - type: str - properties: Properties32 - - -class Reference(BaseModel): - type: str - description: str - - -class Properties33(BaseModel): - reference: Reference - - -class Workflow(BaseModel): - type: str - properties: Properties33 - - -class Name8(BaseModel): - type: str - description: str - examples: list[str] - - -class User2(BaseModel): - description: str - field_ref: str = Field(..., alias="$ref") - - -class Description2(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Properties34(BaseModel): - name: Name8 - uuid: Uuid1 - user: User2 - description: Description2 - - -class Case(BaseModel): - type: str - required: list[str] - properties: Properties34 - - -class Name9(BaseModel): - description: str - type: str - examples: list[str] - - -class Id2(BaseModel): - description: str - type: str - examples: list[int] - - -class RestartFrom(BaseModel): - description: str - field_ref: str = Field(..., alias="$ref") - - -class Properties35(BaseModel): - name: Name9 - id: Id2 - uuid: Uuid1 - restart_from: RestartFrom - - -class Iteration(BaseModel): - type: str - required: list[str] - properties: Properties35 - - -class Id3(BaseModel): - type: str - description: str - examples: list[int] - - -class Items9(BaseModel): - type: str - - -class Parameters(BaseModel): - type: str - description: str - items: Items9 - - -class Jobs(BaseModel): - type: str - description: str - - -class Properties36(BaseModel): - name: Name9 - id: Id3 - uuid: Uuid1 - parameters: Parameters - jobs: Jobs - - -class Realization(BaseModel): - field_comment: str = Field(..., alias="$comment") - type: str - required: list[str] - properties: Properties36 - - -class Operation(BaseModel): - type: str - description: str - - -class Items10(BaseModel): - type: str - examples: list[int] - - -class RealizationIds(BaseModel): - type: str - description: str - items: Items10 - - -class Items11(BaseModel): - type: str - - -class Parameters1(BaseModel): - type: str - description: str - items: Items11 - - -class Id4(BaseModel): - type: str - description: str - field_comment: str = Field(..., alias="$comment") - examples: list[str] - - -class Properties37(BaseModel): - operation: Operation - realization_ids: RealizationIds - parameters: Parameters1 - id: Id4 - - -class Aggregation(BaseModel): - field_comment: str = Field(..., alias="$comment") - type: str - required: list[str] - properties: Properties37 - - -class Stage(BaseModel): - type: str - examples: list[str] - - -class Properties38(BaseModel): - stage: Stage - - -class Context(BaseModel): - description: str - type: str - required: list[str] - properties: Properties38 - - -class Fmu(BaseModel): - model: Model1 - workflow: Workflow - case: Case - iteration: Iteration - realization: Realization - aggregation: Aggregation - context: Context - - -class RelativePath(BaseModel): - type: str - description: str - examples: list[str] - - -class AbsolutePath(BaseModel): - type: str - description: str - examples: list[str] - - -class ChecksumMd5(BaseModel): - description: str - type: str - examples: list[str] - - -class SizeBytes(BaseModel): - description: str - type: str - examples: list[int] - - -class Properties39(BaseModel): - relative_path: RelativePath - absolute_path: AbsolutePath - checksum_md5: ChecksumMd5 - size_bytes: SizeBytes - - -class File(BaseModel): - description: str - field_comment: str = Field(..., alias="$comment") - type: str - required: list[str] - properties: Properties39 - - -class Properties(BaseModel): - generic: Generic - fmu_time: FmuTime - class_: Class = Field(..., alias="class") - source: Source - version: Version - tracklog_event: TracklogEvent - tracklog: Tracklog - data: Data - display: Display - access: Access - masterdata: Masterdata - fmu: Fmu - file: File - - -class Source1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Version1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Class1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Tracklog1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Data1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Access1(BaseModel): - field_ref: str = Field(..., alias="$ref") - required: list[str] - - -class Masterdata1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Model2(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Case2(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Properties41(BaseModel): - model: Model2 - case: Case2 - - -class Fmu1(BaseModel): - description: str - field_comment: str = Field(..., alias="$comment") - type: str - required: list[str] - properties: Properties41 - - -class Properties40(BaseModel): - source: Source1 - version: Version1 - class_: Class1 = Field(..., alias="class") - tracklog: Tracklog1 - data: Data1 - access: Access1 - masterdata: Masterdata1 - fmu: Fmu1 - - -class Case1(BaseModel): - required: list[str] - properties: Properties40 - - -class Case3(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Iteration1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Realization1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Workflow1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Aggregation1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Properties43(BaseModel): - model: Model2 - case: Case3 - iteration: Iteration1 - realization: Realization1 - workflow: Workflow1 - aggregation: Aggregation1 - - -class Not(BaseModel): - required: list[str] - - -class Aggregation2(BaseModel): - not_: Not = Field(..., alias="not") - - -class Realization2(BaseModel): - not_: Not = Field(..., alias="not") - - -class Dependencies1(BaseModel): - aggregation: Aggregation2 - realization: Realization2 - - -class Fmu2(BaseModel): - description: str - field_comment: str = Field(..., alias="$comment") - required: list[str] - properties: Properties43 - dependencies: Dependencies1 - - -class File1(BaseModel): - field_ref: str = Field(..., alias="$ref") - - -class Content1(BaseModel): - const: str - - -class Properties44(BaseModel): - content: Content1 - - -class If(BaseModel): - properties: Properties44 - - -class Then(BaseModel): - required: list[str] - - -class AllOfItem(BaseModel): - field_ref: str | None = Field(default=None, alias="$ref") - field_comment: str | None = Field(default=None, alias="$comment") - if_: If | None = Field(default=None, alias="if") - then: Then | None = None - - -class Data2(BaseModel): - field_comment: str = Field(..., alias="$comment") - all_of: list[AllOfItem] = Field(..., alias="allOf") - - -class Properties42(BaseModel): - fmu: Fmu2 - file: File1 - data: Data2 - access: Access1 - source: Source1 - version: Version1 - class_: Class1 = Field(..., alias="class") - tracklog: Tracklog1 - masterdata: Masterdata1 - - -class Class3(BaseModel): - enum: list[str] - - -class Properties45(BaseModel): - class_: Class3 = Field(..., alias="class") - - -class If1(BaseModel): - properties: Properties45 - - -class Data3(BaseModel): - required: list[str] - - -class Properties46(BaseModel): - data: Data3 - - -class Then1(BaseModel): - properties: Properties46 - - -class DataObject(BaseModel): - properties: Properties42 - if_: If1 = Field(..., alias="if") - then: Then1 - required: list[str] - - -class Subschemas(BaseModel): - case: Case1 - data_object: DataObject - - -class FieldDefs(BaseModel): - properties: Properties - subschemas: Subschemas - - -class Class4(BaseModel): - const: str - - -class Properties47(BaseModel): - class_: Class4 = Field(..., alias="class") - - -class If2(BaseModel): - properties: Properties47 - - -class Then2(BaseModel): - title: str - description: str - field_ref: str = Field(..., alias="$ref") - - -class Else(BaseModel): - title: str - description: str - field_ref: str = Field(..., alias="$ref") - - -class Model(BaseModel): - field_schema: str = Field(..., alias="$schema") - field_id: str = Field(..., alias="$id") - field_contractual: list[str] = Field(..., alias="$contractual") - field_defs: FieldDefs = Field(..., alias="$defs") - field_comment: str = Field(..., alias="$comment") - if_: If2 = Field(..., alias="if") - then: Then2 - else_: Else = Field(..., alias="else") diff --git a/src/fmu/dataio/models/eninja.md b/src/fmu/dataio/models/eninja.md new file mode 100644 index 000000000..1f1446398 --- /dev/null +++ b/src/fmu/dataio/models/eninja.md @@ -0,0 +1,16 @@ +e ninja + +### Find all classes that do not have att. "data.vertical_domain" + +{ +"query": { +"bool": { + "must_not": [{"exists":{"field":"data.vertical_domain"}}] +} +}, +"aggs":{ + "class":{ +"terms":{"field":"class.keyword"} +} +} +} \ No newline at end of file diff --git a/src/fmu/dataio/models/meta.py b/src/fmu/dataio/models/meta.py new file mode 100644 index 000000000..7512c0995 --- /dev/null +++ b/src/fmu/dataio/models/meta.py @@ -0,0 +1,575 @@ +# generated by datamodel-codegen: +# filename: fmu_results.json +# version: 0.25.2 + +from __future__ import annotations + +from typing import Any, Literal, Optional, Union + +from pydantic import BaseModel, Field, RootModel + + +class Generic(RootModel[Any]): + root: Any + + +class Property(BaseModel): + name: Optional[str] = Field(default=None, examples=["MyPropertyName"]) + attribute: Optional[str] = Field(default=None, examples=["MyAttributeName"]) + is_discrete: Optional[bool] = Field(default=None, examples=[True, False]) + + +class GridModel(BaseModel): + name: Optional[str] = Field(default=None, examples=["MyGrid"]) + + +class Spec(BaseModel): + ncol: Optional[int] = Field(default=None, examples=[281]) + nrow: Optional[int] = Field(default=None, examples=[441]) + nlay: Optional[int] = Field(default=None, examples=[333]) + xori: Optional[float] = Field(default=None, examples=[461499.9997558594]) + yori: Optional[float] = Field(default=None, examples=[5926500.224123242]) + xinc: Optional[float] = Field(default=None, examples=[25.0]) + yflip: Optional[Literal[-1, 1]] = Field(default=None, examples=[-1, 1]) + rotation: Optional[float] = Field(default=None, examples=["30.00000000231"]) + undef: Optional[float] = Field(default=None, examples=[1e33]) + npolys: Optional[int] = Field( + default=None, + description="The number of individual polygons in the data object", + examples=[1], + ) + size: Optional[int] = Field( + default=None, description="Size of data object.", examples=[1, 9999] + ) + columns: Optional[list[str]] = Field( + default=None, description="List of columns present in a table." + ) + + +class Bbox(BaseModel): + xmin: float = Field(..., examples=[456012.5003497944]) + xmax: float = Field(..., examples=[467540.52762886323]) + ymin: float = Field(..., examples=[5926499.999511719]) + ymax: float = Field(..., examples=[5939492.128326312]) + zmin: Optional[float] = Field(default=None, examples=[1244.039, None]) + zmax: Optional[float] = Field(default=None, examples=[2302.683, None]) + + +class FluidContact(BaseModel): + """ + Conditional field + """ + + contact: Optional[Literal["owc", "fwl", "goc", "fgl"]] = Field( + default=None, examples=["owc", "fwl"] + ) + truncated: Optional[bool] = Field(default=None, examples=[True]) + + +class FieldOutline(BaseModel): + """ + Conditional field + """ + + contact: str + + +class FieldRegion(BaseModel): + """ + Conditional field + """ + + id: int = Field(..., description="The ID of the region") + + +class Seismic(BaseModel): + """ + Conditional field + """ + + attribute: Optional[str] = Field(default=None, examples=["amplitude_timeshifted"]) + calculation: Optional[str] = Field(default=None, examples=["mean"]) + zrange: Optional[float] = Field(default=None, examples=[12.0]) + filter_size: Optional[float] = Field(default=None, examples=[1.0]) + scaling_factor: Optional[float] = Field(default=None, examples=[1.0]) + stacking_offset: Optional[str] = Field(default=None, examples=["0-15"]) + + +class Line(BaseModel): + show: Optional[bool] = Field(default=None, examples=[True]) + color: Optional[str] = Field(default=None, examples=["black", "#000000"]) + + +class Points(BaseModel): + show: Optional[bool] = Field(default=None, examples=[True]) + color: Optional[str] = Field(default=None, examples=["black", "#000000"]) + + +class Contours(BaseModel): + show: Optional[bool] = Field(default=None, examples=[True]) + color: Optional[str] = Field( + default=None, + description="The color of the contour lines", + examples=["black", "#000000"], + ) + increment: Optional[float] = Field( + default=None, + description="The contour increment in same values as the data", + examples=[20.0], + ) + + +class Fill(BaseModel): + show: Optional[bool] = Field(default=None, examples=[True]) + color: Optional[str] = Field(default=None, examples=["black", "#000000"]) + colormap: Optional[str] = Field( + default=None, + description="named reference to a colormap", + examples=["gist_earth"], + ) + display_min: Optional[float] = Field( + default=None, + description="The low-side boundary to use for fill color", + examples=[1000.0], + ) + display_max: Optional[float] = Field( + default=None, + description="The high-side boundary to use for fill color", + examples=[1600.0], + ) + + +class Display(BaseModel): + name: Optional[str] = Field(default=None, examples=["Top Volantis"]) + subtitle: Optional[str] = Field(default=None, examples=["Some subtitle"]) + line: Optional[Line] = None + points: Optional[Points] = None + contours: Optional[Contours] = None + fill: Optional[Fill] = None + + +class Asset(BaseModel): + name: str = Field(..., examples=["Drogon"]) + + +class Ssdl(BaseModel): + access_level: Literal["internal", "restricted", "asset"] + rep_include: bool = Field(..., examples=[True, False]) + + +class Access(BaseModel): + asset: Optional[Asset] = None + ssdl: Optional[Ssdl] = None + classification: Optional[Literal["internal", "restricted"]] = Field( + default=None, examples=["internal", "restricted"] + ) + + +class Fmu2(RootModel[Any]): + root: Any + + +class File(BaseModel): + """ + Block describing the file as the data appear in FMU context + """ + + relative_path: str = Field( + ..., + description="The file path relative to RUNPATH", + examples=["share/results/maps/volantis_gp_base--depth.gri"], + ) + absolute_path: Optional[str] = Field( + default=None, + description="The absolute file path", + examples=["/abs/path/share/results/maps/volantis_gp_base--depth.gri"], + ) + checksum_md5: str = Field( + ..., + description="md5 checksum of the file or bytestring", + examples=["kjhsdfvsdlfk23knerknvk23"], + ) + size_bytes: Optional[int] = Field( + default=None, description="Size of file object in bytes", examples=[123] + ) + + +class Parameters(BaseModel): + """ + Parameters for this realization + """ + + +class Aggregation(BaseModel): + operation: str = Field(..., description="The aggregation performed") + realization_ids: list[int] = Field( + ..., description="Array of realization ids included in this aggregation" + ) + parameters: Optional[Union[list[dict[str, Any]], Parameters]] = Field( + default=None, description="Parameters for this realization" + ) + id: Optional[str] = Field( + default=None, + description="The ID of this aggregation", + examples=["15ce3b84-766f-4c93-9050-b154861f9100"], + ) + + +class Workflow(BaseModel): + reference: Optional[str] = Field( + default=None, + description="Reference to the part of the FMU workflow that produced this", + ) + + +class User(BaseModel): + id: str = Field(..., examples=["peesv"], title="User ID") + + +class FMUTimeObject(BaseModel): + """ + Time stamp for data object. + """ + + value: Optional[str] = Field(default=None, examples=["2020-10-28T14:28:02"]) + label: Optional[str] = Field(default=None, examples=["base", "monitor", "mylabel"]) + + +class TracklogEvent(BaseModel): + datetime: Optional[str] = Field(default=None, examples=["2020-10-28T14:28:02"]) + user: Optional[User] = None + event: Optional[str] = Field(default=None, examples=["created", "updated"]) + + +class Top(BaseModel): + name: Optional[str] = Field( + default=None, + description="Name of the data object. If stratigraphic, match the entry in the stratigraphic column", + examples=["VIKING GP. Top"], + ) + stratigraphic: Optional[bool] = Field( + default=None, + description="True if data object represents an entity in the stratigraphic column", + examples=[True], + ) + offset: Optional[float] = Field(default=None, examples=[11.2]) + + +class Base(BaseModel): + name: Optional[str] = Field( + default=None, + description="Name of the data object. If stratigraphic, match the entry in the stratigraphic column", + examples=["VIKING GP. Top"], + ) + stratigraphic: Optional[bool] = Field( + default=None, + description="True if data object represents an entity in the stratigraphic column", + examples=[True], + ) + offset: Optional[float] = Field(default=None, examples=[11.2]) + + +class Time(BaseModel): + t0: Optional[FMUTimeObject] = None + t1: Optional[FMUTimeObject] = None + + +class TheDataBlock(BaseModel): + name: str = Field( + ..., + description="Name of the data object. If stratigraphic, match the entry in the stratigraphic column", + examples=["VIKING GP. Top"], + ) + stratigraphic: bool = Field( + ..., + description="True if data object represents an entity in the stratigraphic column", + examples=[True], + ) + alias: Optional[list[str]] = None + stratigraphic_alias: Optional[list[str]] = None + offset: Optional[float] = Field(default=None, examples=[11.2]) + top: Optional[Top] = None + base: Optional[Base] = None + content: Literal[ + "depth", + "time", + "thickness", + "property", + "seismic", + "fluid_contact", + "field_outline", + "field_region", + "regions", + "pinchout", + "subcrop", + "fault_lines", + "velocity", + "volumes", + "volumetrics", + "inplace_volumes", + "khproduct", + "timeseries", + "wellpicks", + "parameters", + "relperm", + "rft", + "pvt", + "lift_curves", + "transmissibilities", + ] = Field( + ..., description="The contents of this data object", examples=["depth", "time"] + ) + tagname: Optional[str] = Field( + default=None, + description="A semi-human readable tag for internal usage and uniqueness", + examples=["ds_extract_geogrid", "ds_post_strucmod"], + ) + properties: Optional[list[Property]] = Field( + default=None, + description="data.properties is an array of property objects, representing the properties present in the data object.", + ) + format: str = Field(..., examples=["irap_binary"]) + layout: Optional[str] = Field(default=None, examples=["regular"]) + unit: Optional[str] = Field(default=None, examples=["m"]) + vertical_domain: Optional[Literal["depth", "time"]] = Field( + default=None, examples=["depth"] + ) + depth_reference: Optional[Literal["msl", "sb", "rkb"]] = Field( + default=None, examples=["msl"] + ) + undef_is_zero: Optional[bool] = Field( + default=None, + description="Flag if undefined values are to be interpreted as zero", + examples=["True"], + ) + grid_model: Optional[GridModel] = None + spec: Optional[Spec] = None + bbox: Optional[Bbox] = None + time: Optional[Time] = None + is_prediction: bool = Field(..., examples=[True], title="Is prediction flag") + is_observation: bool = Field(..., examples=[True], title="Is observation flag") + fluid_contact: Optional[FluidContact] = Field( + default=None, description="Conditional field" + ) + field_outline: Optional[FieldOutline] = Field( + default=None, description="Conditional field" + ) + field_region: Optional[FieldRegion] = Field( + default=None, description="Conditional field" + ) + description: Optional[list[str]] = None + seismic: Optional[Seismic] = Field(default=None, description="Conditional field") + + +class CountryItem(BaseModel): + identifier: str = Field(..., examples=["Norway"]) + uuid: str = Field( + ..., + examples=["ad214d85-8a1d-19da-e053-c918a4889309"], + pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", + ) + + +class DiscoveryItem(BaseModel): + short_identifier: str = Field(..., examples=["SomeDiscovery"]) + uuid: str = Field( + ..., + examples=["ad214d85-8a1d-19da-e053-c918a4889309"], + pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", + ) + + +class FieldItem(BaseModel): + identifier: str = Field(..., examples=["OseFax"]) + uuid: str = Field( + ..., + examples=["ad214d85-8a1d-19da-e053-c918a4889309"], + pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", + ) + + +class CoordinateSystem(BaseModel): + identifier: str = Field(..., examples=["ST_WGS84_UTM37N_P32637"]) + uuid: str = Field( + ..., + examples=["ad214d85-8a1d-19da-e053-c918a4889309"], + pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", + ) + + +class StratigraphicColumn(BaseModel): + identifier: str = Field(..., examples=["DROGON_2020"]) + uuid: str = Field( + ..., + examples=["ad214d85-8a1d-19da-e053-c918a4889309"], + pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", + ) + + +class Smda(BaseModel): + country: list[CountryItem] + discovery: list[DiscoveryItem] + field: list[FieldItem] + coordinate_system: CoordinateSystem + stratigraphic_column: StratigraphicColumn + + +class Masterdata(BaseModel): + smda: Smda + + +class Case(BaseModel): + name: str = Field(..., description="The case name", examples=["MyCaseName"]) + uuid: str = Field( + ..., + examples=["ad214d85-8a1d-19da-e053-c918a4889309"], + pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", + ) + user: User = Field(..., description="The user name used in ERT") + description: Optional[list[str]] = None + + +class Iteration(BaseModel): + name: str = Field( + ..., + description="The convential name of this iteration, e.g. iter-0 or pred", + examples=["iter-0"], + ) + id: Optional[int] = Field( + default=None, + description="The internal identification of this iteration, e.g. the iteration number", + examples=[0], + ) + uuid: str = Field( + ..., + examples=["ad214d85-8a1d-19da-e053-c918a4889309"], + pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", + ) + restart_from: Optional[str] = Field( + default=None, + description="A uuid reference to another iteration that this iteration was restarted from", + examples=["ad214d85-8a1d-19da-e053-c918a4889309"], + pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", + ) + + +class Model(BaseModel): + name: Optional[str] = Field(default=None, examples=["Drogon"]) + revision: Optional[str] = Field(default=None, examples=["21.0.0.dev"]) + description: Optional[list[str]] = Field( + default=None, description="This is a free text description of the model setup" + ) + + +class Realization(BaseModel): + name: str = Field( + ..., + description="The convential name of this iteration, e.g. iter-0 or pred", + examples=["iter-0"], + ) + id: int = Field( + ..., + description="The unique number of this realization as used in FMU", + examples=[33], + ) + uuid: str = Field( + ..., + examples=["ad214d85-8a1d-19da-e053-c918a4889309"], + pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", + ) + parameters: Optional[Union[list[dict[str, Any]], Parameters]] = Field( + default=None, description="Parameters for this realization" + ) + jobs: Optional[dict[str, Any]] = Field( + default=None, + description="Content directly taken from the ERT jobs.json file for this realization", + ) + + +class Fmu(BaseModel): + """ + The FMU block records properties that are specific to FMU + """ + + model: Model + case: Case + + +class Model1(BaseModel): + source: Literal["fmu"] = Field(..., description="Data source (FMU)") + version: Literal["0.8.0"] = Field( + ..., examples=["1.2.3"], title="FMU results metadata version" + ) + data: Optional[TheDataBlock] = None + tracklog: list[TracklogEvent] + access: Access + masterdata: Masterdata + class_: Literal[ + "case", + "surface", + "table", + "cpgrid", + "cpgrid_property", + "polygons", + "cube", + "well", + "points", + "dictionary", + ] = Field( + ..., + alias="class", + examples=["surface", "table", "points"], + title="Metadata class", + ) + fmu: Fmu = Field( + ..., description="The FMU block records properties that are specific to FMU" + ) + + +class Fmu1(BaseModel): + """ + The FMU block records properties that are specific to FMU + """ + + model: Model + case: Case + iteration: Optional[Iteration] = None + realization: Optional[Realization] = None + workflow: Optional[Workflow] = None + aggregation: Optional[Aggregation] = None + + +class Model2(BaseModel): + source: Literal["fmu"] = Field(..., description="Data source (FMU)") + version: Literal["0.8.0"] = Field( + ..., examples=["1.2.3"], title="FMU results metadata version" + ) + data: TheDataBlock + tracklog: list[TracklogEvent] + access: Access + masterdata: Masterdata + class_: Literal[ + "case", + "surface", + "table", + "cpgrid", + "cpgrid_property", + "polygons", + "cube", + "well", + "points", + "dictionary", + ] = Field( + ..., + alias="class", + examples=["surface", "table", "points"], + title="Metadata class", + ) + fmu: Fmu1 = Field( + ..., description="The FMU block records properties that are specific to FMU" + ) + file: File + + +class ModelModel(RootModel[Union[Model1, Model2]]): + root: Union[Model1, Model2] diff --git a/src/fmu/dataio/models/v3.py b/src/fmu/dataio/models/v3.py deleted file mode 100644 index 4a9873be1..000000000 --- a/src/fmu/dataio/models/v3.py +++ /dev/null @@ -1,79 +0,0 @@ -from __future__ import annotations - -from datetime import datetime -from typing import Literal - -from pydantic import BaseModel, Field -from typing_extensions import Annotated - - -class GeologicalModel(BaseModel): - type: Literal["Structural", "Rock"] - - -class RockGeologicalModel(GeologicalModel): - type: Literal["Rock"] = "Rock" - - -class StructuralGeologicalModel(GeologicalModel): - type: Literal["Structural"] = "Structural" - - -class Shape(BaseModel): - ncol: int = Field(ge=0) - nrow: int = Field(ge=0) - nlay: int = Field(ge=0) - - -class Orientation(BaseModel): - x: float - y: float - z: float - - -class Grid(BaseModel): - orientation: Orientation - shape: Shape - undef: float | None - - -class Range(BaseModel): - start: float - stop: float - - -class BoundingBox(BaseModel): - x: Range - y: Range - z: Range - - -class Header(BaseModel): - asset: str - created_at: datetime - created_by: str - version: int - - -class Payland(BaseModel): - type: Literal["fmu.everest", "fmu.ert"] - - -class FMUEverest(Payland): - type: Literal["fmu.everest"] = "fmu.everest" - - -class FMUErt(BaseModel): - type: Literal["fmu.ert"] = "fmu.ert" - model: Annotated[ - StructuralGeologicalModel | RockGeologicalModel, - Field(discriminator="type"), - ] - - -class Export(BaseModel): - header: Header - payload: Annotated[ - FMUEverest | FMUErt, - Field(discriminator="type"), - ] From b71d24794220725c2ad79c9ae82af3bebbe81b18 Mon Sep 17 00:00:00 2001 From: JB Lovland Date: Thu, 18 Jan 2024 12:46:02 +0100 Subject: [PATCH 05/25] meta2 from 090schema --- src/fmu/dataio/models/meta2.py | 458 +++++++++++++++++++++++++++++++++ src/fmu/dataio/models/meta3.py | 49 ++++ 2 files changed, 507 insertions(+) create mode 100644 src/fmu/dataio/models/meta2.py create mode 100644 src/fmu/dataio/models/meta3.py diff --git a/src/fmu/dataio/models/meta2.py b/src/fmu/dataio/models/meta2.py new file mode 100644 index 000000000..68421e013 --- /dev/null +++ b/src/fmu/dataio/models/meta2.py @@ -0,0 +1,458 @@ +# generated by datamodel-codegen: +# filename: fmu_results.json +# version: 0.25.2 + +from __future__ import annotations + +from typing import Any, Literal, Optional, Union + +from pydantic import BaseModel, Field, RootModel + + +class Properties(RootModel[Any]): + root: Any + + +class Subschemas(RootModel[Any]): + root: Any + + +class Asset(BaseModel): + name: str = Field(..., examples=["Drogon"]) + + +class Ssdl(BaseModel): + access_level: Literal["internal", "restricted", "asset"] + rep_include: bool = Field(..., examples=[True, False]) + + +class Access(BaseModel): + asset: Optional[Asset] = None + ssdl: Optional[Ssdl] = None + classification: Optional[Literal["internal", "restricted"]] = Field( + default=None, examples=["internal", "restricted"] + ) + + +class Property(BaseModel): + name: Optional[str] = Field(default=None, examples=["MyPropertyName"]) + attribute: Optional[str] = Field(default=None, examples=["MyAttributeName"]) + is_discrete: Optional[bool] = Field(default=None, examples=[True, False]) + + +class GridModel(BaseModel): + name: Optional[str] = Field(default=None, examples=["MyGrid"]) + + +class Spec(BaseModel): + ncol: Optional[int] = Field(default=None, examples=[281]) + nrow: Optional[int] = Field(default=None, examples=[441]) + nlay: Optional[int] = Field(default=None, examples=[333]) + xori: Optional[float] = Field(default=None, examples=[461499.9997558594]) + yori: Optional[float] = Field(default=None, examples=[5926500.224123242]) + xinc: Optional[float] = Field(default=None, examples=[25.0]) + yflip: Optional[Literal[-1, 1]] = Field(default=None, examples=[-1, 1]) + rotation: Optional[float] = Field(default=None, examples=["30.00000000231"]) + undef: Optional[float] = Field(default=None, examples=[1e33]) + npolys: Optional[int] = Field( + default=None, + description="The number of individual polygons in the data object", + examples=[1], + ) + size: Optional[int] = Field( + default=None, description="Size of data object.", examples=[1, 9999] + ) + columns: Optional[list[str]] = Field( + default=None, description="List of columns present in a table." + ) + + +class Bbox(BaseModel): + xmin: float = Field(..., examples=[456012.5003497944]) + xmax: float = Field(..., examples=[467540.52762886323]) + ymin: float = Field(..., examples=[5926499.999511719]) + ymax: float = Field(..., examples=[5939492.128326312]) + zmin: Optional[float] = Field(default=None, examples=[1244.039, None]) + zmax: Optional[float] = Field(default=None, examples=[2302.683, None]) + + +class FluidContact(BaseModel): + """ + Conditional field + """ + + contact: Optional[Literal["owc", "fwl", "goc", "fgl"]] = Field( + default=None, examples=["owc", "fwl"] + ) + truncated: Optional[bool] = Field(default=None, examples=[True]) + + +class FieldOutline(BaseModel): + """ + Conditional field + """ + + contact: str + + +class FieldRegion(BaseModel): + """ + Conditional field + """ + + id: int = Field(..., description="The ID of the region") + + +class Seismic(BaseModel): + """ + Conditional field + """ + + attribute: Optional[str] = Field(default=None, examples=["amplitude_timeshifted"]) + calculation: Optional[str] = Field(default=None, examples=["mean"]) + zrange: Optional[float] = Field(default=None, examples=[12.0]) + filter_size: Optional[float] = Field(default=None, examples=[1.0]) + scaling_factor: Optional[float] = Field(default=None, examples=[1.0]) + stacking_offset: Optional[str] = Field(default=None, examples=["0-15"]) + + +class File(BaseModel): + """ + Block describing the file as the data appear in FMU context + """ + + relative_path: str = Field( + ..., + description="The file path relative to RUNPATH", + examples=["share/results/maps/volantis_gp_base--depth.gri"], + ) + absolute_path: Optional[str] = Field( + default=None, + description="The absolute file path", + examples=["/abs/path/share/results/maps/volantis_gp_base--depth.gri"], + ) + checksum_md5: str = Field( + ..., + description="md5 checksum of the file or bytestring", + examples=["kjhsdfvsdlfk23knerknvk23"], + ) + size_bytes: Optional[int] = Field( + default=None, description="Size of file object in bytes", examples=[123] + ) + + +class Parameters(BaseModel): + """ + Parameters for this realization + """ + + +class Aggregation(BaseModel): + operation: str = Field(..., description="The aggregation performed") + realization_ids: list[int] = Field( + ..., description="Array of realization ids included in this aggregation" + ) + parameters: Optional[Union[list[dict[str, Any]], Parameters]] = Field( + default=None, description="Parameters for this realization" + ) + id: Optional[str] = Field( + default=None, + description="The ID of this aggregation", + examples=["15ce3b84-766f-4c93-9050-b154861f9100"], + ) + + +class Workflow(BaseModel): + reference: Optional[str] = Field( + default=None, + description="Reference to the part of the FMU workflow that produced this", + ) + + +class User(BaseModel): + id: str = Field(..., examples=["peesv"], title="User ID") + + +class Top(BaseModel): + name: Optional[str] = Field( + default=None, + description="Name of the data object. If stratigraphic, match the entry in the stratigraphic column", + examples=["VIKING GP. Top"], + ) + stratigraphic: Optional[bool] = Field( + default=None, + description="True if data object represents an entity in the stratigraphic column", + examples=[True], + ) + offset: Optional[float] = Field(default=None, examples=[11.2]) + + +class Base(BaseModel): + name: Optional[str] = Field( + default=None, + description="Name of the data object. If stratigraphic, match the entry in the stratigraphic column", + examples=["VIKING GP. Top"], + ) + stratigraphic: Optional[bool] = Field( + default=None, + description="True if data object represents an entity in the stratigraphic column", + examples=[True], + ) + offset: Optional[float] = Field(default=None, examples=[11.2]) + + +class Case(BaseModel): + name: str = Field(..., description="The case name", examples=["MyCaseName"]) + uuid: str = Field( + ..., + examples=["ad214d85-8a1d-19da-e053-c918a4889309"], + pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", + ) + user: User = Field(..., description="The user name used in ERT") + description: Optional[list[str]] = None + + +class Iteration(BaseModel): + name: str = Field( + ..., + description="The convential name of this iteration, e.g. iter-0 or pred", + examples=["iter-0"], + ) + id: Optional[int] = Field( + default=None, + description="The internal identification of this iteration, e.g. the iteration number", + examples=[0], + ) + uuid: str = Field( + ..., + examples=["ad214d85-8a1d-19da-e053-c918a4889309"], + pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", + ) + restart_from: Optional[str] = Field( + default=None, + description="A uuid reference to another iteration that this iteration was restarted from", + examples=["ad214d85-8a1d-19da-e053-c918a4889309"], + pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", + ) + + +class Model(BaseModel): + name: Optional[str] = Field(default=None, examples=["Drogon"]) + revision: Optional[str] = Field(default=None, examples=["21.0.0.dev"]) + description: Optional[list[str]] = Field( + default=None, description="This is a free text description of the model setup" + ) + + +class Realization(BaseModel): + name: str = Field( + ..., + description="The convential name of this iteration, e.g. iter-0 or pred", + examples=["iter-0"], + ) + id: int = Field( + ..., + description="The unique number of this realization as used in FMU", + examples=[33], + ) + uuid: str = Field( + ..., + examples=["ad214d85-8a1d-19da-e053-c918a4889309"], + pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", + ) + parameters: Optional[Union[list[dict[str, Any]], Parameters]] = Field( + default=None, description="Parameters for this realization" + ) + jobs: Optional[dict[str, Any]] = Field( + default=None, + description="Content directly taken from the ERT jobs.json file for this realization", + ) + + +class CountryItem(BaseModel): + identifier: str = Field(..., examples=["Norway"]) + uuid: str = Field( + ..., + examples=["ad214d85-8a1d-19da-e053-c918a4889309"], + pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", + ) + + +class DiscoveryItem(BaseModel): + short_identifier: str = Field(..., examples=["SomeDiscovery"]) + uuid: str = Field( + ..., + examples=["ad214d85-8a1d-19da-e053-c918a4889309"], + pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", + ) + + +class FieldItem(BaseModel): + identifier: str = Field(..., examples=["OseFax"]) + uuid: str = Field( + ..., + examples=["ad214d85-8a1d-19da-e053-c918a4889309"], + pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", + ) + + +class CoordinateSystem(BaseModel): + identifier: str = Field(..., examples=["ST_WGS84_UTM37N_P32637"]) + uuid: str = Field( + ..., + examples=["ad214d85-8a1d-19da-e053-c918a4889309"], + pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", + ) + + +class StratigraphicColumn(BaseModel): + identifier: str = Field(..., examples=["DROGON_2020"]) + uuid: str = Field( + ..., + examples=["ad214d85-8a1d-19da-e053-c918a4889309"], + pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", + ) + + +class Smda(BaseModel): + country: list[CountryItem] + discovery: list[DiscoveryItem] + field: list[FieldItem] + coordinate_system: CoordinateSystem + stratigraphic_column: StratigraphicColumn + + +class Masterdata(BaseModel): + smda: Smda + + +class FMUTimeObject(BaseModel): + """ + Time stamp for data object. + """ + + value: Optional[str] = Field(default=None, examples=["2020-10-28T14:28:02"]) + label: Optional[str] = Field(default=None, examples=["base", "monitor", "mylabel"]) + + +class TracklogEvent(BaseModel): + datetime: Optional[str] = Field(default=None, examples=["2020-10-28T14:28:02"]) + user: Optional[User] = None + event: Optional[str] = Field(default=None, examples=["created", "updated"]) + + +class Fmu(BaseModel): + """ + The FMU block records properties that are specific to FMU + """ + + model: Model + case: Case + iteration: Optional[Iteration] = None + realization: Optional[Realization] = None + workflow: Optional[Workflow] = None + aggregation: Optional[Aggregation] = None + + +class Time(BaseModel): + t0: Optional[FMUTimeObject] = None + t1: Optional[FMUTimeObject] = None + + +class TheDataBlock(BaseModel): + name: str = Field( + ..., + description="Name of the data object. If stratigraphic, match the entry in the stratigraphic column", + examples=["VIKING GP. Top"], + ) + stratigraphic: bool = Field( + ..., + description="True if data object represents an entity in the stratigraphic column", + examples=[True], + ) + alias: Optional[list[str]] = None + stratigraphic_alias: Optional[list[str]] = None + offset: Optional[float] = Field(default=None, examples=[11.2]) + top: Optional[Top] = None + base: Optional[Base] = None + content: str = Field( + ..., description="The contents of this data object", examples=["depth"] + ) + tagname: Optional[str] = Field( + default=None, + description="A semi-human readable tag for internal usage and uniqueness", + examples=["ds_extract_geogrid", "ds_post_strucmod"], + ) + properties: Optional[list[Property]] = Field( + default=None, + description="data.properties is an array of property objects, representing the properties present in the data object.", + ) + format: str = Field(..., examples=["irap_binary"]) + layout: Optional[str] = Field(default=None, examples=["regular"]) + unit: Optional[str] = Field(default=None, examples=["m"]) + vertical_domain: Optional[Literal["depth", "time"]] = Field( + default=None, examples=["depth"] + ) + depth_reference: Optional[Literal["msl", "sb", "rkb"]] = Field( + default=None, examples=["msl"] + ) + undef_is_zero: Optional[bool] = Field( + default=None, + description="Flag if undefined values are to be interpreted as zero", + examples=["True"], + ) + grid_model: Optional[GridModel] = None + spec: Optional[Spec] = None + bbox: Optional[Bbox] = None + time: Optional[Time] = None + is_prediction: bool = Field(..., examples=[True], title="Is prediction flag") + is_observation: bool = Field(..., examples=[True], title="Is observation flag") + fluid_contact: Optional[FluidContact] = Field( + default=None, description="Conditional field" + ) + field_outline: Optional[FieldOutline] = Field( + default=None, description="Conditional field" + ) + field_region: Optional[FieldRegion] = Field( + default=None, description="Conditional field" + ) + description: Optional[list[str]] = None + seismic: Optional[Seismic] = Field(default=None, description="Conditional field") + + +class DataObject(BaseModel): + fmu: Fmu = Field( + ..., description="The FMU block records properties that are specific to FMU" + ) + file: File + data: TheDataBlock + access: Access + source: Literal["fmu"] = Field(..., description="Data source (FMU)") + version: Literal["0.9.0"] = Field( + ..., examples=["1.2.3"], title="FMU results metadata version" + ) + class_: Literal[ + "case", + "surface", + "table", + "cpgrid", + "cpgrid_property", + "polygons", + "cube", + "well", + "points", + "dictionary", + ] = Field( + ..., + alias="class", + examples=["surface", "table", "points"], + title="Metadata class", + ) + tracklog: list[TracklogEvent] + masterdata: Masterdata + + +class CaseObject(RootModel[DataObject]): + root: DataObject = Field( + ..., description="Validation of a case object.", title="Case object" + ) diff --git a/src/fmu/dataio/models/meta3.py b/src/fmu/dataio/models/meta3.py new file mode 100644 index 000000000..306f130c0 --- /dev/null +++ b/src/fmu/dataio/models/meta3.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from datetime import datetime +from typing import Any, Dict, List, Literal, Optional, Union + +from pydantic import BaseModel, Field, RootModel + + +class User(BaseModel): + id: str = Field(description="User id", examples=["peesv", "jlov"]) + + +class Description(RootModel[List[str]]): + ... + + +class FMUTime(BaseModel): + value: datetime + label: str = Field(examples=["base", "monitor", "mylabel"]) + + +class Source(RootModel[Literal["fmu"]]): + ... + + +class Version(RootModel[Literal["0.9.0"]]): + ... + + +class TrackLogEvent(BaseModel): + datetime: datetime + user: User + event: str = Field(examples=["created", "updated"]) + + +class TrackLog(RootModel[List[TrackLogEvent]]): + ... + + +class CaseObj(BaseModel): + ... + + +class DataObj(BaseModel): + ... + + +class Meta(RootModel[Union[CaseObj, DataObj]]): + ... From e01424478bc07ae0e2fa82b4510caf63ea4f73da Mon Sep 17 00:00:00 2001 From: JB Lovland Date: Thu, 18 Jan 2024 14:52:52 +0100 Subject: [PATCH 06/25] Cleaner schema --- boom.py | 55 ++++ src/fmu/dataio/models/meta2.py | 514 ++++++++++++++++++++------------- 2 files changed, 366 insertions(+), 203 deletions(-) create mode 100644 boom.py diff --git a/boom.py b/boom.py new file mode 100644 index 000000000..0b7a2cdb8 --- /dev/null +++ b/boom.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from contextlib import suppress +from random import sample + +from fmu.sumo.explorer import Explorer +from tqdm import tqdm + +from src.fmu.dataio.models import meta2 + + +def lazy_sampler(x, lenx, k=100): + if lenx <= 0: + return + + sampled_idx = sample(range(lenx), k=k) if k < lenx else range(lenx) + + for i in sampled_idx: + with suppress(IndexError): + yield x[i] + + +def gen(): + e = Explorer(env="dev") + for c in tqdm(sample(tuple(e.cases), 3), ascii=True, position=0): + yield c.metadata + + for cube in lazy_sampler(c.cubes, len(c.cubes)): + yield cube.metadata + + for surf in lazy_sampler(c.surfaces, len(c.surfaces)): + yield surf.metadata + + for poly in lazy_sampler(c.polygons, len(c.surfaces)): + yield poly.metadata + + for tab in lazy_sampler(c.tables, len(c.tables)): + yield tab.metadata + + for dic in lazy_sampler(c.dictionaries, len(c.dictionaries)): + yield dic.metadata + + +if __name__ == "__main__": + root_classes = set() + for m in tqdm(gen(), ascii=True, position=1): + try: + root_classes.add(meta2.Meta.model_validate(m).class_) + except Exception: + from pprint import pp + pp(m) + raise + + print("-->>> All good in da hood.") + print(root_classes) diff --git a/src/fmu/dataio/models/meta2.py b/src/fmu/dataio/models/meta2.py index 68421e013..933e7a9bf 100644 --- a/src/fmu/dataio/models/meta2.py +++ b/src/fmu/dataio/models/meta2.py @@ -4,76 +4,137 @@ from __future__ import annotations +from pathlib import Path from typing import Any, Literal, Optional, Union from pydantic import BaseModel, Field, RootModel -class Properties(RootModel[Any]): - root: Any - - -class Subschemas(RootModel[Any]): - root: Any +class UUID(RootModel[str]): + root: str = Field( + examples=["ad214d85-8a1d-19da-e053-c918a4889309"], + pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", + ) class Asset(BaseModel): - name: str = Field(..., examples=["Drogon"]) + name: str = Field( + examples=["Drogon"], + ) class Ssdl(BaseModel): access_level: Literal["internal", "restricted", "asset"] - rep_include: bool = Field(..., examples=[True, False]) + rep_include: bool = Field( + examples=[True, False], + ) class Access(BaseModel): - asset: Optional[Asset] = None - ssdl: Optional[Ssdl] = None - classification: Optional[Literal["internal", "restricted"]] = Field( - default=None, examples=["internal", "restricted"] + asset: Asset + classification: Literal["internal", "restricted"] = Field( + default="internal", + examples=["internal", "restricted"], + ) + ssdl: Ssdl = Field( + default_factory=lambda: Ssdl( + access_level="internal", + rep_include=False, + ) ) class Property(BaseModel): - name: Optional[str] = Field(default=None, examples=["MyPropertyName"]) - attribute: Optional[str] = Field(default=None, examples=["MyAttributeName"]) - is_discrete: Optional[bool] = Field(default=None, examples=[True, False]) + attribute: str = Field( + examples=["MyAttributeName"], + ) + is_discrete: bool = Field( + default=False, + examples=[True, False], + ) + name: str = Field( + examples=["MyPropertyName"], + ) class GridModel(BaseModel): - name: Optional[str] = Field(default=None, examples=["MyGrid"]) + name: str = Field( + examples=["MyGrid"], + ) class Spec(BaseModel): - ncol: Optional[int] = Field(default=None, examples=[281]) - nrow: Optional[int] = Field(default=None, examples=[441]) - nlay: Optional[int] = Field(default=None, examples=[333]) - xori: Optional[float] = Field(default=None, examples=[461499.9997558594]) - yori: Optional[float] = Field(default=None, examples=[5926500.224123242]) - xinc: Optional[float] = Field(default=None, examples=[25.0]) - yflip: Optional[Literal[-1, 1]] = Field(default=None, examples=[-1, 1]) - rotation: Optional[float] = Field(default=None, examples=["30.00000000231"]) - undef: Optional[float] = Field(default=None, examples=[1e33]) - npolys: Optional[int] = Field( + ncol: int = Field( + default=0, + examples=[281], + ) + nrow: int = Field( + default=0, + examples=[441], + ) + nlay: int = Field( + default=0, + examples=[333], + ) + xori: Optional[float] = Field( + default=None, + examples=[461499.9997558594], + ) + yori: Optional[float] = Field( + default=None, + examples=[5926500.224123242], + ) + xinc: Optional[float] = Field( + default=None, + examples=[25.0], + ) + yflip: Optional[Literal[-1, 1]] = Field( default=None, + examples=[-1, 1], + ) + rotation: float = Field( + default=0.0, + examples=["30.00000000231"], + ) + undef: Optional[float] = Field( + default=None, + examples=[1e33], + ) + npolys: int = Field( + default=0, description="The number of individual polygons in the data object", examples=[1], ) - size: Optional[int] = Field( - default=None, description="Size of data object.", examples=[1, 9999] + size: int = Field( + default=0, + description="Size of data object.", + examples=[1, 9999], ) - columns: Optional[list[str]] = Field( - default=None, description="List of columns present in a table." + columns: list[str] = Field( + default_factory=list, + description="List of columns present in a table.", ) class Bbox(BaseModel): - xmin: float = Field(..., examples=[456012.5003497944]) - xmax: float = Field(..., examples=[467540.52762886323]) - ymin: float = Field(..., examples=[5926499.999511719]) - ymax: float = Field(..., examples=[5939492.128326312]) - zmin: Optional[float] = Field(default=None, examples=[1244.039, None]) - zmax: Optional[float] = Field(default=None, examples=[2302.683, None]) + xmin: float = Field( + examples=[456012.5003497944], + ) + xmax: float = Field( + examples=[467540.52762886323], + ) + ymin: float = Field( + examples=[5926499.999511719], + ) + ymax: float = Field( + examples=[5939492.128326312], + ) + zmin: float = Field( + examples=[1244.039], + ) + zmax: float = Field( + examples=[2302.683], + ) class FluidContact(BaseModel): @@ -81,10 +142,13 @@ class FluidContact(BaseModel): Conditional field """ - contact: Optional[Literal["owc", "fwl", "goc", "fgl"]] = Field( - default=None, examples=["owc", "fwl"] + contact: Literal["owc", "fwl", "goc", "fgl"] = Field( + examples=["owc", "fwl"], + ) + truncated: bool = Field( + default=False, + examples=[True], ) - truncated: Optional[bool] = Field(default=None, examples=[True]) class FieldOutline(BaseModel): @@ -100,7 +164,9 @@ class FieldRegion(BaseModel): Conditional field """ - id: int = Field(..., description="The ID of the region") + id: int = Field( + description="The ID of the region", + ) class Seismic(BaseModel): @@ -108,12 +174,30 @@ class Seismic(BaseModel): Conditional field """ - attribute: Optional[str] = Field(default=None, examples=["amplitude_timeshifted"]) - calculation: Optional[str] = Field(default=None, examples=["mean"]) - zrange: Optional[float] = Field(default=None, examples=[12.0]) - filter_size: Optional[float] = Field(default=None, examples=[1.0]) - scaling_factor: Optional[float] = Field(default=None, examples=[1.0]) - stacking_offset: Optional[str] = Field(default=None, examples=["0-15"]) + attribute: Optional[str] = Field( + default=None, + examples=["amplitude_timeshifted"], + ) + calculation: Optional[str] = Field( + default=None, + examples=["mean"], + ) + filter_size: Optional[float] = Field( + default=None, + examples=[1.0], + ) + scaling_factor: Optional[float] = Field( + default=None, + examples=[1.0], + ) + stacking_offset: Optional[str] = Field( + default=None, + examples=["0-15"], + ) + zrange: Optional[float] = Field( + default=None, + examples=[12.0], + ) class File(BaseModel): @@ -121,23 +205,22 @@ class File(BaseModel): Block describing the file as the data appear in FMU context """ - relative_path: str = Field( - ..., - description="The file path relative to RUNPATH", - examples=["share/results/maps/volantis_gp_base--depth.gri"], - ) - absolute_path: Optional[str] = Field( - default=None, + absolute_path: Path = Field( description="The absolute file path", examples=["/abs/path/share/results/maps/volantis_gp_base--depth.gri"], ) checksum_md5: str = Field( - ..., description="md5 checksum of the file or bytestring", examples=["kjhsdfvsdlfk23knerknvk23"], ) - size_bytes: Optional[int] = Field( - default=None, description="Size of file object in bytes", examples=[123] + relative_path: Path = Field( + description="The file path relative to RUNPATH", + examples=["share/results/maps/volantis_gp_base--depth.gri"], + ) + size_bytes: int = Field( + default=-1, + description="Size of file object in bytes", + examples=[123], ) @@ -148,177 +231,169 @@ class Parameters(BaseModel): class Aggregation(BaseModel): - operation: str = Field(..., description="The aggregation performed") - realization_ids: list[int] = Field( - ..., description="Array of realization ids included in this aggregation" + id: str = Field( + description="The ID of this aggregation", + examples=["15ce3b84-766f-4c93-9050-b154861f9100"], + ) + operation: str = Field( + description="The aggregation performed", ) parameters: Optional[Union[list[dict[str, Any]], Parameters]] = Field( default=None, description="Parameters for this realization" ) - id: Optional[str] = Field( - default=None, - description="The ID of this aggregation", - examples=["15ce3b84-766f-4c93-9050-b154861f9100"], + realization_ids: list[int] = Field( + description="Array of realization ids included in this aggregation" ) class Workflow(BaseModel): - reference: Optional[str] = Field( - default=None, - description="Reference to the part of the FMU workflow that produced this", + reference: str = Field( + description="Reference to the part of the FMU workflow that produced this" ) class User(BaseModel): - id: str = Field(..., examples=["peesv"], title="User ID") - - -class Top(BaseModel): - name: Optional[str] = Field( - default=None, - description="Name of the data object. If stratigraphic, match the entry in the stratigraphic column", - examples=["VIKING GP. Top"], - ) - stratigraphic: Optional[bool] = Field( - default=None, - description="True if data object represents an entity in the stratigraphic column", - examples=[True], + id: str = Field( + examples=["peesv"], + title="User ID", ) - offset: Optional[float] = Field(default=None, examples=[11.2]) -class Base(BaseModel): - name: Optional[str] = Field( - default=None, +class Layer(BaseModel): + name: str = Field( description="Name of the data object. If stratigraphic, match the entry in the stratigraphic column", examples=["VIKING GP. Top"], ) - stratigraphic: Optional[bool] = Field( - default=None, + offset: float = Field( + default=0, + examples=[11.2], + ) + stratigraphic: bool = Field( + default=False, description="True if data object represents an entity in the stratigraphic column", examples=[True], ) - offset: Optional[float] = Field(default=None, examples=[11.2]) class Case(BaseModel): - name: str = Field(..., description="The case name", examples=["MyCaseName"]) - uuid: str = Field( - ..., - examples=["ad214d85-8a1d-19da-e053-c918a4889309"], - pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", - ) - user: User = Field(..., description="The user name used in ERT") description: Optional[list[str]] = None + name: str = Field( + description="The case name", + examples=["MyCaseName"], + ) + user: User = Field( + description="The user name used in ERT", + ) + uuid: UUID class Iteration(BaseModel): - name: str = Field( - ..., - description="The convential name of this iteration, e.g. iter-0 or pred", - examples=["iter-0"], - ) id: Optional[int] = Field( default=None, description="The internal identification of this iteration, e.g. the iteration number", examples=[0], ) - uuid: str = Field( - ..., - examples=["ad214d85-8a1d-19da-e053-c918a4889309"], - pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", + name: str = Field( + description="The convential name of this iteration, e.g. iter-0 or pred", + examples=["iter-0"], ) - restart_from: Optional[str] = Field( + restart_from: Optional[UUID] = Field( default=None, description="A uuid reference to another iteration that this iteration was restarted from", + ) + + uuid: UUID = Field( examples=["ad214d85-8a1d-19da-e053-c918a4889309"], - pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", ) class Model(BaseModel): - name: Optional[str] = Field(default=None, examples=["Drogon"]) - revision: Optional[str] = Field(default=None, examples=["21.0.0.dev"]) description: Optional[list[str]] = Field( - default=None, description="This is a free text description of the model setup" + default=None, + description="This is a free text description of the model setup", + ) + name: Optional[str] = Field( + default=None, + examples=["Drogon"], + ) + revision: Optional[str] = Field( + default=None, + examples=["21.0.0.dev"], ) class Realization(BaseModel): - name: str = Field( - ..., - description="The convential name of this iteration, e.g. iter-0 or pred", - examples=["iter-0"], - ) id: int = Field( - ..., description="The unique number of this realization as used in FMU", examples=[33], ) - uuid: str = Field( - ..., - examples=["ad214d85-8a1d-19da-e053-c918a4889309"], - pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", - ) - parameters: Optional[Union[list[dict[str, Any]], Parameters]] = Field( - default=None, description="Parameters for this realization" - ) jobs: Optional[dict[str, Any]] = Field( default=None, description="Content directly taken from the ERT jobs.json file for this realization", ) + name: str = Field( + description="The convential name of this iteration, e.g. iter-0 or pred", + examples=["iter-0"], + ) + parameters: Optional[Union[list[dict[str, Any]], Parameters]] = Field( + default=None, + description="Parameters for this realization", + ) + uuid: UUID = Field( + examples=["ad214d85-8a1d-19da-e053-c918a4889309"], + ) class CountryItem(BaseModel): - identifier: str = Field(..., examples=["Norway"]) - uuid: str = Field( - ..., + identifier: str = Field( + examples=["Norway"], + ) + uuid: UUID = Field( examples=["ad214d85-8a1d-19da-e053-c918a4889309"], - pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", ) class DiscoveryItem(BaseModel): - short_identifier: str = Field(..., examples=["SomeDiscovery"]) - uuid: str = Field( - ..., + short_identifier: str = Field( + examples=["SomeDiscovery"], + ) + uuid: UUID = Field( examples=["ad214d85-8a1d-19da-e053-c918a4889309"], - pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", ) class FieldItem(BaseModel): - identifier: str = Field(..., examples=["OseFax"]) - uuid: str = Field( - ..., + identifier: str = Field( + examples=["OseFax"], + ) + uuid: UUID = Field( examples=["ad214d85-8a1d-19da-e053-c918a4889309"], - pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", ) class CoordinateSystem(BaseModel): - identifier: str = Field(..., examples=["ST_WGS84_UTM37N_P32637"]) - uuid: str = Field( - ..., + identifier: str = Field( + examples=["ST_WGS84_UTM37N_P32637"], + ) + uuid: UUID = Field( examples=["ad214d85-8a1d-19da-e053-c918a4889309"], - pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", ) class StratigraphicColumn(BaseModel): - identifier: str = Field(..., examples=["DROGON_2020"]) - uuid: str = Field( - ..., + identifier: str = Field( + examples=["DROGON_2020"], + ) + uuid: UUID = Field( examples=["ad214d85-8a1d-19da-e053-c918a4889309"], - pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", ) class Smda(BaseModel): + coordinate_system: CoordinateSystem country: list[CountryItem] discovery: list[DiscoveryItem] field: list[FieldItem] - coordinate_system: CoordinateSystem stratigraphic_column: StratigraphicColumn @@ -331,14 +406,26 @@ class FMUTimeObject(BaseModel): Time stamp for data object. """ - value: Optional[str] = Field(default=None, examples=["2020-10-28T14:28:02"]) - label: Optional[str] = Field(default=None, examples=["base", "monitor", "mylabel"]) + label: Optional[str] = Field( + default=None, + examples=["base", "monitor", "mylabel"], + ) + value: Optional[str] = Field( + default=None, + examples=["2020-10-28T14:28:02"], + ) class TracklogEvent(BaseModel): - datetime: Optional[str] = Field(default=None, examples=["2020-10-28T14:28:02"]) + datetime: Optional[str] = Field( + default=None, + examples=["2020-10-28T14:28:02"], + ) + event: Optional[str] = Field( + default=None, + examples=["created", "updated"], + ) user: Optional[User] = None - event: Optional[str] = Field(default=None, examples=["created", "updated"]) class Fmu(BaseModel): @@ -346,12 +433,12 @@ class Fmu(BaseModel): The FMU block records properties that are specific to FMU """ - model: Model + aggregation: Optional[Aggregation] = None case: Case iteration: Optional[Iteration] = None + model: Model realization: Optional[Realization] = None workflow: Optional[Workflow] = None - aggregation: Optional[Aggregation] = None class Time(BaseModel): @@ -360,77 +447,89 @@ class Time(BaseModel): class TheDataBlock(BaseModel): + alias: Optional[list[str]] = None + base: Optional[Layer] = None + bbox: Optional[Bbox] = None + content: str = Field( + description="The contents of this data object", + examples=["depth"], + ) + depth_reference: Optional[Literal["msl", "sb", "rkb"]] = Field( + default=None, + examples=["msl"], + ) + description: Optional[list[str]] = None + field_outline: Optional[FieldOutline] = Field( + default=None, + description="Conditional field", + ) + field_region: Optional[FieldRegion] = Field( + default=None, + description="Conditional field", + ) + fluid_contact: Optional[FluidContact] = Field( + default=None, + description="Conditional field", + ) + format: str = Field( + examples=["irap_binary"], + ) + grid_model: Optional[GridModel] = None + is_observation: bool = Field( + examples=[True], + title="Is observation flag", + ) + is_prediction: bool = Field( + examples=[True], + title="Is prediction flag", + ) + layout: Optional[str] = Field( + default=None, + examples=["regular"], + ) name: str = Field( - ..., description="Name of the data object. If stratigraphic, match the entry in the stratigraphic column", examples=["VIKING GP. Top"], ) + offset: Optional[float] = Field( + default=None, + examples=[11.2], + ) + properties: Optional[list[Property]] = Field( + default=None, + description="data.properties is an array of property objects, representing the properties present in the data object.", + ) + seismic: Optional[Seismic] = Field( + default=None, + description="Conditional field", + ) + spec: Optional[Spec] = None + stratigraphic_alias: Optional[list[str]] = None stratigraphic: bool = Field( - ..., description="True if data object represents an entity in the stratigraphic column", examples=[True], ) - alias: Optional[list[str]] = None - stratigraphic_alias: Optional[list[str]] = None - offset: Optional[float] = Field(default=None, examples=[11.2]) - top: Optional[Top] = None - base: Optional[Base] = None - content: str = Field( - ..., description="The contents of this data object", examples=["depth"] - ) tagname: Optional[str] = Field( default=None, description="A semi-human readable tag for internal usage and uniqueness", examples=["ds_extract_geogrid", "ds_post_strucmod"], ) - properties: Optional[list[Property]] = Field( - default=None, - description="data.properties is an array of property objects, representing the properties present in the data object.", - ) - format: str = Field(..., examples=["irap_binary"]) - layout: Optional[str] = Field(default=None, examples=["regular"]) - unit: Optional[str] = Field(default=None, examples=["m"]) - vertical_domain: Optional[Literal["depth", "time"]] = Field( - default=None, examples=["depth"] - ) - depth_reference: Optional[Literal["msl", "sb", "rkb"]] = Field( - default=None, examples=["msl"] - ) + time: Optional[Time] = None + top: Optional[Layer] = None undef_is_zero: Optional[bool] = Field( default=None, description="Flag if undefined values are to be interpreted as zero", examples=["True"], ) - grid_model: Optional[GridModel] = None - spec: Optional[Spec] = None - bbox: Optional[Bbox] = None - time: Optional[Time] = None - is_prediction: bool = Field(..., examples=[True], title="Is prediction flag") - is_observation: bool = Field(..., examples=[True], title="Is observation flag") - fluid_contact: Optional[FluidContact] = Field( - default=None, description="Conditional field" - ) - field_outline: Optional[FieldOutline] = Field( - default=None, description="Conditional field" - ) - field_region: Optional[FieldRegion] = Field( - default=None, description="Conditional field" + unit: Optional[str] = Field(default=None, examples=["m"],) + vertical_domain: Optional[Literal["depth", "time"]] = Field( + default=None, + examples=["depth"], ) - description: Optional[list[str]] = None - seismic: Optional[Seismic] = Field(default=None, description="Conditional field") -class DataObject(BaseModel): - fmu: Fmu = Field( - ..., description="The FMU block records properties that are specific to FMU" - ) - file: File - data: TheDataBlock +class Meta(BaseModel): access: Access - source: Literal["fmu"] = Field(..., description="Data source (FMU)") - version: Literal["0.9.0"] = Field( - ..., examples=["1.2.3"], title="FMU results metadata version" - ) class_: Literal[ "case", "surface", @@ -443,16 +542,25 @@ class DataObject(BaseModel): "points", "dictionary", ] = Field( - ..., alias="class", examples=["surface", "table", "points"], title="Metadata class", ) - tracklog: list[TracklogEvent] + data: Optional[TheDataBlock] = None + file: Optional[File] = None + fmu: Fmu = Field( + description="The FMU block records properties that are specific to FMU", + ) masterdata: Masterdata + source: Literal["fmu"] = Field( + description="Data source (FMU)", + ) + tracklog: list[TracklogEvent] + version: Literal["0.8.0"] = Field( + title="FMU results metadata version", + ) -class CaseObject(RootModel[DataObject]): - root: DataObject = Field( - ..., description="Validation of a case object.", title="Case object" - ) + +if __name__ == "__main__": + print() \ No newline at end of file From 2baf411196c0ad63312bc5cd202bca27e7c74c72 Mon Sep 17 00:00:00 2001 From: JB Lovland Date: Thu, 18 Jan 2024 16:34:51 +0100 Subject: [PATCH 07/25] Formatting dump func --- boom.py | 4 +- ex.py | 13 ++++ src/fmu/dataio/models/meta2.py | 126 +++++++++++++++++++++++++-------- 3 files changed, 110 insertions(+), 33 deletions(-) create mode 100644 ex.py diff --git a/boom.py b/boom.py index 0b7a2cdb8..c863423f9 100644 --- a/boom.py +++ b/boom.py @@ -6,7 +6,7 @@ from fmu.sumo.explorer import Explorer from tqdm import tqdm -from src.fmu.dataio.models import meta2 +from src.fmu.dataio.models.meta2 import Meta def lazy_sampler(x, lenx, k=100): @@ -45,7 +45,7 @@ def gen(): root_classes = set() for m in tqdm(gen(), ascii=True, position=1): try: - root_classes.add(meta2.Meta.model_validate(m).class_) + root_classes.add(Meta.model_validate(m).class_) except Exception: from pprint import pp pp(m) diff --git a/ex.py b/ex.py new file mode 100644 index 000000000..d7b3e5fea --- /dev/null +++ b/ex.py @@ -0,0 +1,13 @@ +import sys +from pprint import pp + +from fmu.dataio.models.meta2 import Meta +from orjson import dumps +from yaml import safe_load + +print(sys.argv[1]) +try: + Meta.model_validate_json(dumps(safe_load(open(sys.argv[1])))) +except Exception: + pp(safe_load(open(sys.argv[1]))) + raise diff --git a/src/fmu/dataio/models/meta2.py b/src/fmu/dataio/models/meta2.py index 933e7a9bf..1ec4b4b5b 100644 --- a/src/fmu/dataio/models/meta2.py +++ b/src/fmu/dataio/models/meta2.py @@ -4,10 +4,12 @@ from __future__ import annotations +import json from pathlib import Path from typing import Any, Literal, Optional, Union from pydantic import BaseModel, Field, RootModel +from pydantic.json_schema import GenerateJsonSchema class UUID(RootModel[str]): @@ -32,7 +34,10 @@ class Ssdl(BaseModel): class Access(BaseModel): asset: Asset - classification: Literal["internal", "restricted"] = Field( + classification: Literal[ + "internal", + "restricted", + ] = Field( default="internal", examples=["internal", "restricted"], ) @@ -44,19 +49,6 @@ class Access(BaseModel): ) -class Property(BaseModel): - attribute: str = Field( - examples=["MyAttributeName"], - ) - is_discrete: bool = Field( - default=False, - examples=[True, False], - ) - name: str = Field( - examples=["MyPropertyName"], - ) - - class GridModel(BaseModel): name: str = Field( examples=["MyGrid"], @@ -76,6 +68,7 @@ class Spec(BaseModel): default=0, examples=[333], ) + xori: Optional[float] = Field( default=None, examples=[461499.9997558594], @@ -261,7 +254,10 @@ class User(BaseModel): class Layer(BaseModel): name: str = Field( - description="Name of the data object. If stratigraphic, match the entry in the stratigraphic column", + description=( + "Name of the data object. If stratigraphic, " + "match the entry in the stratigraphic column" + ), examples=["VIKING GP. Top"], ) offset: float = Field( @@ -270,7 +266,9 @@ class Layer(BaseModel): ) stratigraphic: bool = Field( default=False, - description="True if data object represents an entity in the stratigraphic column", + description=( + "True if data object represents an entity in the stratigraphic colum" + ), examples=[True], ) @@ -290,7 +288,9 @@ class Case(BaseModel): class Iteration(BaseModel): id: Optional[int] = Field( default=None, - description="The internal identification of this iteration, e.g. the iteration number", + description=( + "The internal identification of this iteration, e.g. the iteration number" + ), examples=[0], ) name: str = Field( @@ -299,7 +299,10 @@ class Iteration(BaseModel): ) restart_from: Optional[UUID] = Field( default=None, - description="A uuid reference to another iteration that this iteration was restarted from", + description=( + "A uuid reference to another iteration that this " + "iteration was restarted from" + ), ) uuid: UUID = Field( @@ -329,7 +332,9 @@ class Realization(BaseModel): ) jobs: Optional[dict[str, Any]] = Field( default=None, - description="Content directly taken from the ERT jobs.json file for this realization", + description=( + "Content directly taken from the ERT jobs.json file for this realization", + ), ) name: str = Field( description="The convential name of this iteration, e.g. iter-0 or pred", @@ -458,7 +463,9 @@ class TheDataBlock(BaseModel): default=None, examples=["msl"], ) - description: Optional[list[str]] = None + description: list[str] = Field( + default_factory=list, + ) field_outline: Optional[FieldOutline] = Field( default=None, description="Conditional field", @@ -488,17 +495,16 @@ class TheDataBlock(BaseModel): examples=["regular"], ) name: str = Field( - description="Name of the data object. If stratigraphic, match the entry in the stratigraphic column", + description=( + "Name of the data object. If stratigraphic, " + "match the entry in the stratigraphic column" + ), examples=["VIKING GP. Top"], ) - offset: Optional[float] = Field( - default=None, + offset: float = Field( + default=0.0, examples=[11.2], ) - properties: Optional[list[Property]] = Field( - default=None, - description="data.properties is an array of property objects, representing the properties present in the data object.", - ) seismic: Optional[Seismic] = Field( default=None, description="Conditional field", @@ -506,7 +512,9 @@ class TheDataBlock(BaseModel): spec: Optional[Spec] = None stratigraphic_alias: Optional[list[str]] = None stratigraphic: bool = Field( - description="True if data object represents an entity in the stratigraphic column", + description=( + "True if data object represents an entity " "in the stratigraphic column" + ), examples=[True], ) tagname: Optional[str] = Field( @@ -521,7 +529,10 @@ class TheDataBlock(BaseModel): description="Flag if undefined values are to be interpreted as zero", examples=["True"], ) - unit: Optional[str] = Field(default=None, examples=["m"],) + unit: str = Field( + default="", + examples=["m"], + ) vertical_domain: Optional[Literal["depth", "time"]] = Field( default=None, examples=["depth"], @@ -561,6 +572,59 @@ class Meta(BaseModel): ) - if __name__ == "__main__": - print() \ No newline at end of file + sg = GenerateJsonSchema + sg.schema_dialect = "http://json-schema.org/draft-07/schema" + dumped = Meta.model_json_schema( + by_alias=True, + schema_generator=sg, + ) + dumped = { + "$id": "https://main-fmu-schemas-dev.radix.equinor.com/schemas/0.8.0/fmu_results.json", + "$schema": sg.schema_dialect, + "$contractual": [ + "class", + "source", + "version", + "tracklog", + "data.format", + "data.name", + "data.stratigraphic", + "data.alias", + "data.stratigraphic_alias", + "data.offset", + "data.content", + "data.tagname", + "data.vertical_domain", + "data.grid_model", + "data.bbox", + "data.time", + "data.is_prediction", + "data.is_observation", + "data.seismic.attribute", + "data.spec.columns", + "access", + "masterdata", + "fmu.model", + "fmu.workflow", + "fmu.case", + "fmu.iteration.name", + "fmu.iteration.uuid", + "fmu.realization.name", + "fmu.realization.id", + "fmu.realization.uuid", + "fmu.aggregation.operation", + "fmu.aggregation.realization_ids", + "fmu.context.stage", + "file.relative_path", + "file.checksum_md5", + "file.size_bytes", + ], + } | dumped + + print( + json.dumps( + dumped, + indent=2, + ) + ) From d6bfffa0bb454cf40aabeb8686baed477c076969 Mon Sep 17 00:00:00 2001 From: JB Lovland Date: Fri, 19 Jan 2024 15:44:57 +0100 Subject: [PATCH 08/25] Sub models --- src/fmu/dataio/models/meta2.py | 423 +++++++++++++++++-------- tests/test_schema/test_schema_logic.py | 19 +- 2 files changed, 298 insertions(+), 144 deletions(-) diff --git a/src/fmu/dataio/models/meta2.py b/src/fmu/dataio/models/meta2.py index 1ec4b4b5b..b0b088ced 100644 --- a/src/fmu/dataio/models/meta2.py +++ b/src/fmu/dataio/models/meta2.py @@ -5,6 +5,8 @@ from __future__ import annotations import json +from collections import ChainMap +from datetime import datetime from pathlib import Path from typing import Any, Literal, Optional, Union @@ -25,7 +27,7 @@ class Asset(BaseModel): ) -class Ssdl(BaseModel): +class SSDL(BaseModel): access_level: Literal["internal", "restricted", "asset"] rep_include: bool = Field( examples=[True, False], @@ -41,8 +43,8 @@ class Access(BaseModel): default="internal", examples=["internal", "restricted"], ) - ssdl: Ssdl = Field( - default_factory=lambda: Ssdl( + ssdl: SSDL = Field( + default_factory=lambda: SSDL( access_level="internal", rep_include=False, ) @@ -55,61 +57,167 @@ class GridModel(BaseModel): ) -class Spec(BaseModel): +class CaseSpec(BaseModel): + ... + + +class SurfaceSpec(BaseModel): ncol: int = Field( - default=0, examples=[281], ) + nlay: int = Field( + examples=[333], + ) nrow: int = Field( - default=0, examples=[441], ) + rotation: float = Field( + examples=["30.00000000231"], + ) + undef: float = Field( + examples=[1e33], + ) + xinc: float = Field( + examples=[25.0], + ) + xori: float = Field( + examples=[461499.9997558594], + ) + yflip: Literal[-1, 1] = Field( + examples=[-1, 1], + ) + yori: float = Field( + examples=[5926500.224123242], + ) + + +class TableSpec(BaseModel): + columns: list[str] = Field( + description="List of columns present in a table.", + ) + size: int = Field( + description="Size of data object.", + examples=[1, 9999], + ) + + +class CPGridSpec(BaseModel): + """Corner point grid""" + + ncol: int = Field( + examples=[281], + ) nlay: int = Field( - default=0, examples=[333], ) + nrow: int = Field( + examples=[441], + ) - xori: Optional[float] = Field( - default=None, - examples=[461499.9997558594], + # Shift + xshift: float = Field( + examples=[1234], ) - yori: Optional[float] = Field( - default=None, - examples=[5926500.224123242], + yshift: float = Field( + examples=[1234], ) - xinc: Optional[float] = Field( - default=None, - examples=[25.0], + zshift: float = Field( + examples=[1234], ) - yflip: Optional[Literal[-1, 1]] = Field( - default=None, - examples=[-1, 1], + + # Scaling + xscale: float = Field( + examples=[1234], ) - rotation: float = Field( - default=0.0, - examples=["30.00000000231"], + yscale: float = Field( + examples=[1234], ) - undef: Optional[float] = Field( - default=None, - examples=[1e33], + zscale: float = Field( + examples=[1234], ) + + +class CPGridPropertySpec(BaseModel): + ncol: int = Field( + examples=[281], + ) + nlay: int = Field( + examples=[333], + ) + nrow: int = Field( + examples=[441], + ) + + +class PolygonsSpec(BaseModel): npolys: int = Field( - default=0, description="The number of individual polygons in the data object", examples=[1], ) - size: int = Field( - default=0, - description="Size of data object.", - examples=[1, 9999], + + +class CubeSpec(SurfaceSpec): + # Shape + ncol: int = Field( + examples=[281], ) - columns: list[str] = Field( - default_factory=list, - description="List of columns present in a table.", + nlay: int = Field( + examples=[333], + ) + nrow: int = Field( + examples=[441], + ) + + # inc. + xinc: float = Field( + examples=[25.0], + ) + yinc: float = Field( + examples=[1.1], + ) + zinc: float = Field( + examples=[1.1], ) + # Ori. + xori: float = Field( + examples=[461499.9997558594], + ) + yori: float = Field( + examples=[461499.9997558594], + ) + zori: float = Field( + examples=[461499.9997558594], + ) -class Bbox(BaseModel): + # Misc. + yflip: Literal[-1, 1] = Field( + examples=[-1, 1], + ) + zflip: Literal[-1, 1] = Field( + examples=[-1, 1], + ) + rotation: float = Field( + examples=["30.00000000231"], + ) + undef: float = Field( + examples=[1e33], + ) + + +class WellSpec(BaseModel): + ... + + +class PointsSpec(BaseModel): + ... + + +class DictionarySpec(BaseModel): + ... + + +class BoundingBox(BaseModel): xmin: float = Field( examples=[456012.5003497944], ) @@ -333,7 +441,7 @@ class Realization(BaseModel): jobs: Optional[dict[str, Any]] = Field( default=None, description=( - "Content directly taken from the ERT jobs.json file for this realization", + "Content directly taken from the ERT jobs.json file for this realization" ), ) name: str = Field( @@ -422,15 +530,13 @@ class FMUTimeObject(BaseModel): class TracklogEvent(BaseModel): - datetime: Optional[str] = Field( - default=None, + datetime: datetime = Field( examples=["2020-10-28T14:28:02"], ) - event: Optional[str] = Field( - default=None, + event: str = Field( examples=["created", "updated"], ) - user: Optional[User] = None + user: User class Fmu(BaseModel): @@ -451,56 +557,115 @@ class Time(BaseModel): t1: Optional[FMUTimeObject] = None -class TheDataBlock(BaseModel): - alias: Optional[list[str]] = None - base: Optional[Layer] = None - bbox: Optional[Bbox] = None - content: str = Field( +class Content(BaseModel): + # From schema + content: Literal[ + "depth", + "time", + "thickness", + "property", + "seismic", + "fluid_contact", + "field_outline", + "field_region", + "regions", + "pinchout", + "subcrop", + "fault_lines", + "velocity", + "volumes", + "volumetrics", + "inplace_volumes", + "khproduct", + "timeseries", + "wellpicks", + "parameters", + "relperm", + "rft", + "pvt", + "lift_curves", + "transmissibilities", + ] = Field( description="The contents of this data object", examples=["depth"], ) - depth_reference: Optional[Literal["msl", "sb", "rkb"]] = Field( - default=None, + name: str = Field( + description=( + "Name of the data object. If stratigraphic, " + "match the entry in the stratigraphic column" + ), + examples=["VIKING GP. Top"], + ) + format: str = Field( + examples=["irap_binary"], + ) + stratigraphic: bool = Field( + description=( + "True if data object represents an entity in the stratigraphic column" + ), + examples=[True], + ) + is_prediction: bool = Field( + examples=[True], + title="Is prediction flag", + ) + is_observation: bool = Field( + examples=[True], + title="Is observation flag", + ) + # end + + +class DepthContent(Content): + content: Literal["depth"] + depth_reference: Literal["msl", "sb", "rkb"] = Field( examples=["msl"], ) - description: list[str] = Field( - default_factory=list, + + +class FieldOutlineContent(Content): + content: Literal["field_outline"] + field_outline: FieldOutline = Field( + description="Conditional field", ) - field_outline: Optional[FieldOutline] = Field( - default=None, + + +class FieldRegionContent(Content): + content: Literal["field_region"] + field_region: FieldRegion = Field( description="Conditional field", ) - field_region: Optional[FieldRegion] = Field( - default=None, + + +class FluidContactContent(Content): + content: Literal["fluid_contact"] + fluid_contact: FluidContact = Field( description="Conditional field", ) - fluid_contact: Optional[FluidContact] = Field( - default=None, + + +class SeismicContent(Content): + content: Literal["fluid_contact"] + seismic: Seismic = Field( description="Conditional field", ) - format: str = Field( - examples=["irap_binary"], + + +class TheDataBlock(BaseModel): + # From schema + + # end + alias: Optional[list[str]] = None + base: Optional[Layer] = None + bbox: Optional[BoundingBox] = None + description: list[str] = Field( + default_factory=list, ) grid_model: Optional[GridModel] = None - is_observation: bool = Field( - examples=[True], - title="Is observation flag", - ) - is_prediction: bool = Field( - examples=[True], - title="Is prediction flag", - ) layout: Optional[str] = Field( default=None, examples=["regular"], ) - name: str = Field( - description=( - "Name of the data object. If stratigraphic, " - "match the entry in the stratigraphic column" - ), - examples=["VIKING GP. Top"], - ) offset: float = Field( default=0.0, examples=[11.2], @@ -511,12 +676,6 @@ class TheDataBlock(BaseModel): ) spec: Optional[Spec] = None stratigraphic_alias: Optional[list[str]] = None - stratigraphic: bool = Field( - description=( - "True if data object represents an entity " "in the stratigraphic column" - ), - examples=[True], - ) tagname: Optional[str] = Field( default=None, description="A semi-human readable tag for internal usage and uniqueness", @@ -539,8 +698,7 @@ class TheDataBlock(BaseModel): ) -class Meta(BaseModel): - access: Access +class ClassMeta(BaseModel): class_: Literal[ "case", "surface", @@ -557,6 +715,14 @@ class Meta(BaseModel): examples=["surface", "table", "points"], title="Metadata class", ) + + +class CaseMeta(ClassMeta): + class_: Literal["case"] + + +class Meta(BaseModel): + access: Access data: Optional[TheDataBlock] = None file: Optional[File] = None fmu: Fmu = Field( @@ -573,58 +739,57 @@ class Meta(BaseModel): if __name__ == "__main__": - sg = GenerateJsonSchema - sg.schema_dialect = "http://json-schema.org/draft-07/schema" - dumped = Meta.model_json_schema( - by_alias=True, - schema_generator=sg, - ) - dumped = { - "$id": "https://main-fmu-schemas-dev.radix.equinor.com/schemas/0.8.0/fmu_results.json", - "$schema": sg.schema_dialect, - "$contractual": [ - "class", - "source", - "version", - "tracklog", - "data.format", - "data.name", - "data.stratigraphic", - "data.alias", - "data.stratigraphic_alias", - "data.offset", - "data.content", - "data.tagname", - "data.vertical_domain", - "data.grid_model", - "data.bbox", - "data.time", - "data.is_prediction", - "data.is_observation", - "data.seismic.attribute", - "data.spec.columns", - "access", - "masterdata", - "fmu.model", - "fmu.workflow", - "fmu.case", - "fmu.iteration.name", - "fmu.iteration.uuid", - "fmu.realization.name", - "fmu.realization.id", - "fmu.realization.uuid", - "fmu.aggregation.operation", - "fmu.aggregation.realization_ids", - "fmu.context.stage", - "file.relative_path", - "file.checksum_md5", - "file.size_bytes", - ], - } | dumped + dumped = ChainMap( + { + "$id": "https://main-fmu-schemas-dev.radix.equinor.com/schemas/0.8.0/fmu_results.json", + "$schema": GenerateJsonSchema.schema_dialect, + "$contractual": [ + "class", + "source", + "version", + "tracklog", + "data.format", + "data.name", + "data.stratigraphic", + "data.alias", + "data.stratigraphic_alias", + "data.offset", + "data.content", + "data.tagname", + "data.vertical_domain", + "data.grid_model", + "data.bbox", + "data.time", + "data.is_prediction", + "data.is_observation", + "data.seismic.attribute", + "data.spec.columns", + "access", + "masterdata", + "fmu.model", + "fmu.workflow", + "fmu.case", + "fmu.iteration.name", + "fmu.iteration.uuid", + "fmu.realization.name", + "fmu.realization.id", + "fmu.realization.uuid", + "fmu.aggregation.operation", + "fmu.aggregation.realization_ids", + "fmu.context.stage", + "file.relative_path", + "file.checksum_md5", + "file.size_bytes", + ], + }, + Meta.model_json_schema( + by_alias=True, + ), + ) print( json.dumps( - dumped, + dict(dumped), indent=2, ) ) diff --git a/tests/test_schema/test_schema_logic.py b/tests/test_schema/test_schema_logic.py index 8625a666b..f9997a82e 100644 --- a/tests/test_schema/test_schema_logic.py +++ b/tests/test_schema/test_schema_logic.py @@ -5,6 +5,7 @@ import jsonschema import pytest from fmu.dataio._definitions import ALLOWED_CONTENTS +from fmu.dataio.models.meta2 import Meta # pylint: disable=no-member @@ -36,21 +37,9 @@ def test_schema_080_validate_examples_as_is(schema_080, metadata_examples): """Confirm that examples are valid against the schema""" for i, (name, metadata) in enumerate(metadata_examples.items()): - try: - jsonschema.validate(instance=metadata, schema=schema_080) - except jsonschema.exceptions.ValidationError: - logger.error("Failed validating existing example: %s", name) - if i == 0: - logger.error( - "This was the first example attempted." - "Error is most likely int he schema." - ) - else: - logger.error( - "This was not the first example attemted." - "Error is most likely in the example." - ) - raise + print("------", name) + jsonschema.validate(instance=metadata, schema=Meta.model_json_schema()) + Meta.model_validate(metadata) def test_schema_080_file_block(schema_080, metadata_examples): From dcf0244a6183c1dd928c5f68f711a3f63a0b027b Mon Sep 17 00:00:00 2001 From: JB Lovland Date: Fri, 19 Jan 2024 17:36:36 +0100 Subject: [PATCH 09/25] Solo questing --- boom.py | 1 + ex.py | 20 +- parameters.py | 38 ++++ src/fmu/dataio/models/meta2.py | 340 +++++++++++---------------------- src/fmu/dataio/models/meta3.py | 2 +- 5 files changed, 162 insertions(+), 239 deletions(-) create mode 100644 parameters.py diff --git a/boom.py b/boom.py index c863423f9..f5f32ba02 100644 --- a/boom.py +++ b/boom.py @@ -48,6 +48,7 @@ def gen(): root_classes.add(Meta.model_validate(m).class_) except Exception: from pprint import pp + pp(m) raise diff --git a/ex.py b/ex.py index d7b3e5fea..aeeace48b 100644 --- a/ex.py +++ b/ex.py @@ -3,11 +3,19 @@ from fmu.dataio.models.meta2 import Meta from orjson import dumps +from pydantic import ValidationError from yaml import safe_load -print(sys.argv[1]) -try: - Meta.model_validate_json(dumps(safe_load(open(sys.argv[1])))) -except Exception: - pp(safe_load(open(sys.argv[1]))) - raise + +def read(file): + with open(file) as f: + return f.read() + +for file in (f.strip() for f in sys.stdin.readlines()): + print(file) + try: + Meta.model_validate_json(dumps(safe_load(read(file)))) + except ValidationError as e: + print(str(e)) + pp(safe_load(read(file))) + print("-" * 100) diff --git a/parameters.py b/parameters.py new file mode 100644 index 000000000..c1abb111c --- /dev/null +++ b/parameters.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from typing import Dict, Union + +from pydantic import RootModel + + +class Parameters(RootModel[Dict[str, Union[int, float, str, "Parameters"]]]): + ... + + # root: dict[str, int | float | str | Parameters] + + +# class Parameters(BaseModel): +# parameters: Union[int, float, str, "Parameters"] + +# print(Parameters.model_rebuild()) +# print(Parameters.model_json_schema()) +print( + Parameters.model_validate({"hello": "hello", "this": {"this": "that", "more": 1}}) +) +print( + Parameters.model_validate( + { + "SENSNAME": "faultseal", + "SENSCASE": "low", + "RMS_SEED": 1006, + "INIT_FILES": { + "PERM_FLUVCHAN_E1_NORM": 0.748433, + "PERM_FLUVCHAN_E21_NORM": 0.782068, + }, + "KVKH_CHANNEL": 0.6, + "KVKH_US": 0.6, + "FAULT_SEAL_SCALING": 0.1, + "FWL_CENTRAL": 1677, + } + ) +) diff --git a/src/fmu/dataio/models/meta2.py b/src/fmu/dataio/models/meta2.py index b0b088ced..f8cb95766 100644 --- a/src/fmu/dataio/models/meta2.py +++ b/src/fmu/dataio/models/meta2.py @@ -1,54 +1,34 @@ -# generated by datamodel-codegen: -# filename: fmu_results.json -# version: 0.25.2 - from __future__ import annotations import json from collections import ChainMap -from datetime import datetime from pathlib import Path -from typing import Any, Literal, Optional, Union +from typing import Dict, Literal, Optional, Union -from pydantic import BaseModel, Field, RootModel +from pydantic import BaseModel, ByteSize, Field, NaiveDatetime, RootModel from pydantic.json_schema import GenerateJsonSchema class UUID(RootModel[str]): root: str = Field( examples=["ad214d85-8a1d-19da-e053-c918a4889309"], - pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", + pattern=r"^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", ) class Asset(BaseModel): - name: str = Field( - examples=["Drogon"], - ) + name: str = Field(examples=["Drogon"]) -class SSDL(BaseModel): +class Ssdl(BaseModel): access_level: Literal["internal", "restricted", "asset"] - rep_include: bool = Field( - examples=[True, False], - ) + rep_include: bool class Access(BaseModel): asset: Asset - classification: Literal[ - "internal", - "restricted", - ] = Field( - default="internal", - examples=["internal", "restricted"], - ) - ssdl: SSDL = Field( - default_factory=lambda: SSDL( - access_level="internal", - rep_include=False, - ) - ) + classification: Literal["internal", "restricted"] | None = Field(default=None) + ssdl: Ssdl | None = Field(default=None) class GridModel(BaseModel): @@ -57,38 +37,23 @@ class GridModel(BaseModel): ) +class Shape(BaseModel): + nrow: int = Field(description="The number of rows") + ncol: int = Field(description="The number of columns") + nlay: int = Field(description="The number of layers") + + class CaseSpec(BaseModel): ... -class SurfaceSpec(BaseModel): - ncol: int = Field( - examples=[281], - ) - nlay: int = Field( - examples=[333], - ) - nrow: int = Field( - examples=[441], - ) - rotation: float = Field( - examples=["30.00000000231"], - ) - undef: float = Field( - examples=[1e33], - ) - xinc: float = Field( - examples=[25.0], - ) - xori: float = Field( - examples=[461499.9997558594], - ) - yflip: Literal[-1, 1] = Field( - examples=[-1, 1], - ) - yori: float = Field( - examples=[5926500.224123242], - ) +class SurfaceSpec(Shape): + rotation: float = Field(description="Rotation angle") + undef: float = Field(description="Value representing undefined data") + xinc: float = Field(description="Increment along the x-axis") + xori: float = Field(description="Origin along the x-axis") + yflip: Literal[-1, 1] = Field(description="Flip along the y-axis, -1 or 1") + yori: float = Field(description="Origin along the y-axis") class TableSpec(BaseModel): @@ -101,108 +66,44 @@ class TableSpec(BaseModel): ) -class CPGridSpec(BaseModel): +class CPGridSpec(Shape): """Corner point grid""" - ncol: int = Field( - examples=[281], - ) - nlay: int = Field( - examples=[333], - ) - nrow: int = Field( - examples=[441], - ) + xshift: float = Field(description="Shift along the x-axis") + yshift: float = Field(description="Shift along the y-axis") + zshift: float = Field(description="Shift along the z-axis") - # Shift - xshift: float = Field( - examples=[1234], - ) - yshift: float = Field( - examples=[1234], - ) - zshift: float = Field( - examples=[1234], - ) - - # Scaling - xscale: float = Field( - examples=[1234], - ) - yscale: float = Field( - examples=[1234], - ) - zscale: float = Field( - examples=[1234], - ) + xscale: float = Field(description="Scaling factor for the x-axis") + yscale: float = Field(description="Scaling factor for the y-axis") + zscale: float = Field(description="Scaling factor for the z-axis") -class CPGridPropertySpec(BaseModel): - ncol: int = Field( - examples=[281], - ) - nlay: int = Field( - examples=[333], - ) - nrow: int = Field( - examples=[441], - ) +class CPGridPropertySpec(Shape): + ... class PolygonsSpec(BaseModel): npolys: int = Field( description="The number of individual polygons in the data object", - examples=[1], ) class CubeSpec(SurfaceSpec): - # Shape - ncol: int = Field( - examples=[281], - ) - nlay: int = Field( - examples=[333], - ) - nrow: int = Field( - examples=[441], - ) + # Increment + xinc: float = Field(description="Increment along the x-axis") + yinc: float = Field(description="Increment along the y-axis") + zinc: float = Field(description="Increment along the z-axis") - # inc. - xinc: float = Field( - examples=[25.0], - ) - yinc: float = Field( - examples=[1.1], - ) - zinc: float = Field( - examples=[1.1], - ) + # Origin + xori: float = Field(description="Origin along the x-axis") + yori: float = Field(description="Origin along the y-axis") + zori: float = Field(description="Origin along the z-axis") - # Ori. - xori: float = Field( - examples=[461499.9997558594], - ) - yori: float = Field( - examples=[461499.9997558594], - ) - zori: float = Field( - examples=[461499.9997558594], - ) - - # Misc. - yflip: Literal[-1, 1] = Field( - examples=[-1, 1], - ) - zflip: Literal[-1, 1] = Field( - examples=[-1, 1], - ) - rotation: float = Field( - examples=["30.00000000231"], - ) - undef: float = Field( - examples=[1e33], - ) + # Miscellaneous + yflip: Literal[-1, 1] = Field(description="Flip along the y-axis, -1 or 1") + zflip: Literal[-1, 1] = Field(description="Flip along the z-axis, -1 or 1") + rotation: float = Field(description="Rotation angle") + undef: float = Field(description="Value representing undefined data") class WellSpec(BaseModel): @@ -218,24 +119,12 @@ class DictionarySpec(BaseModel): class BoundingBox(BaseModel): - xmin: float = Field( - examples=[456012.5003497944], - ) - xmax: float = Field( - examples=[467540.52762886323], - ) - ymin: float = Field( - examples=[5926499.999511719], - ) - ymax: float = Field( - examples=[5939492.128326312], - ) - zmin: float = Field( - examples=[1244.039], - ) - zmax: float = Field( - examples=[2302.683], - ) + xmin: float = Field(description="Minimum x-coordinate") + xmax: float = Field(description="Maximum x-coordinate") + ymin: float = Field(description="Minimum y-coordinate") + ymax: float = Field(description="Maximum y-coordinate") + zmin: float = Field(description="Minimum z-coordinate") + zmax: float = Field(description="Maximum z-coordinate") class FluidContact(BaseModel): @@ -246,10 +135,7 @@ class FluidContact(BaseModel): contact: Literal["owc", "fwl", "goc", "fgl"] = Field( examples=["owc", "fwl"], ) - truncated: bool = Field( - default=False, - examples=[True], - ) + truncated: bool = Field(default=False) class FieldOutline(BaseModel): @@ -285,11 +171,9 @@ class Seismic(BaseModel): ) filter_size: Optional[float] = Field( default=None, - examples=[1.0], ) scaling_factor: Optional[float] = Field( default=None, - examples=[1.0], ) stacking_offset: Optional[str] = Field( default=None, @@ -297,7 +181,6 @@ class Seismic(BaseModel): ) zrange: Optional[float] = Field( default=None, - examples=[12.0], ) @@ -310,37 +193,34 @@ class File(BaseModel): description="The absolute file path", examples=["/abs/path/share/results/maps/volantis_gp_base--depth.gri"], ) - checksum_md5: str = Field( - description="md5 checksum of the file or bytestring", - examples=["kjhsdfvsdlfk23knerknvk23"], - ) relative_path: Path = Field( description="The file path relative to RUNPATH", examples=["share/results/maps/volantis_gp_base--depth.gri"], ) - size_bytes: int = Field( - default=-1, + checksum_md5: str = Field( + description="md5 checksum of the file or bytestring", + examples=["kjhsdfvsdlfk23knerknvk23"], + ) + size_bytes: ByteSize | None = Field( + default=None, description="Size of file object in bytes", - examples=[123], ) -class Parameters(BaseModel): - """ - Parameters for this realization - """ +class Parameters(RootModel[Dict[str, Union[int, float, str, "Parameters"]]]): + ... class Aggregation(BaseModel): - id: str = Field( + id: UUID = Field( description="The ID of this aggregation", examples=["15ce3b84-766f-4c93-9050-b154861f9100"], ) operation: str = Field( description="The aggregation performed", ) - parameters: Optional[Union[list[dict[str, Any]], Parameters]] = Field( - default=None, description="Parameters for this realization" + parameters: Parameters = Field( + description="Parameters for this realization", ) realization_ids: list[int] = Field( description="Array of realization ids included in this aggregation" @@ -355,7 +235,7 @@ class Workflow(BaseModel): class User(BaseModel): id: str = Field( - examples=["peesv"], + examples=["peesv", "jlov"], title="User ID", ) @@ -370,14 +250,12 @@ class Layer(BaseModel): ) offset: float = Field( default=0, - examples=[11.2], ) stratigraphic: bool = Field( default=False, description=( "True if data object represents an entity in the stratigraphic colum" ), - examples=[True], ) @@ -394,18 +272,16 @@ class Case(BaseModel): class Iteration(BaseModel): - id: Optional[int] = Field( - default=None, + id: int = Field( description=( "The internal identification of this iteration, e.g. the iteration number" ), - examples=[0], ) name: str = Field( description="The convential name of this iteration, e.g. iter-0 or pred", examples=["iter-0"], ) - restart_from: Optional[UUID] = Field( + restart_from: UUID | None = Field( default=None, description=( "A uuid reference to another iteration that this " @@ -413,9 +289,7 @@ class Iteration(BaseModel): ), ) - uuid: UUID = Field( - examples=["ad214d85-8a1d-19da-e053-c918a4889309"], - ) + uuid: UUID class Model(BaseModel): @@ -433,12 +307,39 @@ class Model(BaseModel): ) +class RealizationJobListing(BaseModel): + arg_types: list[str] + argList: list[Path] + error_file: Optional[Path] + executable: Path + license_path: Optional[Path] + max_arg: int + max_running_minutes: Optional[int] + max_running: Optional[int] + min_arg: int + name: str + start_file: Optional[str] + stderr: Optional[str] + stdin: Optional[str] + stdout: Optional[str] + target_file: Optional[Path] + + +class RealizationJobs(BaseModel): + data_root: Path = Field(alias="DATA_ROOT") + ert_pid: str + global_environment: dict[str, str] + global_update_path: dict + job_list: list[RealizationJobListing] = Field(alias="jobList") + run_id: str + umask: str + + class Realization(BaseModel): id: int = Field( description="The unique number of this realization as used in FMU", - examples=[33], ) - jobs: Optional[dict[str, Any]] = Field( + jobs: Optional[RealizationJobs] = Field( default=None, description=( "Content directly taken from the ERT jobs.json file for this realization" @@ -448,58 +349,46 @@ class Realization(BaseModel): description="The convential name of this iteration, e.g. iter-0 or pred", examples=["iter-0"], ) - parameters: Optional[Union[list[dict[str, Any]], Parameters]] = Field( + parameters: Parameters | None = Field( default=None, description="Parameters for this realization", ) - uuid: UUID = Field( - examples=["ad214d85-8a1d-19da-e053-c918a4889309"], - ) + uuid: UUID class CountryItem(BaseModel): identifier: str = Field( examples=["Norway"], ) - uuid: UUID = Field( - examples=["ad214d85-8a1d-19da-e053-c918a4889309"], - ) + uuid: UUID class DiscoveryItem(BaseModel): short_identifier: str = Field( examples=["SomeDiscovery"], ) - uuid: UUID = Field( - examples=["ad214d85-8a1d-19da-e053-c918a4889309"], - ) + uuid: UUID class FieldItem(BaseModel): identifier: str = Field( examples=["OseFax"], ) - uuid: UUID = Field( - examples=["ad214d85-8a1d-19da-e053-c918a4889309"], - ) + uuid: UUID class CoordinateSystem(BaseModel): identifier: str = Field( examples=["ST_WGS84_UTM37N_P32637"], ) - uuid: UUID = Field( - examples=["ad214d85-8a1d-19da-e053-c918a4889309"], - ) + uuid: UUID class StratigraphicColumn(BaseModel): identifier: str = Field( examples=["DROGON_2020"], ) - uuid: UUID = Field( - examples=["ad214d85-8a1d-19da-e053-c918a4889309"], - ) + uuid: UUID class Smda(BaseModel): @@ -530,7 +419,7 @@ class FMUTimeObject(BaseModel): class TracklogEvent(BaseModel): - datetime: datetime = Field( + datetime: NaiveDatetime = Field( examples=["2020-10-28T14:28:02"], ) event: str = Field( @@ -603,24 +492,22 @@ class Content(BaseModel): description=( "True if data object represents an entity in the stratigraphic column" ), - examples=[True], ) is_prediction: bool = Field( - examples=[True], title="Is prediction flag", ) is_observation: bool = Field( - examples=[True], title="Is observation flag", ) # end + base: Optional[Layer] = None + top: Optional[Layer] = None + class DepthContent(Content): content: Literal["depth"] - depth_reference: Literal["msl", "sb", "rkb"] = Field( - examples=["msl"], - ) + depth_reference: Literal["msl", "sb", "rkb"] class FieldOutlineContent(Content): @@ -645,18 +532,14 @@ class FluidContactContent(Content): class SeismicContent(Content): - content: Literal["fluid_contact"] + content: Literal["seismic"] seismic: Seismic = Field( description="Conditional field", ) class TheDataBlock(BaseModel): - # From schema - - # end alias: Optional[list[str]] = None - base: Optional[Layer] = None bbox: Optional[BoundingBox] = None description: list[str] = Field( default_factory=list, @@ -668,13 +551,12 @@ class TheDataBlock(BaseModel): ) offset: float = Field( default=0.0, - examples=[11.2], ) seismic: Optional[Seismic] = Field( default=None, description="Conditional field", ) - spec: Optional[Spec] = None + # spec: Optional[Spec] = None stratigraphic_alias: Optional[list[str]] = None tagname: Optional[str] = Field( default=None, @@ -682,11 +564,9 @@ class TheDataBlock(BaseModel): examples=["ds_extract_geogrid", "ds_post_strucmod"], ) time: Optional[Time] = None - top: Optional[Layer] = None undef_is_zero: Optional[bool] = Field( default=None, description="Flag if undefined values are to be interpreted as zero", - examples=["True"], ) unit: str = Field( default="", @@ -717,10 +597,6 @@ class ClassMeta(BaseModel): ) -class CaseMeta(ClassMeta): - class_: Literal["case"] - - class Meta(BaseModel): access: Access data: Optional[TheDataBlock] = None diff --git a/src/fmu/dataio/models/meta3.py b/src/fmu/dataio/models/meta3.py index 306f130c0..1c24f1aa4 100644 --- a/src/fmu/dataio/models/meta3.py +++ b/src/fmu/dataio/models/meta3.py @@ -1,7 +1,7 @@ from __future__ import annotations from datetime import datetime -from typing import Any, Dict, List, Literal, Optional, Union +from typing import List, Literal, Union from pydantic import BaseModel, Field, RootModel From d709e9272ed2808f25b2c8ef210e838aab11cccb Mon Sep 17 00:00:00 2001 From: JB Lovland Date: Mon, 22 Jan 2024 13:29:36 +0100 Subject: [PATCH 10/25] JB and Per pydantic --- Dockerfile | 20 +- ex.py | 1 + radixconfig.yaml | 2 + schema/app/install-schema-files.sh | 36 -- src/fmu/dataio/models/meta2.py | 388 +++++++++++------ tests/test_schema/test_schema_logic.py | 5 +- .../test_schema/test_schema_logic_pydantic.py | 406 ++++++++++++++++++ 7 files changed, 678 insertions(+), 180 deletions(-) delete mode 100644 schema/app/install-schema-files.sh create mode 100644 tests/test_schema/test_schema_logic_pydantic.py diff --git a/Dockerfile b/Dockerfile index 7f957f36e..dbf18710f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,27 +1,13 @@ FROM nginxinc/nginx-unprivileged:alpine -ARG uriprefix -ENV URIPREFIX=${uriprefix} - WORKDIR /app USER 0 -RUN apk update -RUN apk upgrade -RUN apk add jq - -COPY schema/definitions definitions/ -RUN chown -R 101:101 . - -COPY schema/app/install-schema-files.sh /docker-entrypoint.d/50-install-schema-files.sh -RUN chmod +x /docker-entrypoint.d/50-install-schema-files.sh - -COPY schema/app/fmu-schemas.conf /etc/nginx/conf.d - RUN rm /etc/nginx/conf.d/default.conf +COPY schema/app/fmu-schemas.conf /etc/nginx/conf.d +COPY schema/definitions schemas/ +RUN chown -R 101:101 . EXPOSE 8080 USER 101 CMD ["nginx", "-g", "daemon off;"] - -# CMD "/bin/sh" diff --git a/ex.py b/ex.py index aeeace48b..f05beb97a 100644 --- a/ex.py +++ b/ex.py @@ -11,6 +11,7 @@ def read(file): with open(file) as f: return f.read() + for file in (f.strip() for f in sys.stdin.readlines()): print(file) try: diff --git a/radixconfig.yaml b/radixconfig.yaml index 9d7a56876..3fbaa7ac9 100644 --- a/radixconfig.yaml +++ b/radixconfig.yaml @@ -1,3 +1,5 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/equinor/radix-operator/release/json-schema/radixapplication.json +# Only works if the redhat yaml ext. in vs-code is installed. apiVersion: radix.equinor.com/v1 kind: RadixApplication metadata: diff --git a/schema/app/install-schema-files.sh b/schema/app/install-schema-files.sh deleted file mode 100644 index 4ca338d32..000000000 --- a/schema/app/install-schema-files.sh +++ /dev/null @@ -1,36 +0,0 @@ -#! /bin/sh - -die () { - exitcode="$1"; shift - echo $* - exit $exitcode -} - -if [ -z "$URIPREFIX" ]; then - test \! -z "$RADIX_PUBLIC_DOMAIN_NAME" || die 1 "Unable to build PREFIX." - URIPREFIX="https://${RADIX_PUBLIC_DOMAIN_NAME}" -fi - -echo "URIPREFIX=$URIPREFIX" - -test -d ./definitions || die 1 "Definitions not found." - -mkdir schemas || die 1 "Unable to create directory 'schemas'." - -find ./definitions -type d -name schema -print | \ - while read dir; do - pdir=`dirname "$dir"` - sdir=schemas/`basename "$pdir"` - mkdir "$sdir" - find "$dir" -maxdepth 1 -name '*.json' -print | \ - while read file; do - #echo "file: $file" - ffile=`basename "$file"` - id=`echo "$URIPREFIX"/"$sdir"/"$ffile" | sed -e 's/ /%20/g'` - #echo "id: $id" - #echo "creating $sdir/$ffile" - cat "$file" | jq '.["$id"]='\""$id"\" > "$sdir"/"$ffile" - done - done - - diff --git a/src/fmu/dataio/models/meta2.py b/src/fmu/dataio/models/meta2.py index f8cb95766..13f38c9d6 100644 --- a/src/fmu/dataio/models/meta2.py +++ b/src/fmu/dataio/models/meta2.py @@ -3,9 +3,9 @@ import json from collections import ChainMap from pathlib import Path -from typing import Dict, Literal, Optional, Union +from typing import Dict, Literal, Optional, Union, Annotated -from pydantic import BaseModel, ByteSize, Field, NaiveDatetime, RootModel +from pydantic import BaseModel, Field, NaiveDatetime, RootModel, Discriminator from pydantic.json_schema import GenerateJsonSchema @@ -21,20 +21,27 @@ class Asset(BaseModel): class Ssdl(BaseModel): + """ + Sub-Surface Data Lake + """ + access_level: Literal["internal", "restricted", "asset"] rep_include: bool class Access(BaseModel): asset: Asset - classification: Literal["internal", "restricted"] | None = Field(default=None) - ssdl: Ssdl | None = Field(default=None) + classification: Literal["internal", "restricted", "asset"] | None = Field( + default=None + ) + + +class SsdlAccess(Access): + ssdl: Ssdl class GridModel(BaseModel): - name: str = Field( - examples=["MyGrid"], - ) + name: str = Field(examples=["MyGrid"]) class Shape(BaseModel): @@ -201,7 +208,7 @@ class File(BaseModel): description="md5 checksum of the file or bytestring", examples=["kjhsdfvsdlfk23knerknvk23"], ) - size_bytes: ByteSize | None = Field( + size_bytes: int | None = Field( default=None, description="Size of file object in bytes", ) @@ -419,6 +426,8 @@ class FMUTimeObject(BaseModel): class TracklogEvent(BaseModel): + # TODO: Update ex. to inc. timezone + # update NaiveDatetime -> AwareDateime datetime: NaiveDatetime = Field( examples=["2020-10-28T14:28:02"], ) @@ -428,17 +437,30 @@ class TracklogEvent(BaseModel): user: User -class Fmu(BaseModel): +class FMUCase(BaseModel): + case: Case + model: Model + + +class FMUDataObj(FMUCase): + iteration: Optional[Iteration] = None + workflow: Optional[Workflow] = None + + +class FmuAggregation(FMUDataObj): """ The FMU block records properties that are specific to FMU """ - aggregation: Optional[Aggregation] = None - case: Case - iteration: Optional[Iteration] = None - model: Model - realization: Optional[Realization] = None - workflow: Optional[Workflow] = None + aggregation: Aggregation + + +class FmuRealization(FMUDataObj): + """ + The FMU block records properties that are specific to FMU + """ + + realization: Realization class Time(BaseModel): @@ -447,7 +469,6 @@ class Time(BaseModel): class Content(BaseModel): - # From schema content: Literal[ "depth", "time", @@ -478,6 +499,31 @@ class Content(BaseModel): description="The contents of this data object", examples=["depth"], ) + + alias: Optional[list[str]] = Field(default=None) + base: Optional[Layer] = None + + # Only valid for cooridate based meta. + bbox: Optional[BoundingBox] = Field(default=None) + + description: Optional[list[str]] = Field( + default=None, + ) + format: str = Field( + examples=["irap_binary"], + ) + + grid_model: Optional[GridModel] = Field(default=None) + is_observation: bool = Field( + title="Is observation flag", + ) + is_prediction: bool = Field( + title="Is prediction flag", + ) + layout: Optional[str] = Field( + default=None, + examples=["regular"], + ) name: str = Field( description=( "Name of the data object. If stratigraphic, " @@ -485,31 +531,51 @@ class Content(BaseModel): ), examples=["VIKING GP. Top"], ) - format: str = Field( - examples=["irap_binary"], + offset: float = Field( + default=0.0, ) + # spec: Optional[TableSpec | CPGridSpec, ...] = None + stratigraphic_alias: Optional[list[str]] = Field(default=None) stratigraphic: bool = Field( description=( "True if data object represents an entity in the stratigraphic column" ), ) - is_prediction: bool = Field( - title="Is prediction flag", - ) - is_observation: bool = Field( - title="Is observation flag", + tagname: Optional[str] = Field( + default=None, + description="A semi-human readable tag for internal usage and uniqueness", + examples=["ds_extract_geogrid", "ds_post_strucmod"], ) - # end - - base: Optional[Layer] = None + time: Optional[Time] = Field(default=None) top: Optional[Layer] = None + undef_is_zero: Optional[bool] = Field( + default=None, + description="Flag if undefined values are to be interpreted as zero", + ) + unit: str = Field( + default="", + examples=["m"], + ) + vertical_domain: Optional[Literal["depth", "time"]] = Field( + default=None, + examples=["depth"], + ) + class DepthContent(Content): content: Literal["depth"] depth_reference: Literal["msl", "sb", "rkb"] +class FaultLinesContent(Content): + content: Literal["fault_lines"] + + +class Field_regionContent(Content): + content: Literal["field_region"] + + class FieldOutlineContent(Content): content: Literal["field_outline"] field_outline: FieldOutline = Field( @@ -531,51 +597,87 @@ class FluidContactContent(Content): ) -class SeismicContent(Content): +class InplaceVolumesContent(Content): + content: Literal["inplace_volumes"] + + +class KPProductContent(Content): + content: Literal["khproduct"] + + +class LiftCurvesContent(Content): + content: Literal["lift_curves"] + + +class ParametersContent(Content): + content: Literal["parameters"] + + +class PinchoutContent(Content): + content: Literal["pinchout"] + + +class PropertyContent(Content): + content: Literal["property"] + + +class PTVContent(Content): + content: Literal["pvt"] + + +class RegionsContent(Content): + content: Literal["regions"] + + +class RelpermContent(Content): + content: Literal["relperm"] + + +class RFTContent(Content): + content: Literal["rft"] + + +class SeismicContent(Content): # TheDatablock w/seismic content: Literal["seismic"] seismic: Seismic = Field( description="Conditional field", ) -class TheDataBlock(BaseModel): - alias: Optional[list[str]] = None - bbox: Optional[BoundingBox] = None - description: list[str] = Field( - default_factory=list, - ) - grid_model: Optional[GridModel] = None - layout: Optional[str] = Field( - default=None, - examples=["regular"], - ) - offset: float = Field( - default=0.0, - ) - seismic: Optional[Seismic] = Field( - default=None, - description="Conditional field", - ) - # spec: Optional[Spec] = None - stratigraphic_alias: Optional[list[str]] = None - tagname: Optional[str] = Field( - default=None, - description="A semi-human readable tag for internal usage and uniqueness", - examples=["ds_extract_geogrid", "ds_post_strucmod"], - ) - time: Optional[Time] = None - undef_is_zero: Optional[bool] = Field( - default=None, - description="Flag if undefined values are to be interpreted as zero", - ) - unit: str = Field( - default="", - examples=["m"], - ) - vertical_domain: Optional[Literal["depth", "time"]] = Field( - default=None, - examples=["depth"], - ) +class SubcropContent(Content): + content: Literal["subcrop"] + + +class ThicknessContent(Content): + content: Literal["thickness"] + + +class TimeContent(Content): + content: Literal["time"] + + +class TimeSeriesContent(Content): + content: Literal["timeseries"] + + +class TransmissibilitiesContent(Content): + content: Literal["transmissibilities"] + + +class VelocityContent(Content): + content: Literal["velocity"] + + +class VolumesContent(Content): + content: Literal["volumes"] + + +class VolumetricsContent(Content): + content: Literal["volumetrics"] + + +class WellPicksContent(Content): + content: Literal["wellpicks"] class ClassMeta(BaseModel): @@ -597,75 +699,115 @@ class ClassMeta(BaseModel): ) -class Meta(BaseModel): +class FMUCaseClassMeta(ClassMeta): + class_: Literal["case"] + fmu: FMUCase access: Access - data: Optional[TheDataBlock] = None - file: Optional[File] = None - fmu: Fmu = Field( - description="The FMU block records properties that are specific to FMU", - ) masterdata: Masterdata + tracklog: list[TracklogEvent] source: Literal["fmu"] = Field( description="Data source (FMU)", ) - tracklog: list[TracklogEvent] version: Literal["0.8.0"] = Field( title="FMU results metadata version", ) -if __name__ == "__main__": - dumped = ChainMap( - { - "$id": "https://main-fmu-schemas-dev.radix.equinor.com/schemas/0.8.0/fmu_results.json", - "$schema": GenerateJsonSchema.schema_dialect, - "$contractual": [ - "class", - "source", - "version", - "tracklog", - "data.format", - "data.name", - "data.stratigraphic", - "data.alias", - "data.stratigraphic_alias", - "data.offset", - "data.content", - "data.tagname", - "data.vertical_domain", - "data.grid_model", - "data.bbox", - "data.time", - "data.is_prediction", - "data.is_observation", - "data.seismic.attribute", - "data.spec.columns", - "access", - "masterdata", - "fmu.model", - "fmu.workflow", - "fmu.case", - "fmu.iteration.name", - "fmu.iteration.uuid", - "fmu.realization.name", - "fmu.realization.id", - "fmu.realization.uuid", - "fmu.aggregation.operation", - "fmu.aggregation.realization_ids", - "fmu.context.stage", - "file.relative_path", - "file.checksum_md5", - "file.size_bytes", - ], - }, - Meta.model_json_schema( - by_alias=True, - ), +class FMUDataClassMeta(ClassMeta): + class_: Literal[ + "surface", + "table", + "cpgrid", + "cpgrid_property", + "polygons", + "cube", + "well", + "points", + "dictionary", + ] + fmu: Union[FmuAggregation, FmuRealization] # Hmmmmmmm.....? + access: SsdlAccess + data: Union[ + FieldOutlineContent, + SeismicContent, + FieldRegionContent, + ] + file: File # Case will not have a file obj. + + masterdata: Masterdata + + tracklog: list[TracklogEvent] + + source: Literal["fmu"] = Field( + description="Data source (FMU)", + ) + version: Literal["0.8.0"] = Field( + description="FMU results metadata version", ) - print( - json.dumps( - dict(dumped), - indent=2, + +class Root( + RootModel[ + Union[ + FMUCaseClassMeta, + FMUDataClassMeta, + ], + ] +): + ... + + +def dump() -> dict: + return dict( + ChainMap( + { + "$id": "https://main-fmu-schemas-dev.radix.equinor.com/schemas/0.8.0/fmu_results.json", + "$schema": GenerateJsonSchema.schema_dialect, + "$contractual": [ + "class", + "source", + "version", + "tracklog", + "data.format", + "data.name", + "data.stratigraphic", + "data.alias", + "data.stratigraphic_alias", + "data.offset", + "data.content", + "data.tagname", + "data.vertical_domain", + "data.grid_model", + "data.bbox", + "data.time", + "data.is_prediction", + "data.is_observation", + "data.seismic.attribute", + "data.spec.columns", + "access", + "masterdata", + "fmu.model", + "fmu.workflow", + "fmu.case", + "fmu.iteration.name", + "fmu.iteration.uuid", + "fmu.realization.name", + "fmu.realization.id", + "fmu.realization.uuid", + "fmu.aggregation.operation", + "fmu.aggregation.realization_ids", + "fmu.context.stage", + "file.relative_path", + "file.checksum_md5", + "file.size_bytes", + ], + }, + Root.model_json_schema( + by_alias=True, + ), ) ) + + +if __name__ == "__main__": + print(json.dumps(dump(), indent=2)) diff --git a/tests/test_schema/test_schema_logic.py b/tests/test_schema/test_schema_logic.py index f9997a82e..98a7ea2e3 100644 --- a/tests/test_schema/test_schema_logic.py +++ b/tests/test_schema/test_schema_logic.py @@ -5,7 +5,6 @@ import jsonschema import pytest from fmu.dataio._definitions import ALLOWED_CONTENTS -from fmu.dataio.models.meta2 import Meta # pylint: disable=no-member @@ -37,9 +36,7 @@ def test_schema_080_validate_examples_as_is(schema_080, metadata_examples): """Confirm that examples are valid against the schema""" for i, (name, metadata) in enumerate(metadata_examples.items()): - print("------", name) - jsonschema.validate(instance=metadata, schema=Meta.model_json_schema()) - Meta.model_validate(metadata) + jsonschema.validate(instance=metadata, schema=schema_080) def test_schema_080_file_block(schema_080, metadata_examples): diff --git a/tests/test_schema/test_schema_logic_pydantic.py b/tests/test_schema/test_schema_logic_pydantic.py new file mode 100644 index 000000000..b7320a34f --- /dev/null +++ b/tests/test_schema/test_schema_logic_pydantic.py @@ -0,0 +1,406 @@ +"""Test the schema""" +import logging +from copy import deepcopy + +import jsonschema +import pytest +from fmu.dataio._definitions import ALLOWED_CONTENTS +from fmu.dataio.models.meta2 import dump + +# pylint: disable=no-member + +logger = logging.getLogger(__name__) + + +@pytest.fixture +def pydantic_schema(): + return dump() + + +def test_schema_basic_json_syntax(pydantic_schema): + """Confirm that schemas are valid JSON.""" + + assert "$schema" in pydantic_schema + + +def test_schema_example_filenames(metadata_examples): + """Assert that all examples are .yml, not .yaml""" + + # check that examples are there + assert len(metadata_examples) > 0 + + for filename in metadata_examples: + assert filename.endswith(".yml"), filename + + +# ====================================================================================== +# 0.8.0 +# ====================================================================================== + + +def test_pydantic_schema_validate_examples_as_is(pydantic_schema, metadata_examples): + """Confirm that examples are valid against the schema""" + + for i, (name, metadata) in enumerate(metadata_examples.items()): + jsonschema.validate(instance=metadata, schema=pydantic_schema) + + +def test_pydantic_schema_file_block(pydantic_schema, metadata_examples): + """Test variations on the file block.""" + + # get a specific example + example = metadata_examples["surface_depth.yml"] + + # shall validate as-is + jsonschema.validate(instance=example, schema=pydantic_schema) + + # shall validate without absolute_path + _example = deepcopy(example) + del _example["file"]["absolute_path"] + jsonschema.validate(instance=_example, schema=pydantic_schema) + + # md5 checksum shall be a string + _example["file"]["checksum_md5"] = 123.4 + with pytest.raises(jsonschema.exceptions.ValidationError): + jsonschema.validate(instance=_example, schema=pydantic_schema) + + # shall not validate without checksum_md5 + del _example["file"]["checksum_md5"] + with pytest.raises(jsonschema.exceptions.ValidationError): + jsonschema.validate(instance=_example, schema=pydantic_schema) + + # shall validate when checksum is put back in + _example["file"]["checksum_md5"] = "somechecksum" + jsonschema.validate(instance=_example, schema=pydantic_schema) + + # shall not validate without relative_path + del _example["file"]["relative_path"] + with pytest.raises(jsonschema.exceptions.ValidationError): + jsonschema.validate(instance=_example, schema=pydantic_schema) + + +def test_pydantic_schema_logic_case(pydantic_schema, metadata_examples): + """Asserting validation failure when illegal contents in case example""" + + example = metadata_examples["case.yml"] + + # assert validation with no changes + jsonschema.validate(instance=example, schema=pydantic_schema) + + # assert validation error when "fmu" is missing + _example = deepcopy(example) + del _example["fmu"] + + with pytest.raises(jsonschema.exceptions.ValidationError): + jsonschema.validate(instance=_example, schema=pydantic_schema) + + # assert validation error when "fmu.model" is missing + _example = deepcopy(example) + del _example["fmu"]["model"] + + with pytest.raises(jsonschema.exceptions.ValidationError): + jsonschema.validate(instance=_example, schema=pydantic_schema) + + +def test_pydantic_schema_logic_fmu_block_aggr_real(pydantic_schema, metadata_examples): + """Test that fmu.realization and fmu.aggregation are not allowed at the same time""" + + metadata = deepcopy(metadata_examples["surface_depth.yml"]) + # check that assumptions for the test is true + assert "realization" in metadata["fmu"] + assert "aggregation" not in metadata["fmu"] + + # assert validation as-is + jsonschema.validate(instance=metadata, schema=pydantic_schema) + + # add aggregation, shall fail. Get this from an actual example that validates. + _metadata_aggregation = metadata_examples["aggregated_surface_depth.yml"] + metadata["fmu"]["aggregation"] = _metadata_aggregation["fmu"]["aggregation"] + + with pytest.raises(jsonschema.exceptions.ValidationError): + jsonschema.validate(instance=metadata, schema=pydantic_schema) + + +def test_pydantic_schema_logic_data_top_base(pydantic_schema, metadata_examples): + """Test require data.top and data.base. + + * Require both data.top and data.base, or none. + """ + + metadata = metadata_examples["surface_seismic_amplitude.yml"] + + # check that assumptions for the test is true + assert "top" in metadata["data"] + assert "base" in metadata["data"] + + # assert validation as-is + jsonschema.validate(instance=metadata, schema=pydantic_schema) + + # remove "top" - shall fail + _metadata = deepcopy(metadata) + del _metadata["data"]["top"] + with pytest.raises(jsonschema.exceptions.ValidationError): + jsonschema.validate(instance=_metadata, schema=pydantic_schema) + + # remove "base" - shall fail + _metadata = deepcopy(metadata) + del _metadata["data"]["base"] + with pytest.raises(jsonschema.exceptions.ValidationError): + jsonschema.validate(instance=_metadata, schema=pydantic_schema) + + # remove both - shall pass + del _metadata["data"]["top"] + assert "top" not in _metadata["data"] # test assumption + assert "base" not in _metadata["data"] # test assumption + jsonschema.validate(instance=_metadata, schema=pydantic_schema) + + +def test_pydantic_schema_logic_field_outline(pydantic_schema, metadata_examples): + """Test content-specific rule. + + When content == field_outline, require the field_outline field + """ + + metadata = metadata_examples["polygons_field_outline.yml"] + + # check that assumptions for the test is true + assert metadata["data"]["content"] == "field_outline" + assert "field_outline" in metadata["data"] + + # assert validation as-is + jsonschema.validate(instance=metadata, schema=pydantic_schema) + + # assert failure when content is field_outline and fluid_contact is missing + _metadata = deepcopy(metadata) + del _metadata["data"]["field_outline"] + with pytest.raises(jsonschema.exceptions.ValidationError): + jsonschema.validate(instance=_metadata, schema=pydantic_schema) + + +def test_pydantic_schema_logic_field_region(pydantic_schema, metadata_examples): + """Test content-specific rule: field_region + + When content == field_outline, require the data.field_region field. + """ + + metadata = metadata_examples["polygons_field_region.yml"] + + # check assumptions + assert metadata["data"]["content"] == "field_region" + assert "field_region" in metadata["data"] + assert "id" in metadata["data"]["field_region"] + jsonschema.validate(instance=metadata, schema=pydantic_schema) + + # assert that data.field_region is required + _metadata = deepcopy(metadata) + del _metadata["data"]["field_region"] + with pytest.raises(jsonschema.exceptions.ValidationError): + jsonschema.validate(instance=_metadata, schema=pydantic_schema) + + # validation of data.field_region + _metadata = deepcopy(metadata) + del _metadata["data"]["field_region"]["id"] + with pytest.raises(jsonschema.exceptions.ValidationError): + jsonschema.validate(instance=_metadata, schema=pydantic_schema) + + _metadata = deepcopy(metadata) + _metadata["data"]["field_region"]["id"] = "NotANumber" + with pytest.raises(jsonschema.exceptions.ValidationError): + jsonschema.validate(instance=_metadata, schema=pydantic_schema) + + +def test_pydantic_schema_logic_fluid_contact(pydantic_schema, metadata_examples): + """Test content-specific rule. + + When content == fluid_contact, require the fluid_contact field + """ + + # parse the schema and polygons + metadata = metadata_examples["surface_fluid_contact.yml"] + + # check that assumptions for the test is true + assert metadata["data"]["content"] == "fluid_contact" + assert "fluid_contact" in metadata["data"] + + # assert failure when content is fluid_contact and fluid_contact block missing + _metadata = deepcopy(metadata) + del _metadata["data"]["fluid_contact"] + with pytest.raises(jsonschema.exceptions.ValidationError): + jsonschema.validate(instance=_metadata, schema=pydantic_schema) + + +def test_pydantic_schema_masterdata_smda(pydantic_schema, metadata_examples): + """Test schema logic for masterdata.smda.""" + + example = metadata_examples["case.yml"] + + # assert validation with no changes + jsonschema.validate(instance=example, schema=pydantic_schema) + + # assert validation error when masterdata block is missing + _example = deepcopy(example) + del _example["masterdata"] + with pytest.raises(jsonschema.exceptions.ValidationError): + jsonschema.validate(instance=_example, schema=pydantic_schema) + + # assert validation error when masterdata.smda is missing + # print(example["masterdata"]) + _example = deepcopy(example) + del _example["masterdata"]["smda"] + with pytest.raises(jsonschema.exceptions.ValidationError): + jsonschema.validate(instance=_example, schema=pydantic_schema) + + # assert validation error when missing attribute + for block in [ + "country", + "discovery", + "field", + "coordinate_system", + "stratigraphic_column", + ]: + _example = deepcopy(example) + del _example["masterdata"]["smda"][block] + with pytest.raises(jsonschema.exceptions.ValidationError): + jsonschema.validate(instance=_example, schema=pydantic_schema) + + # assert validation error if not correct type + for block, type_ in [ + ("country", list), + ("discovery", list), + ("coordinate_system", dict), + ("stratigraphic_column", dict), + ]: + _example = deepcopy(example) + assert isinstance(_example["masterdata"]["smda"][block], type_) + + _example["masterdata"]["smda"][block] = "somestring" + + with pytest.raises(jsonschema.exceptions.ValidationError): + jsonschema.validate(instance=_example, schema=pydantic_schema) + + +def test_pydantic_schema_data_time(pydantic_schema, metadata_examples): + """Test schema logic for data.time.""" + + # fetch one example that contains the data.time element + example = metadata_examples["surface_seismic_amplitude.yml"] + assert "time" in example["data"] + + # assert validation with no changes + jsonschema.validate(instance=example, schema=pydantic_schema) + + # valid when data.time is missing + _example = deepcopy(example) + del _example["data"]["time"] + jsonschema.validate(instance=_example, schema=pydantic_schema) + + # valid when only t0 + _example = deepcopy(example) + del _example["data"]["time"]["t1"] + assert "t0" in _example["data"]["time"] # test assumption + jsonschema.validate(instance=_example, schema=pydantic_schema) + + # valid without labels + _example = deepcopy(example) + del _example["data"]["time"]["t0"]["label"] + jsonschema.validate(instance=_example, schema=pydantic_schema) + + # NOT valid when other types + for testvalue in [ + [{"t0": "2020-10-28T14:28:02", "label": "mylabel"}], + "2020-10-28T14:28:02", + 123, + 123.4, + ]: + _example = deepcopy(example) + _example["data"]["time"] = testvalue + with pytest.raises(jsonschema.exceptions.ValidationError): + jsonschema.validate(instance=_example, schema=pydantic_schema) + + +def test_schema_logic_classification(pydantic_schema, metadata_examples): + """Test the classification of individual files.""" + + # fetch example + example = deepcopy(metadata_examples["surface_depth.yml"]) + + # assert validation with no changes + jsonschema.validate(instance=example, schema=pydantic_schema) + + # assert "internal" and "restricted" validates + example["access"]["classification"] = "internal" + jsonschema.validate(instance=example, schema=pydantic_schema) + + example["access"]["classification"] = "restricted" + jsonschema.validate(instance=example, schema=pydantic_schema) + + # assert erroneous value does not validate + example["access"]["classification"] = "open" + with pytest.raises(jsonschema.exceptions.ValidationError): + jsonschema.validate(instance=example, schema=pydantic_schema) + + +def test_schema_logic_data_spec(pydantic_schema, metadata_examples): + """Test schema logic for data.spec""" + + # fetch surface example + example_surface = deepcopy(metadata_examples["surface_depth.yml"]) + + # assert validation with no changes + jsonschema.validate(instance=example_surface, schema=pydantic_schema) + + # assert data.spec required when class == surface + del example_surface["data"]["spec"] + with pytest.raises(jsonschema.exceptions.ValidationError): + jsonschema.validate(instance=example_surface, schema=pydantic_schema) + + # fetch table example + example_table = deepcopy(metadata_examples["table_inplace.yml"]) + + # assert validation with no changes + jsonschema.validate(instance=example_table, schema=pydantic_schema) + + # assert data.spec required when class == table + del example_table["data"]["spec"] + with pytest.raises(jsonschema.exceptions.ValidationError): + jsonschema.validate(instance=example_table, schema=pydantic_schema) + + # fetch dictionary example + example_dict = deepcopy(metadata_examples["dictionary_parameters.yml"]) + + # assert data.spec is not present + with pytest.raises(KeyError): + example_dict["data"]["spec"] + + # assert data.spec not required when class === dictionary + jsonschema.validate(instance=example_dict, schema=pydantic_schema) + + +def test_schema_logic_content_whitelist(pydantic_schema, metadata_examples): + """Test that validation fails when value of data.content is not in + the whitelist.""" + + # fetch surface example + example_surface = deepcopy(metadata_examples["surface_depth.yml"]) + + # assert validation with no changes + jsonschema.validate(instance=example_surface, schema=pydantic_schema) + + # shall fail when content is not in whitelist + example_surface["data"]["content"] = "not_valid_content" + with pytest.raises(jsonschema.exceptions.ValidationError): + jsonschema.validate(instance=example_surface, schema=pydantic_schema) + + +def test_schema_content_synch_with_code(pydantic_schema): + """Currently, the whitelist for content is maintained both in the schema + and in the code. This test asserts that list used in the code is in synch + with schema. Note! This is one-way, and will not fail if additional + elements are added to the schema only.""" + + schema_allowed = pydantic_schema["$defs"]["data"]["properties"]["content"]["enum"] + for allowed_content in ALLOWED_CONTENTS: + if allowed_content not in schema_allowed: + raise ValueError( + f"content '{allowed_content}' allowed in code, but not schema." + ) From 68330f48ac67f13d33f8db119b683a6740762f4a Mon Sep 17 00:00:00 2001 From: JB Lovland Date: Mon, 22 Jan 2024 17:10:35 +0100 Subject: [PATCH 11/25] Split model into meta and content files --- src/fmu/dataio/models/eninja.md | 16 - src/fmu/dataio/models/meta.py | 575 ------------------ src/fmu/dataio/models/meta/__init__.py | 6 + src/fmu/dataio/models/meta/__main__.py | 8 + src/fmu/dataio/models/meta/content.py | 351 +++++++++++ .../dataio/models/{meta2.py => meta/meta.py} | 349 +---------- src/fmu/dataio/models/meta3.py | 49 -- 7 files changed, 372 insertions(+), 982 deletions(-) delete mode 100644 src/fmu/dataio/models/eninja.md delete mode 100644 src/fmu/dataio/models/meta.py create mode 100644 src/fmu/dataio/models/meta/__init__.py create mode 100644 src/fmu/dataio/models/meta/__main__.py create mode 100644 src/fmu/dataio/models/meta/content.py rename src/fmu/dataio/models/{meta2.py => meta/meta.py} (60%) delete mode 100644 src/fmu/dataio/models/meta3.py diff --git a/src/fmu/dataio/models/eninja.md b/src/fmu/dataio/models/eninja.md deleted file mode 100644 index 1f1446398..000000000 --- a/src/fmu/dataio/models/eninja.md +++ /dev/null @@ -1,16 +0,0 @@ -e ninja - -### Find all classes that do not have att. "data.vertical_domain" - -{ -"query": { -"bool": { - "must_not": [{"exists":{"field":"data.vertical_domain"}}] -} -}, -"aggs":{ - "class":{ -"terms":{"field":"class.keyword"} -} -} -} \ No newline at end of file diff --git a/src/fmu/dataio/models/meta.py b/src/fmu/dataio/models/meta.py deleted file mode 100644 index 7512c0995..000000000 --- a/src/fmu/dataio/models/meta.py +++ /dev/null @@ -1,575 +0,0 @@ -# generated by datamodel-codegen: -# filename: fmu_results.json -# version: 0.25.2 - -from __future__ import annotations - -from typing import Any, Literal, Optional, Union - -from pydantic import BaseModel, Field, RootModel - - -class Generic(RootModel[Any]): - root: Any - - -class Property(BaseModel): - name: Optional[str] = Field(default=None, examples=["MyPropertyName"]) - attribute: Optional[str] = Field(default=None, examples=["MyAttributeName"]) - is_discrete: Optional[bool] = Field(default=None, examples=[True, False]) - - -class GridModel(BaseModel): - name: Optional[str] = Field(default=None, examples=["MyGrid"]) - - -class Spec(BaseModel): - ncol: Optional[int] = Field(default=None, examples=[281]) - nrow: Optional[int] = Field(default=None, examples=[441]) - nlay: Optional[int] = Field(default=None, examples=[333]) - xori: Optional[float] = Field(default=None, examples=[461499.9997558594]) - yori: Optional[float] = Field(default=None, examples=[5926500.224123242]) - xinc: Optional[float] = Field(default=None, examples=[25.0]) - yflip: Optional[Literal[-1, 1]] = Field(default=None, examples=[-1, 1]) - rotation: Optional[float] = Field(default=None, examples=["30.00000000231"]) - undef: Optional[float] = Field(default=None, examples=[1e33]) - npolys: Optional[int] = Field( - default=None, - description="The number of individual polygons in the data object", - examples=[1], - ) - size: Optional[int] = Field( - default=None, description="Size of data object.", examples=[1, 9999] - ) - columns: Optional[list[str]] = Field( - default=None, description="List of columns present in a table." - ) - - -class Bbox(BaseModel): - xmin: float = Field(..., examples=[456012.5003497944]) - xmax: float = Field(..., examples=[467540.52762886323]) - ymin: float = Field(..., examples=[5926499.999511719]) - ymax: float = Field(..., examples=[5939492.128326312]) - zmin: Optional[float] = Field(default=None, examples=[1244.039, None]) - zmax: Optional[float] = Field(default=None, examples=[2302.683, None]) - - -class FluidContact(BaseModel): - """ - Conditional field - """ - - contact: Optional[Literal["owc", "fwl", "goc", "fgl"]] = Field( - default=None, examples=["owc", "fwl"] - ) - truncated: Optional[bool] = Field(default=None, examples=[True]) - - -class FieldOutline(BaseModel): - """ - Conditional field - """ - - contact: str - - -class FieldRegion(BaseModel): - """ - Conditional field - """ - - id: int = Field(..., description="The ID of the region") - - -class Seismic(BaseModel): - """ - Conditional field - """ - - attribute: Optional[str] = Field(default=None, examples=["amplitude_timeshifted"]) - calculation: Optional[str] = Field(default=None, examples=["mean"]) - zrange: Optional[float] = Field(default=None, examples=[12.0]) - filter_size: Optional[float] = Field(default=None, examples=[1.0]) - scaling_factor: Optional[float] = Field(default=None, examples=[1.0]) - stacking_offset: Optional[str] = Field(default=None, examples=["0-15"]) - - -class Line(BaseModel): - show: Optional[bool] = Field(default=None, examples=[True]) - color: Optional[str] = Field(default=None, examples=["black", "#000000"]) - - -class Points(BaseModel): - show: Optional[bool] = Field(default=None, examples=[True]) - color: Optional[str] = Field(default=None, examples=["black", "#000000"]) - - -class Contours(BaseModel): - show: Optional[bool] = Field(default=None, examples=[True]) - color: Optional[str] = Field( - default=None, - description="The color of the contour lines", - examples=["black", "#000000"], - ) - increment: Optional[float] = Field( - default=None, - description="The contour increment in same values as the data", - examples=[20.0], - ) - - -class Fill(BaseModel): - show: Optional[bool] = Field(default=None, examples=[True]) - color: Optional[str] = Field(default=None, examples=["black", "#000000"]) - colormap: Optional[str] = Field( - default=None, - description="named reference to a colormap", - examples=["gist_earth"], - ) - display_min: Optional[float] = Field( - default=None, - description="The low-side boundary to use for fill color", - examples=[1000.0], - ) - display_max: Optional[float] = Field( - default=None, - description="The high-side boundary to use for fill color", - examples=[1600.0], - ) - - -class Display(BaseModel): - name: Optional[str] = Field(default=None, examples=["Top Volantis"]) - subtitle: Optional[str] = Field(default=None, examples=["Some subtitle"]) - line: Optional[Line] = None - points: Optional[Points] = None - contours: Optional[Contours] = None - fill: Optional[Fill] = None - - -class Asset(BaseModel): - name: str = Field(..., examples=["Drogon"]) - - -class Ssdl(BaseModel): - access_level: Literal["internal", "restricted", "asset"] - rep_include: bool = Field(..., examples=[True, False]) - - -class Access(BaseModel): - asset: Optional[Asset] = None - ssdl: Optional[Ssdl] = None - classification: Optional[Literal["internal", "restricted"]] = Field( - default=None, examples=["internal", "restricted"] - ) - - -class Fmu2(RootModel[Any]): - root: Any - - -class File(BaseModel): - """ - Block describing the file as the data appear in FMU context - """ - - relative_path: str = Field( - ..., - description="The file path relative to RUNPATH", - examples=["share/results/maps/volantis_gp_base--depth.gri"], - ) - absolute_path: Optional[str] = Field( - default=None, - description="The absolute file path", - examples=["/abs/path/share/results/maps/volantis_gp_base--depth.gri"], - ) - checksum_md5: str = Field( - ..., - description="md5 checksum of the file or bytestring", - examples=["kjhsdfvsdlfk23knerknvk23"], - ) - size_bytes: Optional[int] = Field( - default=None, description="Size of file object in bytes", examples=[123] - ) - - -class Parameters(BaseModel): - """ - Parameters for this realization - """ - - -class Aggregation(BaseModel): - operation: str = Field(..., description="The aggregation performed") - realization_ids: list[int] = Field( - ..., description="Array of realization ids included in this aggregation" - ) - parameters: Optional[Union[list[dict[str, Any]], Parameters]] = Field( - default=None, description="Parameters for this realization" - ) - id: Optional[str] = Field( - default=None, - description="The ID of this aggregation", - examples=["15ce3b84-766f-4c93-9050-b154861f9100"], - ) - - -class Workflow(BaseModel): - reference: Optional[str] = Field( - default=None, - description="Reference to the part of the FMU workflow that produced this", - ) - - -class User(BaseModel): - id: str = Field(..., examples=["peesv"], title="User ID") - - -class FMUTimeObject(BaseModel): - """ - Time stamp for data object. - """ - - value: Optional[str] = Field(default=None, examples=["2020-10-28T14:28:02"]) - label: Optional[str] = Field(default=None, examples=["base", "monitor", "mylabel"]) - - -class TracklogEvent(BaseModel): - datetime: Optional[str] = Field(default=None, examples=["2020-10-28T14:28:02"]) - user: Optional[User] = None - event: Optional[str] = Field(default=None, examples=["created", "updated"]) - - -class Top(BaseModel): - name: Optional[str] = Field( - default=None, - description="Name of the data object. If stratigraphic, match the entry in the stratigraphic column", - examples=["VIKING GP. Top"], - ) - stratigraphic: Optional[bool] = Field( - default=None, - description="True if data object represents an entity in the stratigraphic column", - examples=[True], - ) - offset: Optional[float] = Field(default=None, examples=[11.2]) - - -class Base(BaseModel): - name: Optional[str] = Field( - default=None, - description="Name of the data object. If stratigraphic, match the entry in the stratigraphic column", - examples=["VIKING GP. Top"], - ) - stratigraphic: Optional[bool] = Field( - default=None, - description="True if data object represents an entity in the stratigraphic column", - examples=[True], - ) - offset: Optional[float] = Field(default=None, examples=[11.2]) - - -class Time(BaseModel): - t0: Optional[FMUTimeObject] = None - t1: Optional[FMUTimeObject] = None - - -class TheDataBlock(BaseModel): - name: str = Field( - ..., - description="Name of the data object. If stratigraphic, match the entry in the stratigraphic column", - examples=["VIKING GP. Top"], - ) - stratigraphic: bool = Field( - ..., - description="True if data object represents an entity in the stratigraphic column", - examples=[True], - ) - alias: Optional[list[str]] = None - stratigraphic_alias: Optional[list[str]] = None - offset: Optional[float] = Field(default=None, examples=[11.2]) - top: Optional[Top] = None - base: Optional[Base] = None - content: Literal[ - "depth", - "time", - "thickness", - "property", - "seismic", - "fluid_contact", - "field_outline", - "field_region", - "regions", - "pinchout", - "subcrop", - "fault_lines", - "velocity", - "volumes", - "volumetrics", - "inplace_volumes", - "khproduct", - "timeseries", - "wellpicks", - "parameters", - "relperm", - "rft", - "pvt", - "lift_curves", - "transmissibilities", - ] = Field( - ..., description="The contents of this data object", examples=["depth", "time"] - ) - tagname: Optional[str] = Field( - default=None, - description="A semi-human readable tag for internal usage and uniqueness", - examples=["ds_extract_geogrid", "ds_post_strucmod"], - ) - properties: Optional[list[Property]] = Field( - default=None, - description="data.properties is an array of property objects, representing the properties present in the data object.", - ) - format: str = Field(..., examples=["irap_binary"]) - layout: Optional[str] = Field(default=None, examples=["regular"]) - unit: Optional[str] = Field(default=None, examples=["m"]) - vertical_domain: Optional[Literal["depth", "time"]] = Field( - default=None, examples=["depth"] - ) - depth_reference: Optional[Literal["msl", "sb", "rkb"]] = Field( - default=None, examples=["msl"] - ) - undef_is_zero: Optional[bool] = Field( - default=None, - description="Flag if undefined values are to be interpreted as zero", - examples=["True"], - ) - grid_model: Optional[GridModel] = None - spec: Optional[Spec] = None - bbox: Optional[Bbox] = None - time: Optional[Time] = None - is_prediction: bool = Field(..., examples=[True], title="Is prediction flag") - is_observation: bool = Field(..., examples=[True], title="Is observation flag") - fluid_contact: Optional[FluidContact] = Field( - default=None, description="Conditional field" - ) - field_outline: Optional[FieldOutline] = Field( - default=None, description="Conditional field" - ) - field_region: Optional[FieldRegion] = Field( - default=None, description="Conditional field" - ) - description: Optional[list[str]] = None - seismic: Optional[Seismic] = Field(default=None, description="Conditional field") - - -class CountryItem(BaseModel): - identifier: str = Field(..., examples=["Norway"]) - uuid: str = Field( - ..., - examples=["ad214d85-8a1d-19da-e053-c918a4889309"], - pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", - ) - - -class DiscoveryItem(BaseModel): - short_identifier: str = Field(..., examples=["SomeDiscovery"]) - uuid: str = Field( - ..., - examples=["ad214d85-8a1d-19da-e053-c918a4889309"], - pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", - ) - - -class FieldItem(BaseModel): - identifier: str = Field(..., examples=["OseFax"]) - uuid: str = Field( - ..., - examples=["ad214d85-8a1d-19da-e053-c918a4889309"], - pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", - ) - - -class CoordinateSystem(BaseModel): - identifier: str = Field(..., examples=["ST_WGS84_UTM37N_P32637"]) - uuid: str = Field( - ..., - examples=["ad214d85-8a1d-19da-e053-c918a4889309"], - pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", - ) - - -class StratigraphicColumn(BaseModel): - identifier: str = Field(..., examples=["DROGON_2020"]) - uuid: str = Field( - ..., - examples=["ad214d85-8a1d-19da-e053-c918a4889309"], - pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", - ) - - -class Smda(BaseModel): - country: list[CountryItem] - discovery: list[DiscoveryItem] - field: list[FieldItem] - coordinate_system: CoordinateSystem - stratigraphic_column: StratigraphicColumn - - -class Masterdata(BaseModel): - smda: Smda - - -class Case(BaseModel): - name: str = Field(..., description="The case name", examples=["MyCaseName"]) - uuid: str = Field( - ..., - examples=["ad214d85-8a1d-19da-e053-c918a4889309"], - pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", - ) - user: User = Field(..., description="The user name used in ERT") - description: Optional[list[str]] = None - - -class Iteration(BaseModel): - name: str = Field( - ..., - description="The convential name of this iteration, e.g. iter-0 or pred", - examples=["iter-0"], - ) - id: Optional[int] = Field( - default=None, - description="The internal identification of this iteration, e.g. the iteration number", - examples=[0], - ) - uuid: str = Field( - ..., - examples=["ad214d85-8a1d-19da-e053-c918a4889309"], - pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", - ) - restart_from: Optional[str] = Field( - default=None, - description="A uuid reference to another iteration that this iteration was restarted from", - examples=["ad214d85-8a1d-19da-e053-c918a4889309"], - pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", - ) - - -class Model(BaseModel): - name: Optional[str] = Field(default=None, examples=["Drogon"]) - revision: Optional[str] = Field(default=None, examples=["21.0.0.dev"]) - description: Optional[list[str]] = Field( - default=None, description="This is a free text description of the model setup" - ) - - -class Realization(BaseModel): - name: str = Field( - ..., - description="The convential name of this iteration, e.g. iter-0 or pred", - examples=["iter-0"], - ) - id: int = Field( - ..., - description="The unique number of this realization as used in FMU", - examples=[33], - ) - uuid: str = Field( - ..., - examples=["ad214d85-8a1d-19da-e053-c918a4889309"], - pattern="^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", - ) - parameters: Optional[Union[list[dict[str, Any]], Parameters]] = Field( - default=None, description="Parameters for this realization" - ) - jobs: Optional[dict[str, Any]] = Field( - default=None, - description="Content directly taken from the ERT jobs.json file for this realization", - ) - - -class Fmu(BaseModel): - """ - The FMU block records properties that are specific to FMU - """ - - model: Model - case: Case - - -class Model1(BaseModel): - source: Literal["fmu"] = Field(..., description="Data source (FMU)") - version: Literal["0.8.0"] = Field( - ..., examples=["1.2.3"], title="FMU results metadata version" - ) - data: Optional[TheDataBlock] = None - tracklog: list[TracklogEvent] - access: Access - masterdata: Masterdata - class_: Literal[ - "case", - "surface", - "table", - "cpgrid", - "cpgrid_property", - "polygons", - "cube", - "well", - "points", - "dictionary", - ] = Field( - ..., - alias="class", - examples=["surface", "table", "points"], - title="Metadata class", - ) - fmu: Fmu = Field( - ..., description="The FMU block records properties that are specific to FMU" - ) - - -class Fmu1(BaseModel): - """ - The FMU block records properties that are specific to FMU - """ - - model: Model - case: Case - iteration: Optional[Iteration] = None - realization: Optional[Realization] = None - workflow: Optional[Workflow] = None - aggregation: Optional[Aggregation] = None - - -class Model2(BaseModel): - source: Literal["fmu"] = Field(..., description="Data source (FMU)") - version: Literal["0.8.0"] = Field( - ..., examples=["1.2.3"], title="FMU results metadata version" - ) - data: TheDataBlock - tracklog: list[TracklogEvent] - access: Access - masterdata: Masterdata - class_: Literal[ - "case", - "surface", - "table", - "cpgrid", - "cpgrid_property", - "polygons", - "cube", - "well", - "points", - "dictionary", - ] = Field( - ..., - alias="class", - examples=["surface", "table", "points"], - title="Metadata class", - ) - fmu: Fmu1 = Field( - ..., description="The FMU block records properties that are specific to FMU" - ) - file: File - - -class ModelModel(RootModel[Union[Model1, Model2]]): - root: Union[Model1, Model2] diff --git a/src/fmu/dataio/models/meta/__init__.py b/src/fmu/dataio/models/meta/__init__.py new file mode 100644 index 000000000..1e35c2d3a --- /dev/null +++ b/src/fmu/dataio/models/meta/__init__.py @@ -0,0 +1,6 @@ +from .meta import Root, dump + +__all__ = [ + "dump", + "Root", +] diff --git a/src/fmu/dataio/models/meta/__main__.py b/src/fmu/dataio/models/meta/__main__.py new file mode 100644 index 000000000..990d73f19 --- /dev/null +++ b/src/fmu/dataio/models/meta/__main__.py @@ -0,0 +1,8 @@ +from __future__ import annotations + +import json + +from . import dump + +if __name__ == "__main__": + print(json.dumps(dump(), indent=2)) diff --git a/src/fmu/dataio/models/meta/content.py b/src/fmu/dataio/models/meta/content.py new file mode 100644 index 000000000..acdf742fa --- /dev/null +++ b/src/fmu/dataio/models/meta/content.py @@ -0,0 +1,351 @@ +from __future__ import annotations + +from typing import Annotated, Literal, Optional, Union + +from pydantic import BaseModel, Field + + +class FMUTimeObject(BaseModel): + """ + Time stamp for data object. + """ + + label: Optional[str] = Field( + default=None, + examples=["base", "monitor", "mylabel"], + ) + value: Optional[str] = Field( + default=None, + examples=["2020-10-28T14:28:02"], + ) + + +class Time(BaseModel): + t0: Optional[FMUTimeObject] = None + t1: Optional[FMUTimeObject] = None + + +class Seismic(BaseModel): + """ + Conditional field + """ + + attribute: Optional[str] = Field( + default=None, + examples=["amplitude_timeshifted"], + ) + calculation: Optional[str] = Field( + default=None, + examples=["mean"], + ) + filter_size: Optional[float] = Field( + default=None, + ) + scaling_factor: Optional[float] = Field( + default=None, + ) + stacking_offset: Optional[str] = Field( + default=None, + examples=["0-15"], + ) + zrange: Optional[float] = Field( + default=None, + ) + + +class FluidContact(BaseModel): + """ + Conditional field + """ + + contact: Literal["owc", "fwl", "goc", "fgl"] = Field( + examples=["owc", "fwl"], + ) + truncated: bool = Field(default=False) + + +class FieldOutline(BaseModel): + """ + Conditional field + """ + + contact: str + + +class FieldRegion(BaseModel): + """ + Conditional field + """ + + id: int = Field( + description="The ID of the region", + ) + + +class GridModel(BaseModel): + name: str = Field(examples=["MyGrid"]) + + +class Layer(BaseModel): + name: str = Field( + description=( + "Name of the data object. If stratigraphic, " + "match the entry in the stratigraphic column" + ), + examples=["VIKING GP. Top"], + ) + offset: float = Field( + default=0, + ) + stratigraphic: bool = Field( + default=False, + description=( + "True if data object represents an entity in the stratigraphic colum" + ), + ) + + +class BoundingBox(BaseModel): + xmin: float = Field(description="Minimum x-coordinate") + xmax: float = Field(description="Maximum x-coordinate") + ymin: float = Field(description="Minimum y-coordinate") + ymax: float = Field(description="Maximum y-coordinate") + zmin: float = Field(description="Minimum z-coordinate") + zmax: float = Field(description="Maximum z-coordinate") + + +class Content(BaseModel): + content: Literal[ + "depth", + "time", + "thickness", + "property", + "seismic", + "fluid_contact", + "field_outline", + "field_region", + "regions", + "pinchout", + "subcrop", + "fault_lines", + "velocity", + "volumes", + "volumetrics", + "inplace_volumes", + "khproduct", + "timeseries", + "wellpicks", + "parameters", + "relperm", + "rft", + "pvt", + "lift_curves", + "transmissibilities", + ] = Field( + description="The contents of this data object", + examples=["depth"], + ) + + alias: Optional[list[str]] = Field(default=None) + base: Optional[Layer] = None + + # Only valid for cooridate based meta. + bbox: Optional[BoundingBox] = Field(default=None) + + description: Optional[list[str]] = Field( + default=None, + ) + format: str = Field( + examples=["irap_binary"], + ) + + grid_model: Optional[GridModel] = Field(default=None) + is_observation: bool = Field( + title="Is observation flag", + ) + is_prediction: bool = Field( + title="Is prediction flag", + ) + layout: Optional[str] = Field( + default=None, + examples=["regular"], + ) + name: str = Field( + description=( + "Name of the data object. If stratigraphic, " + "match the entry in the stratigraphic column" + ), + examples=["VIKING GP. Top"], + ) + offset: float = Field( + default=0.0, + ) + # spec: Optional[TableSpec | CPGridSpec, ...] = None + stratigraphic_alias: Optional[list[str]] = Field(default=None) + stratigraphic: bool = Field( + description=( + "True if data object represents an entity in the stratigraphic column" + ), + ) + tagname: Optional[str] = Field( + default=None, + description="A semi-human readable tag for internal usage and uniqueness", + examples=["ds_extract_geogrid", "ds_post_strucmod"], + ) + time: Optional[Time] = Field(default=None) + top: Optional[Layer] = None + + undef_is_zero: Optional[bool] = Field( + default=None, + description="Flag if undefined values are to be interpreted as zero", + ) + unit: str = Field( + default="", + examples=["m"], + ) + vertical_domain: Optional[Literal["depth", "time"]] = Field( + default=None, + examples=["depth"], + ) + + +class DepthContent(Content): + content: Literal["depth"] + depth_reference: Literal["msl", "sb", "rkb"] + + +class FaultLinesContent(Content): + content: Literal["fault_lines"] + + +class FieldOutlineContent(Content): + content: Literal["field_outline"] + field_outline: FieldOutline = Field( + description="Conditional field", + ) + + +class FieldRegionContent(Content): + content: Literal["field_region"] + field_region: FieldRegion = Field( + description="Conditional field", + ) + + +class FluidContactContent(Content): + content: Literal["fluid_contact"] + fluid_contact: FluidContact = Field( + description="Conditional field", + ) + + +class InplaceVolumesContent(Content): + content: Literal["inplace_volumes"] + + +class KPProductContent(Content): + content: Literal["khproduct"] + + +class LiftCurvesContent(Content): + content: Literal["lift_curves"] + + +class ParametersContent(Content): + content: Literal["parameters"] + + +class PinchoutContent(Content): + content: Literal["pinchout"] + + +class PropertyContent(Content): + content: Literal["property"] + + +class PTVContent(Content): + content: Literal["pvt"] + + +class RegionsContent(Content): + content: Literal["regions"] + + +class RelpermContent(Content): + content: Literal["relperm"] + + +class RFTContent(Content): + content: Literal["rft"] + + +class SeismicContent(Content): + content: Literal["seismic"] + seismic: Seismic = Field( + description="Conditional field", + ) + + +class SubcropContent(Content): + content: Literal["subcrop"] + + +class ThicknessContent(Content): + content: Literal["thickness"] + + +class TimeContent(Content): + content: Literal["time"] + + +class TimeSeriesContent(Content): + content: Literal["timeseries"] + + +class TransmissibilitiesContent(Content): + content: Literal["transmissibilities"] + + +class VelocityContent(Content): + content: Literal["velocity"] + + +class VolumesContent(Content): + content: Literal["volumes"] + + +class VolumetricsContent(Content): + content: Literal["volumetrics"] + + +class WellPicksContent(Content): + content: Literal["wellpicks"] + + +AnyContent = Annotated[ + Union[ + DepthContent, + FaultLinesContent, + FieldRegionContent, + InplaceVolumesContent, + KPProductContent, + LiftCurvesContent, + ParametersContent, + PinchoutContent, + PropertyContent, + PTVContent, + RegionsContent, + RelpermContent, + RFTContent, + SeismicContent, + SubcropContent, + ThicknessContent, + TimeContent, + TimeSeriesContent, + VelocityContent, + VolumesContent, + VolumetricsContent, + WellPicksContent, + ], + Field(discriminator="content"), +] diff --git a/src/fmu/dataio/models/meta2.py b/src/fmu/dataio/models/meta/meta.py similarity index 60% rename from src/fmu/dataio/models/meta2.py rename to src/fmu/dataio/models/meta/meta.py index 13f38c9d6..251ad8611 100644 --- a/src/fmu/dataio/models/meta2.py +++ b/src/fmu/dataio/models/meta/meta.py @@ -1,13 +1,14 @@ from __future__ import annotations -import json from collections import ChainMap from pathlib import Path -from typing import Dict, Literal, Optional, Union, Annotated +from typing import Dict, Literal, Optional, Union -from pydantic import BaseModel, Field, NaiveDatetime, RootModel, Discriminator +from pydantic import BaseModel, Field, NaiveDatetime, RootModel from pydantic.json_schema import GenerateJsonSchema +from . import content + class UUID(RootModel[str]): root: str = Field( @@ -40,10 +41,6 @@ class SsdlAccess(Access): ssdl: Ssdl -class GridModel(BaseModel): - name: str = Field(examples=["MyGrid"]) - - class Shape(BaseModel): nrow: int = Field(description="The number of rows") ncol: int = Field(description="The number of columns") @@ -125,72 +122,6 @@ class DictionarySpec(BaseModel): ... -class BoundingBox(BaseModel): - xmin: float = Field(description="Minimum x-coordinate") - xmax: float = Field(description="Maximum x-coordinate") - ymin: float = Field(description="Minimum y-coordinate") - ymax: float = Field(description="Maximum y-coordinate") - zmin: float = Field(description="Minimum z-coordinate") - zmax: float = Field(description="Maximum z-coordinate") - - -class FluidContact(BaseModel): - """ - Conditional field - """ - - contact: Literal["owc", "fwl", "goc", "fgl"] = Field( - examples=["owc", "fwl"], - ) - truncated: bool = Field(default=False) - - -class FieldOutline(BaseModel): - """ - Conditional field - """ - - contact: str - - -class FieldRegion(BaseModel): - """ - Conditional field - """ - - id: int = Field( - description="The ID of the region", - ) - - -class Seismic(BaseModel): - """ - Conditional field - """ - - attribute: Optional[str] = Field( - default=None, - examples=["amplitude_timeshifted"], - ) - calculation: Optional[str] = Field( - default=None, - examples=["mean"], - ) - filter_size: Optional[float] = Field( - default=None, - ) - scaling_factor: Optional[float] = Field( - default=None, - ) - stacking_offset: Optional[str] = Field( - default=None, - examples=["0-15"], - ) - zrange: Optional[float] = Field( - default=None, - ) - - class File(BaseModel): """ Block describing the file as the data appear in FMU context @@ -247,25 +178,6 @@ class User(BaseModel): ) -class Layer(BaseModel): - name: str = Field( - description=( - "Name of the data object. If stratigraphic, " - "match the entry in the stratigraphic column" - ), - examples=["VIKING GP. Top"], - ) - offset: float = Field( - default=0, - ) - stratigraphic: bool = Field( - default=False, - description=( - "True if data object represents an entity in the stratigraphic colum" - ), - ) - - class Case(BaseModel): description: Optional[list[str]] = None name: str = Field( @@ -410,21 +322,6 @@ class Masterdata(BaseModel): smda: Smda -class FMUTimeObject(BaseModel): - """ - Time stamp for data object. - """ - - label: Optional[str] = Field( - default=None, - examples=["base", "monitor", "mylabel"], - ) - value: Optional[str] = Field( - default=None, - examples=["2020-10-28T14:28:02"], - ) - - class TracklogEvent(BaseModel): # TODO: Update ex. to inc. timezone # update NaiveDatetime -> AwareDateime @@ -463,223 +360,6 @@ class FmuRealization(FMUDataObj): realization: Realization -class Time(BaseModel): - t0: Optional[FMUTimeObject] = None - t1: Optional[FMUTimeObject] = None - - -class Content(BaseModel): - content: Literal[ - "depth", - "time", - "thickness", - "property", - "seismic", - "fluid_contact", - "field_outline", - "field_region", - "regions", - "pinchout", - "subcrop", - "fault_lines", - "velocity", - "volumes", - "volumetrics", - "inplace_volumes", - "khproduct", - "timeseries", - "wellpicks", - "parameters", - "relperm", - "rft", - "pvt", - "lift_curves", - "transmissibilities", - ] = Field( - description="The contents of this data object", - examples=["depth"], - ) - - alias: Optional[list[str]] = Field(default=None) - base: Optional[Layer] = None - - # Only valid for cooridate based meta. - bbox: Optional[BoundingBox] = Field(default=None) - - description: Optional[list[str]] = Field( - default=None, - ) - format: str = Field( - examples=["irap_binary"], - ) - - grid_model: Optional[GridModel] = Field(default=None) - is_observation: bool = Field( - title="Is observation flag", - ) - is_prediction: bool = Field( - title="Is prediction flag", - ) - layout: Optional[str] = Field( - default=None, - examples=["regular"], - ) - name: str = Field( - description=( - "Name of the data object. If stratigraphic, " - "match the entry in the stratigraphic column" - ), - examples=["VIKING GP. Top"], - ) - offset: float = Field( - default=0.0, - ) - # spec: Optional[TableSpec | CPGridSpec, ...] = None - stratigraphic_alias: Optional[list[str]] = Field(default=None) - stratigraphic: bool = Field( - description=( - "True if data object represents an entity in the stratigraphic column" - ), - ) - tagname: Optional[str] = Field( - default=None, - description="A semi-human readable tag for internal usage and uniqueness", - examples=["ds_extract_geogrid", "ds_post_strucmod"], - ) - time: Optional[Time] = Field(default=None) - top: Optional[Layer] = None - - undef_is_zero: Optional[bool] = Field( - default=None, - description="Flag if undefined values are to be interpreted as zero", - ) - unit: str = Field( - default="", - examples=["m"], - ) - vertical_domain: Optional[Literal["depth", "time"]] = Field( - default=None, - examples=["depth"], - ) - - -class DepthContent(Content): - content: Literal["depth"] - depth_reference: Literal["msl", "sb", "rkb"] - - -class FaultLinesContent(Content): - content: Literal["fault_lines"] - - -class Field_regionContent(Content): - content: Literal["field_region"] - - -class FieldOutlineContent(Content): - content: Literal["field_outline"] - field_outline: FieldOutline = Field( - description="Conditional field", - ) - - -class FieldRegionContent(Content): - content: Literal["field_region"] - field_region: FieldRegion = Field( - description="Conditional field", - ) - - -class FluidContactContent(Content): - content: Literal["fluid_contact"] - fluid_contact: FluidContact = Field( - description="Conditional field", - ) - - -class InplaceVolumesContent(Content): - content: Literal["inplace_volumes"] - - -class KPProductContent(Content): - content: Literal["khproduct"] - - -class LiftCurvesContent(Content): - content: Literal["lift_curves"] - - -class ParametersContent(Content): - content: Literal["parameters"] - - -class PinchoutContent(Content): - content: Literal["pinchout"] - - -class PropertyContent(Content): - content: Literal["property"] - - -class PTVContent(Content): - content: Literal["pvt"] - - -class RegionsContent(Content): - content: Literal["regions"] - - -class RelpermContent(Content): - content: Literal["relperm"] - - -class RFTContent(Content): - content: Literal["rft"] - - -class SeismicContent(Content): # TheDatablock w/seismic - content: Literal["seismic"] - seismic: Seismic = Field( - description="Conditional field", - ) - - -class SubcropContent(Content): - content: Literal["subcrop"] - - -class ThicknessContent(Content): - content: Literal["thickness"] - - -class TimeContent(Content): - content: Literal["time"] - - -class TimeSeriesContent(Content): - content: Literal["timeseries"] - - -class TransmissibilitiesContent(Content): - content: Literal["transmissibilities"] - - -class VelocityContent(Content): - content: Literal["velocity"] - - -class VolumesContent(Content): - content: Literal["volumes"] - - -class VolumetricsContent(Content): - content: Literal["volumetrics"] - - -class WellPicksContent(Content): - content: Literal["wellpicks"] - - class ClassMeta(BaseModel): class_: Literal[ "case", @@ -727,23 +407,12 @@ class FMUDataClassMeta(ClassMeta): ] fmu: Union[FmuAggregation, FmuRealization] # Hmmmmmmm.....? access: SsdlAccess - data: Union[ - FieldOutlineContent, - SeismicContent, - FieldRegionContent, - ] + data: content.AnyContent file: File # Case will not have a file obj. - masterdata: Masterdata - tracklog: list[TracklogEvent] - - source: Literal["fmu"] = Field( - description="Data source (FMU)", - ) - version: Literal["0.8.0"] = Field( - description="FMU results metadata version", - ) + source: Literal["fmu"] = Field(description="Data source (FMU)") + version: Literal["0.8.0"] = Field(description="FMU results metadata version") class Root( @@ -807,7 +476,3 @@ def dump() -> dict: ), ) ) - - -if __name__ == "__main__": - print(json.dumps(dump(), indent=2)) diff --git a/src/fmu/dataio/models/meta3.py b/src/fmu/dataio/models/meta3.py deleted file mode 100644 index 1c24f1aa4..000000000 --- a/src/fmu/dataio/models/meta3.py +++ /dev/null @@ -1,49 +0,0 @@ -from __future__ import annotations - -from datetime import datetime -from typing import List, Literal, Union - -from pydantic import BaseModel, Field, RootModel - - -class User(BaseModel): - id: str = Field(description="User id", examples=["peesv", "jlov"]) - - -class Description(RootModel[List[str]]): - ... - - -class FMUTime(BaseModel): - value: datetime - label: str = Field(examples=["base", "monitor", "mylabel"]) - - -class Source(RootModel[Literal["fmu"]]): - ... - - -class Version(RootModel[Literal["0.9.0"]]): - ... - - -class TrackLogEvent(BaseModel): - datetime: datetime - user: User - event: str = Field(examples=["created", "updated"]) - - -class TrackLog(RootModel[List[TrackLogEvent]]): - ... - - -class CaseObj(BaseModel): - ... - - -class DataObj(BaseModel): - ... - - -class Meta(RootModel[Union[CaseObj, DataObj]]): - ... From abd5fda073f96d637e5c9668d6eac00ea23cacec Mon Sep 17 00:00:00 2001 From: JB Lovland Date: Mon, 22 Jan 2024 21:27:11 +0100 Subject: [PATCH 12/25] Cleanup --- Dockerfile | 20 +- boom.py | 5 +- ex.py | 11 +- .../fmuconfig/output/global_variables.yml | 2 +- foo.py | 80 - parameters.py | 38 - pyproject.toml | 2 +- schema/app/install-schema-files.sh | 36 + .../definitions/0.9.0/schema/fmu_results.json | 1418 ----------------- schema/docs/src/public/fmu_results_080.json | 23 +- src/fmu/dataio/models/meta/__init__.py | 2 +- src/fmu/dataio/models/meta/content.py | 35 +- .../dataio/models/meta/{meta.py => model.py} | 0 tests/test_schema/test_schema_logic.py | 16 +- 14 files changed, 110 insertions(+), 1578 deletions(-) delete mode 100644 foo.py delete mode 100644 parameters.py create mode 100644 schema/app/install-schema-files.sh delete mode 100644 schema/definitions/0.9.0/schema/fmu_results.json rename src/fmu/dataio/models/meta/{meta.py => model.py} (100%) diff --git a/Dockerfile b/Dockerfile index dbf18710f..7f957f36e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,27 @@ FROM nginxinc/nginx-unprivileged:alpine +ARG uriprefix +ENV URIPREFIX=${uriprefix} + WORKDIR /app USER 0 -RUN rm /etc/nginx/conf.d/default.conf -COPY schema/app/fmu-schemas.conf /etc/nginx/conf.d -COPY schema/definitions schemas/ +RUN apk update +RUN apk upgrade +RUN apk add jq + +COPY schema/definitions definitions/ RUN chown -R 101:101 . +COPY schema/app/install-schema-files.sh /docker-entrypoint.d/50-install-schema-files.sh +RUN chmod +x /docker-entrypoint.d/50-install-schema-files.sh + +COPY schema/app/fmu-schemas.conf /etc/nginx/conf.d + +RUN rm /etc/nginx/conf.d/default.conf + EXPOSE 8080 USER 101 CMD ["nginx", "-g", "daemon off;"] + +# CMD "/bin/sh" diff --git a/boom.py b/boom.py index f5f32ba02..e8f6acebb 100644 --- a/boom.py +++ b/boom.py @@ -3,11 +3,10 @@ from contextlib import suppress from random import sample +from fmu.dataio.models.meta.model import Root from fmu.sumo.explorer import Explorer from tqdm import tqdm -from src.fmu.dataio.models.meta2 import Meta - def lazy_sampler(x, lenx, k=100): if lenx <= 0: @@ -45,7 +44,7 @@ def gen(): root_classes = set() for m in tqdm(gen(), ascii=True, position=1): try: - root_classes.add(Meta.model_validate(m).class_) + root_classes.add(Root.model_validate(m)) except Exception: from pprint import pp diff --git a/ex.py b/ex.py index f05beb97a..862c5d8d7 100644 --- a/ex.py +++ b/ex.py @@ -1,9 +1,7 @@ import sys -from pprint import pp -from fmu.dataio.models.meta2 import Meta +from fmu.dataio.models.meta.model import Root from orjson import dumps -from pydantic import ValidationError from yaml import safe_load @@ -14,9 +12,4 @@ def read(file): for file in (f.strip() for f in sys.stdin.readlines()): print(file) - try: - Meta.model_validate_json(dumps(safe_load(read(file)))) - except ValidationError as e: - print(str(e)) - pp(safe_load(read(file))) - print("-" * 100) + Root.model_validate_json(dumps(safe_load(read(file)))) diff --git a/examples/s/d/nn/xcase/realization-1/iter-0/fmuconfig/output/global_variables.yml b/examples/s/d/nn/xcase/realization-1/iter-0/fmuconfig/output/global_variables.yml index 5aecd2e6b..91e801c69 100644 --- a/examples/s/d/nn/xcase/realization-1/iter-0/fmuconfig/output/global_variables.yml +++ b/examples/s/d/nn/xcase/realization-1/iter-0/fmuconfig/output/global_variables.yml @@ -17,7 +17,7 @@ masterdata: uuid: ad214d85-dac7-19da-e053-c918a4889309 stratigraphic_column: identifier: DROGON_2020 - uuid: ad214d85-dac7-19da-e053-c918a4889308 + uuid: some-unique-id-to-be-provided-by-smda access: asset: name: Drogon diff --git a/foo.py b/foo.py deleted file mode 100644 index 314a09599..000000000 --- a/foo.py +++ /dev/null @@ -1,80 +0,0 @@ -from __future__ import annotations - -from pprint import pp -from typing import Literal -from uuid import UUID - -from pydantic import BaseModel, Field -from yaml import safe_load - - -class SMDAAttribute(BaseModel): - short_identifier: str | None = None - identifier: str | None = None - uuid: UUID = Field(title="...", description="....") - - -class SMDA(BaseModel): - country: list[SMDAAttribute] - discovery: list[SMDAAttribute] - field: list[SMDAAttribute] - coordinate_system: SMDAAttribute - stratigraphic_column: SMDAAttribute - - -class Masterdata(BaseModel): - smda: SMDA - - -class Asset(BaseModel): - name: str - - -class SSDL(BaseModel): - access_level: Literal["internal", "external"] - rep_include: bool - - -class Access(BaseModel): - asset: Asset - ssdl: SSDL - - -class Model(BaseModel): - name: str - revision: str = Field(pattern=r"^\d+\.\d+.\d+\.(dev|prod)$") - - -class StratigraphyAttribute(BaseModel): - stratigraphic: bool - name: str - alias: list[str] = [] - stratigraphic_alias: list[str] = [] - - -class RMS(BaseModel): - horizons: dict[str, list[str]] - zones: dict[str, list[str]] - - -class Root(BaseModel): - masterdata: Masterdata - access: Access - model: Model - stratigraphy: dict[str, StratigraphyAttribute] - global_: dict[str, str | float | int] = Field(alias="global") - rms: RMS - - -with open( - "examples/s/d/nn/xcase/realization-1/iter-0/fmuconfig/output/global_variables.yml" -) as f: - pp(safe_load(f)) - -print("-" * 100) - -with open( - "examples/s/d/nn/xcase/realization-1/iter-0/fmuconfig/output/global_variables.yml" -) as f: - m = Root.model_validate(safe_load(f)) - print(m.rms) diff --git a/parameters.py b/parameters.py deleted file mode 100644 index c1abb111c..000000000 --- a/parameters.py +++ /dev/null @@ -1,38 +0,0 @@ -from __future__ import annotations - -from typing import Dict, Union - -from pydantic import RootModel - - -class Parameters(RootModel[Dict[str, Union[int, float, str, "Parameters"]]]): - ... - - # root: dict[str, int | float | str | Parameters] - - -# class Parameters(BaseModel): -# parameters: Union[int, float, str, "Parameters"] - -# print(Parameters.model_rebuild()) -# print(Parameters.model_json_schema()) -print( - Parameters.model_validate({"hello": "hello", "this": {"this": "that", "more": 1}}) -) -print( - Parameters.model_validate( - { - "SENSNAME": "faultseal", - "SENSCASE": "low", - "RMS_SEED": 1006, - "INIT_FILES": { - "PERM_FLUVCHAN_E1_NORM": 0.748433, - "PERM_FLUVCHAN_E21_NORM": 0.782068, - }, - "KVKH_CHANNEL": 0.6, - "KVKH_US": 0.6, - "FAULT_SEAL_SCALING": 0.1, - "FWL_CENTRAL": 1677, - } - ) -) diff --git a/pyproject.toml b/pyproject.toml index f00ef65df..febe57817 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ dependencies = [ "numpy", "pandas", "pyarrow", - "pydantic>=2.0.0", + "pydantic>=2.5.0", "PyYAML", "xtgeo>=2.16", ] diff --git a/schema/app/install-schema-files.sh b/schema/app/install-schema-files.sh new file mode 100644 index 000000000..4ca338d32 --- /dev/null +++ b/schema/app/install-schema-files.sh @@ -0,0 +1,36 @@ +#! /bin/sh + +die () { + exitcode="$1"; shift + echo $* + exit $exitcode +} + +if [ -z "$URIPREFIX" ]; then + test \! -z "$RADIX_PUBLIC_DOMAIN_NAME" || die 1 "Unable to build PREFIX." + URIPREFIX="https://${RADIX_PUBLIC_DOMAIN_NAME}" +fi + +echo "URIPREFIX=$URIPREFIX" + +test -d ./definitions || die 1 "Definitions not found." + +mkdir schemas || die 1 "Unable to create directory 'schemas'." + +find ./definitions -type d -name schema -print | \ + while read dir; do + pdir=`dirname "$dir"` + sdir=schemas/`basename "$pdir"` + mkdir "$sdir" + find "$dir" -maxdepth 1 -name '*.json' -print | \ + while read file; do + #echo "file: $file" + ffile=`basename "$file"` + id=`echo "$URIPREFIX"/"$sdir"/"$ffile" | sed -e 's/ /%20/g'` + #echo "id: $id" + #echo "creating $sdir/$ffile" + cat "$file" | jq '.["$id"]='\""$id"\" > "$sdir"/"$ffile" + done + done + + diff --git a/schema/definitions/0.9.0/schema/fmu_results.json b/schema/definitions/0.9.0/schema/fmu_results.json deleted file mode 100644 index 36569bd02..000000000 --- a/schema/definitions/0.9.0/schema/fmu_results.json +++ /dev/null @@ -1,1418 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "fmu_results.json", - "$contractual": [ - "class", - "source", - "version", - "tracklog", - "data.format", - "data.name", - "data.stratigraphic", - "data.alias", - "data.stratigraphic_alias", - "data.offset", - "data.content", - "data.tagname", - "data.vertical_domain", - "data.grid_model", - "data.bbox", - "data.time", - "data.is_prediction", - "data.is_observation", - "data.seismic.attribute", - "data.spec.columns", - "access", - "masterdata", - "fmu.model", - "fmu.workflow", - "fmu.case", - "fmu.iteration.name", - "fmu.iteration.uuid", - "fmu.realization.name", - "fmu.realization.id", - "fmu.realization.uuid", - "fmu.aggregation.operation", - "fmu.aggregation.realization_ids", - "fmu.context.stage", - "file.relative_path", - "file.checksum_md5", - "file.size_bytes" - ], - "$defs": { - "properties": { - "generic": { - "$comment": "GENERIC DEFINITIONS", - "datetime": { - "type": "string", - "examples": [ - "2020-10-28T14:28:02" - ] - }, - "uuid": { - "type": "string", - "pattern": "^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", - "examples": [ - "ad214d85-8a1d-19da-e053-c918a4889309" - ] - }, - "user": { - "type": "object", - "required": [ - "id" - ], - "properties": { - "id": { - "title": "User ID", - "type": "string", - "examples": [ - "peesv" - ] - } - } - }, - "_description": { - "$comment": "Underscore to keep JSON Schema validator functioning", - "type": "array", - "items": { - "type": "string", - "examples": [ - "Some description" - ] - } - } - }, - "fmu_time": { - "title": "FMU time object", - "description": "Time stamp for data object.", - "type": [ - "object", - "null" - ], - "properties": { - "value": { - "$ref": "#/$defs/properties/generic/datetime" - }, - "label": { - "type": "string", - "examples": [ - "base", - "monitor", - "mylabel" - ] - } - } - }, - "class": { - "title": "Metadata class", - "type": "string", - "examples": [ - "surface", - "table", - "points" - ], - "enum": [ - "case", - "surface", - "table", - "cpgrid", - "cpgrid_property", - "polygons", - "cube", - "well", - "points", - "dictionary" - ] - }, - "source": { - "$comment": "Fixed field, only valid value is fmu.", - "description": "Data source (FMU)", - "type": "string", - "const": "fmu" - }, - "version": { - "$comment": "Must always be on root level.", - "title": "FMU results metadata version", - "type": "string", - "$comment": "This is 0.9.0, so require version to be == 0.9.0", - "enum": [ - "0.9.0" - ], - "example": "1.2.3" - }, - "tracklog_event": { - "type": "object", - "properties": { - "datetime": { - "$ref": "#/$defs/properties/generic/datetime" - }, - "user": { - "$ref": "#/$defs/properties/generic/user" - }, - "event": { - "type": "string", - "examples": [ - "created", - "updated" - ] - } - } - }, - "tracklog": { - "type": "array", - "items": { - "$ref": "#/$defs/properties/tracklog_event" - } - }, - "data": { - "type": "object", - "title": "The data block", - "required": [ - "content", - "name", - "format", - "stratigraphic", - "is_prediction", - "is_observation" - ], - "dependencies": { - "top": { - "required": [ - "base" - ] - }, - "base": { - "required": [ - "top" - ] - } - }, - "properties": { - "name": { - "type": "string", - "description": "Name of the data object. If stratigraphic, match the entry in the stratigraphic column", - "examples": [ - "VIKING GP. Top" - ] - }, - "stratigraphic": { - "$comment": "Determines of validation against strat column should happen or not", - "type": "boolean", - "description": "True if data object represents an entity in the stratigraphic column", - "examples": [ - true - ] - }, - "alias": { - "type": "array", - "items": { - "$ref": "#/$defs/properties/data/properties/name" - } - }, - "stratigraphic_alias": { - "type": "array", - "items": { - "$ref": "#/$defs/properties/data/properties/name" - } - }, - "offset": { - "$comment": "To be used if data represents an offset from the specified name", - "$comment": "e.g. a surface representing an offset from a known horizon", - "type": "number", - "example": 11.2 - }, - "top": { - "$comment": "To be used if data object is an interval", - "properties": { - "name": { - "$ref": "#/$defs/properties/data/properties/name" - }, - "stratigraphic": { - "$ref": "#/$defs/properties/data/properties/stratigraphic" - }, - "offset": { - "$ref": "#/$defs/properties/data/properties/offset" - } - } - }, - "base": { - "$comment": "To be used if data object is an interval", - "properties": { - "name": { - "$ref": "#/$defs/properties/data/properties/name" - }, - "stratigraphic": { - "$ref": "#/$defs/properties/data/properties/stratigraphic" - }, - "offset": { - "$ref": "#/$defs/properties/data/properties/offset" - } - } - }, - "content": { - "type": "string", - "description": "The contents of this data object", - "examples": [ - "depth" - ] - }, - "tagname": { - "type": "string", - "description": "A semi-human readable tag for internal usage and uniqueness", - "examples": [ - "ds_extract_geogrid", - "ds_post_strucmod" - ] - }, - "properties": { - "$comment": "This is not the JSON Schema properties, this is FMU properties", - "description": "data.properties is an array of property objects, representing the properties present in the data object.", - "type": "array", - "items": { - "type": "object", - "properties": { - "name": { - "type": "string", - "examples": [ - "MyPropertyName" - ] - }, - "attribute": { - "type": "string", - "examples": [ - "MyAttributeName" - ] - }, - "is_discrete": { - "type": "boolean", - "examples": [ - true, - false - ] - } - } - } - }, - "format": { - "type": "string", - "examples": [ - "irap_binary" - ] - }, - "layout": { - "type": "string", - "examples": [ - "regular" - ] - }, - "unit": { - "type": [ - "string", - "null" - ], - "examples": [ - "m" - ] - }, - "vertical_domain": { - "type": "string", - "enum": [ - "depth", - "time" - ], - "examples": [ - "depth" - ] - }, - "depth_reference": { - "type": [ - "string", - "null" - ], - "examples": [ - "msl" - ], - "enum": [ - "msl", - "sb", - "rkb", - null - ] - }, - "undef_is_zero": { - "description": "Flag if undefined values are to be interpreted as zero", - "type": "boolean", - "examples": [ - "True" - ] - }, - "grid_model": { - "type": "object", - "properties": { - "name": { - "type": [ - "string", - "null" - ], - "examples": [ - "MyGrid" - ] - } - } - }, - "spec": { - "type": "object", - "properties": { - "ncol": { - "$comment": "https://json-schema.org/understanding-json-schema/reference/numeric.html", - "type": "integer", - "examples": [ - 281 - ] - }, - "nrow": { - "type": "integer", - "examples": [ - 441 - ] - }, - "nlay": { - "type": "integer", - "examples": [ - 333 - ] - }, - "xori": { - "type": "number", - "examples": [ - 461499.9997558594 - ] - }, - "yori": { - "type": "number", - "examples": [ - 5926500.224123242 - ] - }, - "xinc": { - "type": "number", - "examples": [ - 25.0 - ] - }, - "yflip": { - "type": "integer", - "$comment": "Orientation indicator of coordinate system", - "enum": [ - -1, - 1 - ], - "examples": [ - -1, - 1 - ] - }, - "rotation": { - "type": "number", - "examples": [ - "30.00000000231" - ] - }, - "undef": { - "type": "number", - "$comment": "How is the undefined value represented?", - "examples": [ - 1.0e+33 - ] - }, - "npolys": { - "description": "The number of individual polygons in the data object", - "type": "integer", - "examples": [ - 1 - ] - }, - "size": { - "description": "Size of data object.", - "type": "integer", - "examples": [ - 1, - 9999 - ] - }, - "columns": { - "description": "List of columns present in a table.", - "type": "array", - "items": { - "type": "string", - "examples": [ - "FOPT", - "STOIIP_OIL", - "COL_1" - ] - } - } - } - }, - "bbox": { - "type": "object", - "required": [ - "xmin", - "xmax", - "ymin", - "ymax" - ], - "properties": { - "xmin": { - "type": "number", - "examples": [ - 456012.5003497944 - ] - }, - "xmax": { - "type": "number", - "examples": [ - 467540.52762886323 - ] - }, - "ymin": { - "type": "number", - "examples": [ - 5926499.999511719 - ] - }, - "ymax": { - "type": "number", - "examples": [ - 5939492.128326312 - ] - }, - "zmin": { - "type": [ - "number", - "null" - ], - "examples": [ - 1244.039, - null - ] - }, - "zmax": { - "type": [ - "number", - "null" - ], - "examples": [ - 2302.683, - null - ] - } - } - }, - "time": { - "type": "object", - "properties": { - "t0": { - "$ref": "#/$defs/properties/fmu_time" - }, - "t1": { - "$ref": "#/$defs/properties/fmu_time" - } - } - }, - "is_prediction": { - "title": "Is prediction flag", - "$comment": "For flagging predictions, and separating them from non-predictions", - "type": "boolean", - "examples": [ - true - ] - }, - "is_observation": { - "title": "Is observation flag", - "$comment": "For flagging observations, and separating them from non-observations", - "type": "boolean", - "examples": [ - true - ] - }, - "fluid_contact": { - "type": "object", - "description": "Conditional field", - "properties": { - "contact": { - "type": "string", - "enum": [ - "owc", - "fwl", - "goc", - "fgl" - ], - "$comment": "Not sure if wise to enum this", - "examples": [ - "owc", - "fwl" - ] - }, - "truncated": { - "type": "boolean", - "$comment": "Is this contact truncated to the stratigraphy?", - "examples": [ - true - ] - } - } - }, - "field_outline": { - "description": "Conditional field", - "type": "object", - "required": [ - "contact" - ], - "properties": { - "contact": { - "type": "string" - } - } - }, - "field_region": { - "description": "Conditional field", - "type": "object", - "required": [ - "id" - ], - "properties": { - "id": { - "description": "The ID of the region", - "type": "integer" - } - } - }, - "description": { - "$ref": "#/$defs/properties/generic/_description" - }, - "seismic": { - "type": "object", - "description": "Conditional field", - "properties": { - "attribute": { - "type": "string", - "examples": [ - "amplitude_timeshifted" - ] - }, - "calculation": { - "type": "string", - "examples": [ - "mean" - ] - }, - "zrange": { - "type": "number", - "examples": [ - 12.0 - ] - }, - "filter_size": { - "type": "number", - "examples": [ - 1.0 - ] - }, - "scaling_factor": { - "type": "number", - "examples": [ - 1.0 - ] - }, - "stacking_offset": { - "type": "string", - "examples": [ - "0-15" - ] - } - } - } - } - }, - "display": { - "type": "object", - "properties": { - "name": { - "type": "string", - "examples": [ - "Top Volantis" - ] - }, - "subtitle": { - "type": "string", - "examples": [ - "Some subtitle" - ] - }, - "line": { - "type": "object", - "properties": { - "show": { - "type": "boolean", - "examples": [ - true - ] - }, - "color": { - "type": "string", - "examples": [ - "black", - "#000000" - ] - } - } - }, - "points": { - "type": "object", - "properties": { - "show": { - "type": "boolean", - "examples": [ - true - ] - }, - "color": { - "type": "string", - "examples": [ - "black", - "#000000" - ] - } - } - }, - "contours": { - "type": "object", - "properties": { - "show": { - "type": "boolean", - "examples": [ - true - ] - }, - "color": { - "description": "The color of the contour lines", - "type": "string", - "examples": [ - "black", - "#000000" - ] - }, - "increment": { - "description": "The contour increment in same values as the data", - "type": "number", - "examples": [ - 20.0 - ] - } - } - }, - "fill": { - "type": "object", - "properties": { - "show": { - "type": "boolean", - "examples": [ - true - ] - }, - "color": { - "type": "string", - "examples": [ - "black", - "#000000" - ] - }, - "colormap": { - "$comment": "colormap can be given alongside fill.color. Clients must choose.", - "description": "named reference to a colormap", - "type": "string", - "examples": [ - "gist_earth" - ] - }, - "display_min": { - "description": "The low-side boundary to use for fill color", - "type": "number", - "examples": [ - 1000.0 - ] - }, - "display_max": { - "description": "The high-side boundary to use for fill color", - "type": "number", - "examples": [ - 1600.0 - ] - } - } - } - } - }, - "access": { - "type": "object", - "properties": { - "asset": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "examples": [ - "Drogon" - ] - } - } - }, - "ssdl": { - "type": "object", - "$comment": "Required for data objects", - "required": [ - "access_level", - "rep_include" - ], - "properties": { - "access_level": { - "$comment": "'asset' is legacy, but will be allowed in a transition", - "type": "string", - "enum": [ - "internal", - "restricted", - "asset" - ] - }, - "rep_include": { - "$comment": "Should REP display this data?", - "type": "boolean", - "examples": [ - true, - false - ] - } - } - }, - "classification": { - "type": "string", - "enum": [ - "internal", - "restricted" - ], - "examples": [ - "internal", - "restricted" - ] - } - } - }, - "masterdata": { - "$comment": "Block holds references to master data", - "type": "object", - "required": [ - "smda" - ], - "properties": { - "smda": { - "type": "object", - "required": [ - "country", - "discovery", - "field", - "coordinate_system", - "stratigraphic_column" - ], - "properties": { - "country": { - "$comment": "Array of SMDA-valid countries. First entry is primary.", - "type": "array", - "items": { - "type": "object", - "required": [ - "identifier", - "uuid" - ], - "properties": { - "identifier": { - "type": "string", - "examples": [ - "Norway" - ] - }, - "uuid": { - "$ref": "#/$defs/properties/generic/uuid" - } - } - } - }, - "discovery": { - "$comment": "Array of SMDA-valid discoveries. First entry is primary.", - "type": "array", - "items": { - "type": "object", - "required": [ - "short_identifier", - "uuid" - ], - "properties": { - "short_identifier": { - "type": "string", - "examples": [ - "SomeDiscovery" - ] - }, - "uuid": { - "$ref": "#/$defs/properties/generic/uuid" - } - } - } - }, - "field": { - "$comment": "Array of SMDA-valid (oil & gas) fields. First entry is primary.", - "type": "array", - "items": { - "type": "object", - "required": [ - "identifier", - "uuid" - ], - "properties": { - "identifier": { - "type": "string", - "examples": [ - "OseFax" - ] - }, - "uuid": { - "$ref": "#/$defs/properties/generic/uuid" - } - } - } - }, - "coordinate_system": { - "$comment": "SMDA-valid coordinate system.", - "type": "object", - "required": [ - "identifier", - "uuid" - ], - "properties": { - "identifier": { - "type": "string", - "examples": [ - "ST_WGS84_UTM37N_P32637" - ] - }, - "uuid": { - "$ref": "#/$defs/properties/generic/uuid" - } - } - }, - "stratigraphic_column": { - "$comment": "SMDA-valid stratigraphic column.", - "type": "object", - "required": [ - "identifier", - "uuid" - ], - "properties": { - "identifier": { - "type": "string", - "examples": [ - "DROGON_2020" - ] - }, - "uuid": { - "$ref": "#/$defs/properties/generic/uuid" - } - } - } - } - } - } - }, - "fmu": { - "model": { - "type": "object", - "properties": { - "name": { - "type": "string", - "examples": [ - "Drogon" - ] - }, - "revision": { - "type": "string", - "examples": [ - "21.0.0.dev" - ] - }, - "description": { - "description": "This is a free text description of the model setup", - "$ref": "#/$defs/properties/generic/_description" - } - } - }, - "workflow": { - "type": "object", - "properties": { - "reference": { - "type": "string", - "description": "Reference to the part of the FMU workflow that produced this" - } - } - }, - "case": { - "type": "object", - "required": [ - "name", - "uuid", - "user" - ], - "properties": { - "name": { - "type": "string", - "description": "The case name", - "examples": [ - "MyCaseName" - ] - }, - "uuid": { - "$ref": "#/$defs/properties/generic/uuid" - }, - "user": { - "description": "The user name used in ERT", - "$ref": "#/$defs/properties/generic/user" - }, - "description": { - "$ref": "#/$defs/properties/generic/_description" - } - } - }, - "iteration": { - "type": "object", - "required": [ - "name", - "uuid" - ], - "properties": { - "name": { - "description": "The convential name of this iteration, e.g. iter-0 or pred", - "type": "string", - "examples": [ - "iter-0" - ] - }, - "id": { - "description": "The internal identification of this iteration, e.g. the iteration number", - "type": "integer", - "examples": [ - 0 - ] - }, - "uuid": { - "$ref": "#/$defs/properties/generic/uuid" - }, - "restart_from": { - "description": "A uuid reference to another iteration that this iteration was restarted from", - "$ref": "#/$defs/properties/generic/uuid" - } - } - }, - "realization": { - "$comment": "Some schema repetition between realization and aggregation", - "type": "object", - "required": [ - "name", - "id", - "uuid" - ], - "properties": { - "name": { - "description": "The convential name of this iteration, e.g. iter-0 or pred", - "type": "string", - "examples": [ - "iter-0" - ] - }, - "id": { - "type": "integer", - "description": "The unique number of this realization as used in FMU", - "examples": [ - 33 - ] - }, - "uuid": { - "$ref": "#/$defs/properties/generic/uuid" - }, - "parameters": { - "type": "object", - "description": "Parameters for this realization", - "items": { - "type": "object" - } - }, - "jobs": { - "type": "object", - "description": "Content directly taken from the ERT jobs.json file for this realization" - } - } - }, - "aggregation": { - "$comment": "Some schema repetition between realization and aggregation", - "type": "object", - "required": [ - "operation", - "realization_ids" - ], - "properties": { - "operation": { - "type": "string", - "description": "The aggregation performed" - }, - "realization_ids": { - "type": "array", - "description": "Array of realization ids included in this aggregation", - "items": { - "type": "integer", - "examples": [ - 0, - 1, - 2, - 3 - ] - } - }, - "parameters": { - "type": "object", - "description": "Parameters for this realization", - "items": { - "type": "object" - } - }, - "id": { - "type": "string", - "description": "The ID of this aggregation", - "$comment": "Used for tying together representations of the same entity", - "$comment": "Example: Statistics for various time steps of the same surface", - "$comment": "Not used in Drogon but widely used on e.g. Johan Sverdrup", - "$comment": "Used for enabling common display and joint visual settings", - "$comment": "A unique uuid is currently used, and included as example but any ID should be sufficient", - "examples": [ - "15ce3b84-766f-4c93-9050-b154861f9100" - ] - } - } - }, - "context": { - "description": "The internal FMU context in which this data object was produced", - "type": "object", - "required": [ - "stage" - ], - "properties": { - "stage": { - "type": "string", - "examples": [ - "case", - "iteration", - "realization" - ] - } - } - } - }, - "file": { - "description": "Block describing the file as the data appear in FMU context", - "$comment": "While the file may seize to exist, the concept is still useful", - "$comment": "A lot of logic exists on top of the file name now, legacy but not likely to disappear soon", - "type": "object", - "required": [ - "relative_path", - "checksum_md5" - ], - "properties": { - "relative_path": { - "type": "string", - "description": "The file path relative to RUNPATH", - "examples": [ - "share/results/maps/volantis_gp_base--depth.gri" - ] - }, - "absolute_path": { - "type": "string", - "description": "The absolute file path", - "examples": [ - "/abs/path/share/results/maps/volantis_gp_base--depth.gri" - ] - }, - "checksum_md5": { - "description": "md5 checksum of the file or bytestring", - "type": "string", - "examples": [ - "kjhsdfvsdlfk23knerknvk23" - ] - }, - "size_bytes": { - "description": "Size of file object in bytes", - "type": "integer", - "examples": [ - 123 - ] - } - } - } - }, - "subschemas": { - "case": { - "required": [ - "source", - "version", - "class", - "fmu", - "access", - "tracklog", - "masterdata" - ], - "properties": { - "source": { - "$ref": "#/$defs/properties/source" - }, - "version": { - "$ref": "#/$defs/properties/version" - }, - "class": { - "$ref": "#/$defs/properties/class" - }, - "tracklog": { - "$ref": "#/$defs/properties/tracklog" - }, - "data": { - "$ref": "#/$defs/properties/data" - }, - "access": { - "$ref": "#/$defs/properties/access", - "required": [ - "asset" - ] - }, - "masterdata": { - "$ref": "#/$defs/properties/masterdata" - }, - "fmu": { - "description": "The FMU block records properties that are specific to FMU", - "$comment": "This is the fmu block as it appears in data objects", - "type": "object", - "required": [ - "model", - "case" - ], - "properties": { - "model": { - "$ref": "#/$defs/properties/fmu/model" - }, - "case": { - "$ref": "#/$defs/properties/fmu/case" - } - } - } - } - }, - "data_object": { - "properties": { - "fmu": { - "description": "The FMU block records properties that are specific to FMU", - "$comment": "This is the fmu block as it appears in data objects", - "required": [ - "model", - "case" - ], - "properties": { - "model": { - "$ref": "#/$defs/properties/fmu/model" - }, - "case": { - "$ref": "#/$defs/properties/fmu/case" - }, - "iteration": { - "$ref": "#/$defs/properties/fmu/iteration" - }, - "realization": { - "$ref": "#/$defs/properties/fmu/realization" - }, - "workflow": { - "$ref": "#/$defs/properties/fmu/workflow" - }, - "aggregation": { - "$ref": "#/$defs/properties/fmu/aggregation" - } - }, - "$comment": "implementation below allows realization or aggregation or none of them, never both", - "dependencies": { - "aggregation": { - "not": { - "required": [ - "realization" - ] - } - }, - "realization": { - "not": { - "required": [ - "aggregation" - ] - } - } - } - }, - "file": { - "$ref": "#/$defs/properties/file" - }, - "data": { - "$comment": "Conditionals", - "allOf": [ - { - "$ref": "#/$defs/properties/data" - }, - { - "$comment": "When content == field_outline, require data.field_outline", - "if": { - "properties": { - "content": { - "const": "field_outline" - } - } - }, - "then": { - "required": [ - "field_outline" - ] - } - }, - { - "$comment": "When content == field_region, require data.field_region", - "if": { - "properties": { - "content": { - "const": "field_region" - } - } - }, - "then": { - "required": [ - "field_region" - ] - } - }, - { - "$comment": "When content == fluid_contact, require data.fluid_contact", - "if": { - "properties": { - "content": { - "const": "fluid_contact" - } - } - }, - "then": { - "required": [ - "fluid_contact" - ] - } - }, - { - "$comment": "When content == seismic, require data.seismic", - "if": { - "properties": { - "content": { - "const": "seismic" - } - } - }, - "then": { - "required": [ - "seismic" - ] - } - } - ] - }, - "access": { - "$ref": "#/$defs/properties/access", - "required": [ - "asset", - "ssdl" - ] - }, - "source": { - "$ref": "#/$defs/properties/source" - }, - "version": { - "$ref": "#/$defs/properties/version" - }, - "class": { - "$ref": "#/$defs/properties/class" - }, - "tracklog": { - "$ref": "#/$defs/properties/tracklog" - }, - "masterdata": { - "$ref": "#/$defs/properties/masterdata" - } - }, - "if": { - "properties": { - "class": { - "enum": [ - "table", - "surface" - ] - } - } - }, - "then": { - "properties": { - "data": { - "required": [ - "spec" - ] - } - } - }, - "required": [ - "source", - "version", - "class", - "fmu", - "access", - "tracklog", - "masterdata", - "data", - "file" - ] - } - } - }, - "$comment": "=== MAIN SCHEMA ENTRY POINT ===", - "if": { - "properties": { - "class": { - "const": "case" - } - } - }, - "then": { - "title": "Case object", - "description": "Validation of a case object.", - "$ref": "#/$defs/subschemas/case" - }, - "else": { - "title": "Data object", - "description": "Validation of a non-case object (data object).", - "$ref": "#/$defs/subschemas/data_object" - } -} \ No newline at end of file diff --git a/schema/docs/src/public/fmu_results_080.json b/schema/docs/src/public/fmu_results_080.json index 2491378b1..5c137fcc6 100644 --- a/schema/docs/src/public/fmu_results_080.json +++ b/schema/docs/src/public/fmu_results_080.json @@ -129,9 +129,10 @@ "const": "fmu" }, "version": { - "$comment": "Must always be on root level. This is 0.8.0, so require version to be == 0.8.0", + "$comment": "Must always be on root level.", "title": "FMU results metadata version", "type": "string", + "$comment": "This is 0.8.0, so require version to be == 0.8.0", "enum": [ "0.8.0" ], @@ -214,7 +215,8 @@ } }, "offset": { - "$comment": "To be used if data represents an offset from the specified name e.g. a surface representing an offset from a known horizon", + "$comment": "To be used if data represents an offset from the specified name", + "$comment": "e.g. a surface representing an offset from a known horizon", "type": "number", "example": 11.2 }, @@ -1095,7 +1097,11 @@ "id": { "type": "string", "description": "The ID of this aggregation", - "$comment": "Used for tying together representations of the same entity Example: Statistics for various time steps of the same surface Not used in Drogon but widely used on e.g. Johan Sverdrup Used for enabling common display and joint visual settings. A unique uuid is currently used, and included as example but any ID should be sufficient", + "$comment": "Used for tying together representations of the same entity", + "$comment": "Example: Statistics for various time steps of the same surface", + "$comment": "Not used in Drogon but widely used on e.g. Johan Sverdrup", + "$comment": "Used for enabling common display and joint visual settings", + "$comment": "A unique uuid is currently used, and included as example but any ID should be sufficient", "examples": [ "15ce3b84-766f-4c93-9050-b154861f9100" ] @@ -1122,7 +1128,8 @@ }, "file": { "description": "Block describing the file as the data appear in FMU context", - "$comment": "While the file may seize to exist, the concept is still useful. A lot of logic exists on top of the file name now, legacy but not likely to disappear soon", + "$comment": "While the file may seize to exist, the concept is still useful", + "$comment": "A lot of logic exists on top of the file name now, legacy but not likely to disappear soon", "type": "object", "required": [ "relative_path", @@ -1160,7 +1167,10 @@ } } }, - "$comment": "DEFINITIONS END HERE oneOf below reflects different structures depending on type The fmu-block is different for cases and data objects, and the data block is only present for data objects.", + "$comment": "DEFINITIONS END HERE", + "$comment": "oneOf below reflects different structures depending on type", + "$comment": "The fmu-block is different for cases and data objects,", + "$comment": "and the data block is only present for data objects.", "required": [ "source", "version", @@ -1242,7 +1252,7 @@ }, "fmu": { "description": "The FMU block records properties that are specific to FMU", - "$comment": "This is the fmu block as it appears in data objects implementation below allows realization or aggregation or none of them, never both", + "$comment": "This is the fmu block as it appears in data objects", "required": [ "model", "case" @@ -1267,6 +1277,7 @@ "$ref": "#/definitions/fmu/aggregation" } }, + "$comment": "implementation below allows realization or aggregation or none of them, never both", "dependencies": { "aggregation": { "not": { diff --git a/src/fmu/dataio/models/meta/__init__.py b/src/fmu/dataio/models/meta/__init__.py index 1e35c2d3a..e3ebe81f8 100644 --- a/src/fmu/dataio/models/meta/__init__.py +++ b/src/fmu/dataio/models/meta/__init__.py @@ -1,4 +1,4 @@ -from .meta import Root, dump +from .model import Root, dump __all__ = [ "dump", diff --git a/src/fmu/dataio/models/meta/content.py b/src/fmu/dataio/models/meta/content.py index acdf742fa..55f49c63a 100644 --- a/src/fmu/dataio/models/meta/content.py +++ b/src/fmu/dataio/models/meta/content.py @@ -1,8 +1,9 @@ from __future__ import annotations -from typing import Annotated, Literal, Optional, Union +from typing import Literal, Optional, Union from pydantic import BaseModel, Field +from typing_extensions import Annotated class FMUTimeObject(BaseModel): @@ -117,30 +118,30 @@ class BoundingBox(BaseModel): class Content(BaseModel): content: Literal[ "depth", - "time", - "thickness", - "property", - "seismic", - "fluid_contact", + "fault_lines", "field_outline", "field_region", - "regions", - "pinchout", - "subcrop", - "fault_lines", - "velocity", - "volumes", - "volumetrics", + "fluid_contact", "inplace_volumes", "khproduct", - "timeseries", - "wellpicks", + "lift_curves", "parameters", + "pinchout", + "property", + "pvt", + "regions", "relperm", "rft", - "pvt", - "lift_curves", + "seismic", + "subcrop", + "thickness", + "time", + "timeseries", "transmissibilities", + "velocity", + "volumes", + "volumetrics", + "wellpicks", ] = Field( description="The contents of this data object", examples=["depth"], diff --git a/src/fmu/dataio/models/meta/meta.py b/src/fmu/dataio/models/meta/model.py similarity index 100% rename from src/fmu/dataio/models/meta/meta.py rename to src/fmu/dataio/models/meta/model.py diff --git a/tests/test_schema/test_schema_logic.py b/tests/test_schema/test_schema_logic.py index 98a7ea2e3..8625a666b 100644 --- a/tests/test_schema/test_schema_logic.py +++ b/tests/test_schema/test_schema_logic.py @@ -36,7 +36,21 @@ def test_schema_080_validate_examples_as_is(schema_080, metadata_examples): """Confirm that examples are valid against the schema""" for i, (name, metadata) in enumerate(metadata_examples.items()): - jsonschema.validate(instance=metadata, schema=schema_080) + try: + jsonschema.validate(instance=metadata, schema=schema_080) + except jsonschema.exceptions.ValidationError: + logger.error("Failed validating existing example: %s", name) + if i == 0: + logger.error( + "This was the first example attempted." + "Error is most likely int he schema." + ) + else: + logger.error( + "This was not the first example attemted." + "Error is most likely in the example." + ) + raise def test_schema_080_file_block(schema_080, metadata_examples): From 097dfaa7269973650c49c1365a2a8a0ab5df9c2b Mon Sep 17 00:00:00 2001 From: JB Lovland Date: Tue, 23 Jan 2024 15:02:44 +0100 Subject: [PATCH 13/25] Move enums to own file --- src/fmu/dataio/models/meta/content.py | 84 +++++++++----------------- src/fmu/dataio/models/meta/enums.py | 44 ++++++++++++++ src/fmu/dataio/models/meta/model.py | 85 ++++++++++++--------------- 3 files changed, 111 insertions(+), 102 deletions(-) create mode 100644 src/fmu/dataio/models/meta/enums.py diff --git a/src/fmu/dataio/models/meta/content.py b/src/fmu/dataio/models/meta/content.py index 55f49c63a..a9897dc12 100644 --- a/src/fmu/dataio/models/meta/content.py +++ b/src/fmu/dataio/models/meta/content.py @@ -5,6 +5,8 @@ from pydantic import BaseModel, Field from typing_extensions import Annotated +from . import enums + class FMUTimeObject(BaseModel): """ @@ -116,36 +118,7 @@ class BoundingBox(BaseModel): class Content(BaseModel): - content: Literal[ - "depth", - "fault_lines", - "field_outline", - "field_region", - "fluid_contact", - "inplace_volumes", - "khproduct", - "lift_curves", - "parameters", - "pinchout", - "property", - "pvt", - "regions", - "relperm", - "rft", - "seismic", - "subcrop", - "thickness", - "time", - "timeseries", - "transmissibilities", - "velocity", - "volumes", - "volumetrics", - "wellpicks", - ] = Field( - description="The contents of this data object", - examples=["depth"], - ) + content: enums.ContentEnum = Field(description="The contents of this data object") alias: Optional[list[str]] = Field(default=None) base: Optional[Layer] = None @@ -211,122 +184,123 @@ class Content(BaseModel): class DepthContent(Content): - content: Literal["depth"] + content: Literal[enums.ContentEnum.depth] depth_reference: Literal["msl", "sb", "rkb"] class FaultLinesContent(Content): - content: Literal["fault_lines"] + content: Literal[enums.ContentEnum.fault_lines] class FieldOutlineContent(Content): - content: Literal["field_outline"] + content: Literal[enums.ContentEnum.field_outline] field_outline: FieldOutline = Field( description="Conditional field", ) class FieldRegionContent(Content): - content: Literal["field_region"] + content: Literal[enums.ContentEnum.field_region] field_region: FieldRegion = Field( description="Conditional field", ) class FluidContactContent(Content): - content: Literal["fluid_contact"] + content: Literal[enums.ContentEnum.fluid_contact] fluid_contact: FluidContact = Field( description="Conditional field", ) class InplaceVolumesContent(Content): - content: Literal["inplace_volumes"] + content: Literal[enums.ContentEnum.inplace_volumes] class KPProductContent(Content): - content: Literal["khproduct"] + content: Literal[enums.ContentEnum.khproduct] class LiftCurvesContent(Content): - content: Literal["lift_curves"] + content: Literal[enums.ContentEnum.lift_curves] class ParametersContent(Content): - content: Literal["parameters"] + content: Literal[enums.ContentEnum.parameters] class PinchoutContent(Content): - content: Literal["pinchout"] + content: Literal[enums.ContentEnum.pinchout] class PropertyContent(Content): - content: Literal["property"] + content: Literal[enums.ContentEnum.property] class PTVContent(Content): - content: Literal["pvt"] + content: Literal[enums.ContentEnum.pvt] class RegionsContent(Content): - content: Literal["regions"] + content: Literal[enums.ContentEnum.regions] class RelpermContent(Content): - content: Literal["relperm"] + content: Literal[enums.ContentEnum.relperm] class RFTContent(Content): - content: Literal["rft"] + content: Literal[enums.ContentEnum.rft] class SeismicContent(Content): - content: Literal["seismic"] + content: Literal[enums.ContentEnum.seismic] seismic: Seismic = Field( description="Conditional field", ) class SubcropContent(Content): - content: Literal["subcrop"] + content: Literal[enums.ContentEnum.subcrop] class ThicknessContent(Content): - content: Literal["thickness"] + content: Literal[enums.ContentEnum.thickness] class TimeContent(Content): - content: Literal["time"] + content: Literal[enums.ContentEnum.time] class TimeSeriesContent(Content): - content: Literal["timeseries"] + content: Literal[enums.ContentEnum.timeseries] class TransmissibilitiesContent(Content): - content: Literal["transmissibilities"] + content: Literal[enums.ContentEnum.transmissibilities] class VelocityContent(Content): - content: Literal["velocity"] + content: Literal[enums.ContentEnum.velocity] class VolumesContent(Content): - content: Literal["volumes"] + content: Literal[enums.ContentEnum.volumes] class VolumetricsContent(Content): - content: Literal["volumetrics"] + content: Literal[enums.ContentEnum.volumetrics] class WellPicksContent(Content): - content: Literal["wellpicks"] + content: Literal[enums.ContentEnum.wellpicks] AnyContent = Annotated[ Union[ DepthContent, FaultLinesContent, + FieldOutlineContent, FieldRegionContent, InplaceVolumesContent, KPProductContent, diff --git a/src/fmu/dataio/models/meta/enums.py b/src/fmu/dataio/models/meta/enums.py new file mode 100644 index 000000000..28467a383 --- /dev/null +++ b/src/fmu/dataio/models/meta/enums.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +from enum import Enum + + +class ContentEnum(str, Enum): + depth = "depth" + fault_lines = "fault_lines" + field_outline = "field_outline" + field_region = "field_region" + fluid_contact = "fluid_contact" + inplace_volumes = "inplace_volumes" + khproduct = "khproduct" + lift_curves = "lift_curves" + parameters = "parameters" + pinchout = "pinchout" + property = "property" + pvt = "pvt" + regions = "regions" + relperm = "relperm" + rft = "rft" + seismic = "seismic" + subcrop = "subcrop" + thickness = "thickness" + time = "time" + timeseries = "timeseries" + transmissibilities = "transmissibilities" + velocity = "velocity" + volumes = "volumes" + volumetrics = "volumetrics" + wellpicks = "wellpicks" + + +class FMUClassEnum(str, Enum): + case = "case" + surface = "surface" + table = "table" + cpgrid = "cpgrid" + cpgrid_property = "cpgrid_property" + polygons = "polygons" + cube = "cube" + well = "well" + points = "points" + dictionary = "dictionary" diff --git a/src/fmu/dataio/models/meta/model.py b/src/fmu/dataio/models/meta/model.py index 251ad8611..b6d2e7cdc 100644 --- a/src/fmu/dataio/models/meta/model.py +++ b/src/fmu/dataio/models/meta/model.py @@ -6,8 +6,9 @@ from pydantic import BaseModel, Field, NaiveDatetime, RootModel from pydantic.json_schema import GenerateJsonSchema +from typing_extensions import Annotated -from . import content +from . import content, enums class UUID(RootModel[str]): @@ -145,7 +146,7 @@ class File(BaseModel): ) -class Parameters(RootModel[Dict[str, Union[int, float, str, "Parameters"]]]): +class Parameters(RootModel[Dict[str, Union[int, float, str, dict]]]): ... @@ -324,7 +325,7 @@ class Masterdata(BaseModel): class TracklogEvent(BaseModel): # TODO: Update ex. to inc. timezone - # update NaiveDatetime -> AwareDateime + # update NaiveDatetime -> AwareDatetime datetime: NaiveDatetime = Field( examples=["2020-10-28T14:28:02"], ) @@ -344,7 +345,7 @@ class FMUDataObj(FMUCase): workflow: Optional[Workflow] = None -class FmuAggregation(FMUDataObj): +class FMUAggregation(FMUDataObj): """ The FMU block records properties that are specific to FMU """ @@ -352,7 +353,7 @@ class FmuAggregation(FMUDataObj): aggregation: Aggregation -class FmuRealization(FMUDataObj): +class FMURealization(FMUDataObj): """ The FMU block records properties that are specific to FMU """ @@ -360,67 +361,57 @@ class FmuRealization(FMUDataObj): realization: Realization + class ClassMeta(BaseModel): - class_: Literal[ - "case", - "surface", - "table", - "cpgrid", - "cpgrid_property", - "polygons", - "cube", - "well", - "points", - "dictionary", - ] = Field( + class_: enums.FMUClassEnum = Field( alias="class", - examples=["surface", "table", "points"], title="Metadata class", ) + masterdata: Masterdata + tracklog: list[TracklogEvent] + source: Literal["fmu"] = Field(description="Data source (FMU)") + version: Literal["0.8.0"] = Field(title="FMU results metadata version") class FMUCaseClassMeta(ClassMeta): - class_: Literal["case"] + class_: Literal[enums.FMUClassEnum.case] = Field( + alias="class", + title="Metadata class", + ) fmu: FMUCase access: Access - masterdata: Masterdata - tracklog: list[TracklogEvent] - source: Literal["fmu"] = Field( - description="Data source (FMU)", - ) - version: Literal["0.8.0"] = Field( - title="FMU results metadata version", - ) class FMUDataClassMeta(ClassMeta): class_: Literal[ - "surface", - "table", - "cpgrid", - "cpgrid_property", - "polygons", - "cube", - "well", - "points", - "dictionary", - ] - fmu: Union[FmuAggregation, FmuRealization] # Hmmmmmmm.....? + enums.FMUClassEnum.surface, + enums.FMUClassEnum.table, + enums.FMUClassEnum.cpgrid, + enums.FMUClassEnum.cpgrid_property, + enums.FMUClassEnum.polygons, + enums.FMUClassEnum.cube, + enums.FMUClassEnum.well, + enums.FMUClassEnum.points, + enums.FMUClassEnum.dictionary, + ] = Field( + alias="class", + title="Metadata class", + ) + fmu: Union[FMUAggregation, FMURealization] access: SsdlAccess data: content.AnyContent - file: File # Case will not have a file obj. - masterdata: Masterdata - tracklog: list[TracklogEvent] - source: Literal["fmu"] = Field(description="Data source (FMU)") - version: Literal["0.8.0"] = Field(description="FMU results metadata version") + file: File class Root( RootModel[ - Union[ - FMUCaseClassMeta, - FMUDataClassMeta, - ], + Annotated[ + Union[ + FMUCaseClassMeta, + FMUDataClassMeta, + ], + Field(discriminator="class_"), + ] ] ): ... From 77e29fe9c925e93b8bc728c3941955fab41509d1 Mon Sep 17 00:00:00 2001 From: JB Lovland Date: Tue, 23 Jan 2024 16:15:43 +0100 Subject: [PATCH 14/25] All ex. passes --- ex.py | 8 +++++++- src/fmu/dataio/models/meta/model.py | 3 +-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/ex.py b/ex.py index 862c5d8d7..b8954cba7 100644 --- a/ex.py +++ b/ex.py @@ -12,4 +12,10 @@ def read(file): for file in (f.strip() for f in sys.stdin.readlines()): print(file) - Root.model_validate_json(dumps(safe_load(read(file)))) + try: + Root.model_validate_json(dumps(safe_load(read(file)))) + except ValueError: + from pprint import pp + + pp(safe_load(read(file))) + raise diff --git a/src/fmu/dataio/models/meta/model.py b/src/fmu/dataio/models/meta/model.py index b6d2e7cdc..0bd374a36 100644 --- a/src/fmu/dataio/models/meta/model.py +++ b/src/fmu/dataio/models/meta/model.py @@ -361,7 +361,6 @@ class FMURealization(FMUDataObj): realization: Realization - class ClassMeta(BaseModel): class_: enums.FMUClassEnum = Field( alias="class", @@ -397,7 +396,7 @@ class FMUDataClassMeta(ClassMeta): alias="class", title="Metadata class", ) - fmu: Union[FMUAggregation, FMURealization] + fmu: Union[FMUAggregation, FMURealization, FMUCase] access: SsdlAccess data: content.AnyContent file: File From df7aacc37791b09cb515df5e0db14549c1d90ddc Mon Sep 17 00:00:00 2001 From: JB Lovland Date: Tue, 23 Jan 2024 16:57:34 +0100 Subject: [PATCH 15/25] Added callable discriminator --- boom.py | 2 + ex.py | 2 + pyproject.toml | 4 -- src/fmu/dataio/models/meta/content.py | 1 + src/fmu/dataio/models/meta/discriminator.py | 45 +++++++++++++++++++ src/fmu/dataio/models/meta/model.py | 18 ++++++-- .../test_schema/test_schema_logic_pydantic.py | 2 +- 7 files changed, 66 insertions(+), 8 deletions(-) create mode 100644 src/fmu/dataio/models/meta/discriminator.py diff --git a/boom.py b/boom.py index e8f6acebb..0a5360e7d 100644 --- a/boom.py +++ b/boom.py @@ -1,3 +1,5 @@ +# type: ignore + from __future__ import annotations from contextlib import suppress diff --git a/ex.py b/ex.py index b8954cba7..94e98c351 100644 --- a/ex.py +++ b/ex.py @@ -1,3 +1,5 @@ +# type: ignore + import sys from fmu.dataio.models.meta.model import Root diff --git a/pyproject.toml b/pyproject.toml index febe57817..e944f2b98 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,10 +82,6 @@ write_to = "src/fmu/dataio/version.py" [tool.pytest.ini_options] minversion = "6.0" -addopts = "--verbose" -log_cli = "False" -log_cli_format = "%(levelname)8s (%(relativeCreated)6.0fms) %(filename)44s [%(funcName)40s()] %(lineno)4d >> %(message)s" -log_cli_level = "INFO" testpaths = "tests" markers = ["integration: marks a test as an integration test"] xfail_strict = true diff --git a/src/fmu/dataio/models/meta/content.py b/src/fmu/dataio/models/meta/content.py index a9897dc12..662f77576 100644 --- a/src/fmu/dataio/models/meta/content.py +++ b/src/fmu/dataio/models/meta/content.py @@ -302,6 +302,7 @@ class WellPicksContent(Content): FaultLinesContent, FieldOutlineContent, FieldRegionContent, + FluidContactContent, InplaceVolumesContent, KPProductContent, LiftCurvesContent, diff --git a/src/fmu/dataio/models/meta/discriminator.py b/src/fmu/dataio/models/meta/discriminator.py new file mode 100644 index 000000000..ba0e316dc --- /dev/null +++ b/src/fmu/dataio/models/meta/discriminator.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from typing import Literal + +from pydantic import ValidationError + + +def fmu_discriminator( + value: dict, +) -> Literal[ + "FMUAggregation", + "FMURealization", + "FMUCase", +]: + """ + Discriminate the type of FMU based on the fields present in the value. + + This function determines the type of an FMU object by checking the presence + of specific fields ('aggregation' and 'realization') in the given value. + It returns a string literal indicating the determined type of the FMU. + + - If both 'aggregation' and 'realization' fields are present, + it raises a ValidationError. + - If 'aggregation' is present, it returns 'FMUAggregation'. + - If 'realization' is present, it returns 'FMURealization'. + - If neither is present, it defaults to 'FMUCase'. + """ + + if not isinstance(value, dict): + raise ValidationError("Input value must be a dictionary.") + + if "aggregation" in value and "realization" in value: + raise ValidationError( + "Value cannot have both 'aggregation' and 'realization' " + "fields. It must exclusively be either an 'FMUAggregation' " + "or an 'FMURealization'." + ) + + if "aggregation" in value: + return "FMUAggregation" + + if "realization" in value: + return "FMURealization" + + return "FMUCase" diff --git a/src/fmu/dataio/models/meta/model.py b/src/fmu/dataio/models/meta/model.py index 0bd374a36..c4bb19b57 100644 --- a/src/fmu/dataio/models/meta/model.py +++ b/src/fmu/dataio/models/meta/model.py @@ -4,11 +4,11 @@ from pathlib import Path from typing import Dict, Literal, Optional, Union -from pydantic import BaseModel, Field, NaiveDatetime, RootModel +from pydantic import BaseModel, Discriminator, Field, NaiveDatetime, RootModel, Tag from pydantic.json_schema import GenerateJsonSchema from typing_extensions import Annotated -from . import content, enums +from . import content, discriminator, enums class UUID(RootModel[str]): @@ -396,7 +396,19 @@ class FMUDataClassMeta(ClassMeta): alias="class", title="Metadata class", ) - fmu: Union[FMUAggregation, FMURealization, FMUCase] + + # The presence of the a feild controlls what kind of + # FMUObj it is. The fmu_discriminator will inspects + # the obj. and returns a tag that tells pydantic + # what model to use. + fmu: Annotated[ + Union[ + Annotated[FMUAggregation, Tag("FMUAggregation")], + Annotated[FMURealization, Tag("FMURealization")], + Annotated[FMUCase, Tag("FMUCase")], + ], + Discriminator(discriminator.fmu_discriminator), + ] access: SsdlAccess data: content.AnyContent file: File diff --git a/tests/test_schema/test_schema_logic_pydantic.py b/tests/test_schema/test_schema_logic_pydantic.py index b7320a34f..1d628beb1 100644 --- a/tests/test_schema/test_schema_logic_pydantic.py +++ b/tests/test_schema/test_schema_logic_pydantic.py @@ -5,7 +5,7 @@ import jsonschema import pytest from fmu.dataio._definitions import ALLOWED_CONTENTS -from fmu.dataio.models.meta2 import dump +from fmu.dataio.models.meta import dump # pylint: disable=no-member From 50b4e027ccd8981d9c3d723d74861b8b06151cc7 Mon Sep 17 00:00:00 2001 From: JB Lovland Date: Wed, 24 Jan 2024 09:41:48 +0100 Subject: [PATCH 16/25] Added AxisOrientation, AccessLevel, use nativ UUID --- src/fmu/dataio/models/meta/enums.py | 11 ++++++++ src/fmu/dataio/models/meta/model.py | 44 +++++++++++++---------------- 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/src/fmu/dataio/models/meta/enums.py b/src/fmu/dataio/models/meta/enums.py index 28467a383..221f0efe1 100644 --- a/src/fmu/dataio/models/meta/enums.py +++ b/src/fmu/dataio/models/meta/enums.py @@ -42,3 +42,14 @@ class FMUClassEnum(str, Enum): well = "well" points = "points" dictionary = "dictionary" + + +class AccessLevel(str, Enum): + asset = "asset" + internal = "internal" + restricted = "restricted" + + +class AxisOrientation(int, Enum): + normal = 1 + flipped = -1 diff --git a/src/fmu/dataio/models/meta/model.py b/src/fmu/dataio/models/meta/model.py index c4bb19b57..31a5df329 100644 --- a/src/fmu/dataio/models/meta/model.py +++ b/src/fmu/dataio/models/meta/model.py @@ -3,21 +3,14 @@ from collections import ChainMap from pathlib import Path from typing import Dict, Literal, Optional, Union +from uuid import UUID from pydantic import BaseModel, Discriminator, Field, NaiveDatetime, RootModel, Tag -from pydantic.json_schema import GenerateJsonSchema from typing_extensions import Annotated from . import content, discriminator, enums -class UUID(RootModel[str]): - root: str = Field( - examples=["ad214d85-8a1d-19da-e053-c918a4889309"], - pattern=r"^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$", - ) - - class Asset(BaseModel): name: str = Field(examples=["Drogon"]) @@ -27,15 +20,13 @@ class Ssdl(BaseModel): Sub-Surface Data Lake """ - access_level: Literal["internal", "restricted", "asset"] + access_level: enums.AccessLevel rep_include: bool class Access(BaseModel): asset: Asset - classification: Literal["internal", "restricted", "asset"] | None = Field( - default=None - ) + classification: Optional[enums.AccessLevel] = Field(default=None) class SsdlAccess(Access): @@ -57,7 +48,7 @@ class SurfaceSpec(Shape): undef: float = Field(description="Value representing undefined data") xinc: float = Field(description="Increment along the x-axis") xori: float = Field(description="Origin along the x-axis") - yflip: Literal[-1, 1] = Field(description="Flip along the y-axis, -1 or 1") + yflip: enums.AxisOrientation = Field(description="Flip along the y-axis, -1 or 1") yori: float = Field(description="Origin along the y-axis") @@ -105,8 +96,8 @@ class CubeSpec(SurfaceSpec): zori: float = Field(description="Origin along the z-axis") # Miscellaneous - yflip: Literal[-1, 1] = Field(description="Flip along the y-axis, -1 or 1") - zflip: Literal[-1, 1] = Field(description="Flip along the z-axis, -1 or 1") + yflip: enums.AxisOrientation = Field(description="Flip along the y-axis, -1 or 1") + zflip: enums.AxisOrientation = Field(description="Flip along the z-axis, -1 or 1") rotation: float = Field(description="Rotation angle") undef: float = Field(description="Value representing undefined data") @@ -188,7 +179,9 @@ class Case(BaseModel): user: User = Field( description="The user name used in ERT", ) - uuid: UUID + uuid: UUID = Field( + examples=["15ce3b84-766f-4c93-9050-b154861f9100"], + ) class Iteration(BaseModel): @@ -207,9 +200,12 @@ class Iteration(BaseModel): "A uuid reference to another iteration that this " "iteration was restarted from" ), + examples=["15ce3b84-766f-4c93-9050-b154861f9100"], ) - uuid: UUID + uuid: UUID = Field( + examples=["15ce3b84-766f-4c93-9050-b154861f9100"], + ) class Model(BaseModel): @@ -273,42 +269,42 @@ class Realization(BaseModel): default=None, description="Parameters for this realization", ) - uuid: UUID + uuid: UUID = Field(examples=["15ce3b84-766f-4c93-9050-b154861f9100"]) class CountryItem(BaseModel): identifier: str = Field( examples=["Norway"], ) - uuid: UUID + uuid: UUID = Field(examples=["15ce3b84-766f-4c93-9050-b154861f9100"]) class DiscoveryItem(BaseModel): short_identifier: str = Field( examples=["SomeDiscovery"], ) - uuid: UUID + uuid: UUID = Field(examples=["15ce3b84-766f-4c93-9050-b154861f9100"]) class FieldItem(BaseModel): identifier: str = Field( examples=["OseFax"], ) - uuid: UUID + uuid: UUID = Field(examples=["15ce3b84-766f-4c93-9050-b154861f9100"]) class CoordinateSystem(BaseModel): identifier: str = Field( examples=["ST_WGS84_UTM37N_P32637"], ) - uuid: UUID + uuid: UUID = Field(examples=["15ce3b84-766f-4c93-9050-b154861f9100"]) class StratigraphicColumn(BaseModel): identifier: str = Field( examples=["DROGON_2020"], ) - uuid: UUID + uuid: UUID = Field(examples=["15ce3b84-766f-4c93-9050-b154861f9100"]) class Smda(BaseModel): @@ -432,8 +428,6 @@ def dump() -> dict: return dict( ChainMap( { - "$id": "https://main-fmu-schemas-dev.radix.equinor.com/schemas/0.8.0/fmu_results.json", - "$schema": GenerateJsonSchema.schema_dialect, "$contractual": [ "class", "source", From 838b629c0a1f4d50807b2deb3158f91f03336f57 Mon Sep 17 00:00:00 2001 From: JB Lovland Date: Wed, 24 Jan 2024 09:55:40 +0100 Subject: [PATCH 17/25] Remove autouse on conftest fixtures --- tests/conftest.py | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 2cfc20b5b..345c0f0ac 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -75,12 +75,12 @@ def wrapper(*args, **kwargs): return wrapper -@pytest.fixture(name="testroot", scope="session", autouse=True) +@pytest.fixture(name="testroot", scope="session") def fixture_testroot(): return ROOTPWD -@pytest.fixture(name="fmurun", scope="session", autouse=True) +@pytest.fixture(name="fmurun", scope="session") def fixture_fmurun(tmp_path_factory): """Create a tmp folder structure for testing; here a new fmurun.""" tmppath = tmp_path_factory.mktemp("data") @@ -90,7 +90,7 @@ def fixture_fmurun(tmp_path_factory): return newpath -@pytest.fixture(name="fmurun_w_casemetadata", scope="session", autouse=True) +@pytest.fixture(name="fmurun_w_casemetadata", scope="session") def fixture_fmurun_w_casemetadata(tmp_path_factory): """Create a tmp folder structure for testing; here existing fmurun w/ case meta!""" tmppath = tmp_path_factory.mktemp("data3") @@ -101,7 +101,7 @@ def fixture_fmurun_w_casemetadata(tmp_path_factory): return rootpath -@pytest.fixture(name="fmurun_w_casemetadata_pred", scope="session", autouse=True) +@pytest.fixture(name="fmurun_w_casemetadata_pred", scope="session") def fixture_fmurun_w_casemetadata_pred(tmp_path_factory): """Create a tmp folder structure for testing; here existing fmurun w/ case meta!""" tmppath = tmp_path_factory.mktemp("data3") @@ -112,7 +112,7 @@ def fixture_fmurun_w_casemetadata_pred(tmp_path_factory): return rootpath -@pytest.fixture(name="fmurun_pred", scope="session", autouse=True) +@pytest.fixture(name="fmurun_pred", scope="session") def fixture_fmurun_pred(tmp_path_factory): """Create a tmp folder structure for testing; here a new fmurun for prediction.""" tmppath = tmp_path_factory.mktemp("data_pred") @@ -122,7 +122,7 @@ def fixture_fmurun_pred(tmp_path_factory): return newpath -@pytest.fixture(name="rmsrun_fmu_w_casemetadata", scope="session", autouse=True) +@pytest.fixture(name="rmsrun_fmu_w_casemetadata", scope="session") def fixture_rmsrun_fmu_w_casemetadata(tmp_path_factory): """Create a tmp folder structure for testing; here existing fmurun w/ case meta! @@ -139,7 +139,7 @@ def fixture_rmsrun_fmu_w_casemetadata(tmp_path_factory): return rmspath -@pytest.fixture(name="rmssetup", scope="module", autouse=True) +@pytest.fixture(name="rmssetup", scope="module") def fixture_rmssetup(tmp_path_factory): """Create the folder structure to mimic RMS project.""" @@ -157,7 +157,7 @@ def fixture_rmssetup(tmp_path_factory): return rmspath -@pytest.fixture(name="rmsglobalconfig", scope="module", autouse=True) +@pytest.fixture(name="rmsglobalconfig", scope="module") def fixture_rmsglobalconfig(rmssetup): """Read global config.""" # read the global config @@ -171,7 +171,7 @@ def fixture_rmsglobalconfig(rmssetup): return global_cfg -@pytest.fixture(name="globalvars_norw_letters", scope="module", autouse=True) +@pytest.fixture(name="globalvars_norw_letters", scope="module") def fixture_globalvars_norw_letters(tmp_path_factory): """Read a global config with norwegian special letters w/ fmu.config utilities.""" @@ -193,7 +193,7 @@ def fixture_globalvars_norw_letters(tmp_path_factory): return (rmspath, cfg, gname) -@pytest.fixture(name="casesetup", scope="module", autouse=True) +@pytest.fixture(name="casesetup", scope="module") def fixture_casesetup(tmp_path_factory): """Create the folder structure to mimic a fmu run""" @@ -206,7 +206,7 @@ def fixture_casesetup(tmp_path_factory): return tmppath -@pytest.fixture(name="caseglobalconfig", scope="module", autouse=True) +@pytest.fixture(name="caseglobalconfig", scope="module") def fixture_caseglobalconfig(): """Create as global config for case testing.""" gconfig = {} @@ -343,14 +343,14 @@ def fixture_edataobj2(globalconfig2): # ====================================================================================== -@pytest.fixture(name="schema_080", scope="session", autouse=True) +@pytest.fixture(name="schema_080", scope="session") def fixture_schema_080(): """Return 0.8.0 version of schema as json.""" return _parse_json(ROOTPWD / "schema/definitions/0.8.0/schema/fmu_results.json") -@pytest.fixture(name="metadata_examples", scope="session", autouse=True) +@pytest.fixture(name="metadata_examples", scope="session") def fixture_metadata_examples(): """Parse all metadata examples. @@ -371,14 +371,14 @@ def fixture_metadata_examples(): # ====================================================================================== -@pytest.fixture(name="regsurf", scope="module", autouse=True) +@pytest.fixture(name="regsurf", scope="module") def fixture_regsurf(): """Create an xtgeo surface.""" logger.info("Ran %s", inspect.currentframe().f_code.co_name) return xtgeo.RegularSurface(ncol=12, nrow=10, xinc=20, yinc=20, values=1234.0) -@pytest.fixture(name="polygons", scope="module", autouse=True) +@pytest.fixture(name="polygons", scope="module") def fixture_polygons(): """Create an xtgeo polygons.""" logger.info("Ran %s", inspect.currentframe().f_code.co_name) @@ -392,7 +392,7 @@ def fixture_polygons(): ) -@pytest.fixture(name="points", scope="module", autouse=True) +@pytest.fixture(name="points", scope="module") def fixture_points(): """Create an xtgeo points instance.""" logger.info("Ran %s", inspect.currentframe().f_code.co_name) @@ -407,35 +407,35 @@ def fixture_points(): ) -@pytest.fixture(name="cube", scope="module", autouse=True) +@pytest.fixture(name="cube", scope="module") def fixture_cube(): """Create an xtgeo cube instance.""" logger.info("Ran %s", inspect.currentframe().f_code.co_name) return xtgeo.Cube(ncol=3, nrow=4, nlay=5, xinc=12, yinc=12, zinc=4, rotation=30) -@pytest.fixture(name="grid", scope="module", autouse=True) +@pytest.fixture(name="grid", scope="module") def fixture_grid(): """Create an xtgeo grid instance.""" logger.info("Ran %s", inspect.currentframe().f_code.co_name) return xtgeo.create_box_grid((3, 4, 5)) -@pytest.fixture(name="gridproperty", scope="module", autouse=True) +@pytest.fixture(name="gridproperty", scope="module") def fixture_gridproperty(): """Create an xtgeo gridproperty instance.""" logger.info("Ran %s", inspect.currentframe().f_code.co_name) return xtgeo.GridProperty(ncol=3, nrow=7, nlay=3, values=123.0) -@pytest.fixture(name="dataframe", scope="module", autouse=True) +@pytest.fixture(name="dataframe", scope="module") def fixture_dataframe(): """Create an pandas dataframe instance.""" logger.info("Ran %s", inspect.currentframe().f_code.co_name) return pd.DataFrame({"COL1": [1, 2, 3, 4], "COL2": [99.0, 98.0, 97.0, 96.0]}) -@pytest.fixture(name="wellpicks", scope="module", autouse=True) +@pytest.fixture(name="wellpicks", scope="module") def fixture_wellpicks(): """Create a pandas dataframe containing wellpicks""" logger.info("Ran %s", inspect.currentframe().f_code.co_name) @@ -467,7 +467,7 @@ def fixture_wellpicks(): ) -@pytest.fixture(name="arrowtable", scope="module", autouse=True) +@pytest.fixture(name="arrowtable", scope="module") def fixture_arrowtable(): """Create an arrow table instance.""" try: @@ -485,7 +485,7 @@ def fixture_arrowtable(): return None -@pytest.fixture(name="aggr_surfs_mean", scope="module", autouse=True) +@pytest.fixture(name="aggr_surfs_mean", scope="module") def fixture_aggr_surfs_mean(fmurun_w_casemetadata, rmsglobalconfig, regsurf): """Create aggregated surfaces, and return aggr. mean surface + lists of metadata""" logger.info("Ran %s", inspect.currentframe().f_code.co_name) From 5ad550f5b677f0c8974b2cb701d31f921938c9ba Mon Sep 17 00:00:00 2001 From: JB Lovland Date: Wed, 24 Jan 2024 16:25:06 +0100 Subject: [PATCH 18/25] Two tests fail --- src/fmu/dataio/models/meta/content.py | 3 +- src/fmu/dataio/models/meta/discriminator.py | 45 ----------- src/fmu/dataio/models/meta/model.py | 81 ++++++++----------- tests/conftest.py | 14 +++- .../test_schema/test_schema_logic_pydantic.py | 36 +++------ 5 files changed, 59 insertions(+), 120 deletions(-) delete mode 100644 src/fmu/dataio/models/meta/discriminator.py diff --git a/src/fmu/dataio/models/meta/content.py b/src/fmu/dataio/models/meta/content.py index 662f77576..ce5fe256e 100644 --- a/src/fmu/dataio/models/meta/content.py +++ b/src/fmu/dataio/models/meta/content.py @@ -121,7 +121,6 @@ class Content(BaseModel): content: enums.ContentEnum = Field(description="The contents of this data object") alias: Optional[list[str]] = Field(default=None) - base: Optional[Layer] = None # Only valid for cooridate based meta. bbox: Optional[BoundingBox] = Field(default=None) @@ -167,6 +166,8 @@ class Content(BaseModel): examples=["ds_extract_geogrid", "ds_post_strucmod"], ) time: Optional[Time] = Field(default=None) + + base: Optional[Layer] = None top: Optional[Layer] = None undef_is_zero: Optional[bool] = Field( diff --git a/src/fmu/dataio/models/meta/discriminator.py b/src/fmu/dataio/models/meta/discriminator.py deleted file mode 100644 index ba0e316dc..000000000 --- a/src/fmu/dataio/models/meta/discriminator.py +++ /dev/null @@ -1,45 +0,0 @@ -from __future__ import annotations - -from typing import Literal - -from pydantic import ValidationError - - -def fmu_discriminator( - value: dict, -) -> Literal[ - "FMUAggregation", - "FMURealization", - "FMUCase", -]: - """ - Discriminate the type of FMU based on the fields present in the value. - - This function determines the type of an FMU object by checking the presence - of specific fields ('aggregation' and 'realization') in the given value. - It returns a string literal indicating the determined type of the FMU. - - - If both 'aggregation' and 'realization' fields are present, - it raises a ValidationError. - - If 'aggregation' is present, it returns 'FMUAggregation'. - - If 'realization' is present, it returns 'FMURealization'. - - If neither is present, it defaults to 'FMUCase'. - """ - - if not isinstance(value, dict): - raise ValidationError("Input value must be a dictionary.") - - if "aggregation" in value and "realization" in value: - raise ValidationError( - "Value cannot have both 'aggregation' and 'realization' " - "fields. It must exclusively be either an 'FMUAggregation' " - "or an 'FMURealization'." - ) - - if "aggregation" in value: - return "FMUAggregation" - - if "realization" in value: - return "FMURealization" - - return "FMUCase" diff --git a/src/fmu/dataio/models/meta/model.py b/src/fmu/dataio/models/meta/model.py index 31a5df329..98dfae5f1 100644 --- a/src/fmu/dataio/models/meta/model.py +++ b/src/fmu/dataio/models/meta/model.py @@ -5,10 +5,10 @@ from typing import Dict, Literal, Optional, Union from uuid import UUID -from pydantic import BaseModel, Discriminator, Field, NaiveDatetime, RootModel, Tag +from pydantic import BaseModel, Field, NaiveDatetime, RootModel from typing_extensions import Annotated -from . import content, discriminator, enums +from . import content, enums class Asset(BaseModel): @@ -119,7 +119,8 @@ class File(BaseModel): Block describing the file as the data appear in FMU context """ - absolute_path: Path = Field( + absolute_path: Optional[Path] = Field( + default=None, description="The absolute file path", examples=["/abs/path/share/results/maps/volantis_gp_base--depth.gri"], ) @@ -131,14 +132,14 @@ class File(BaseModel): description="md5 checksum of the file or bytestring", examples=["kjhsdfvsdlfk23knerknvk23"], ) - size_bytes: int | None = Field( + size_bytes: Optional[int] = Field( default=None, description="Size of file object in bytes", ) -class Parameters(RootModel[Dict[str, Union[int, float, str, dict]]]): - ... +class Parameters(RootModel): + root: Dict[str, Union[Parameters, int, float, str, Parameters]] class Aggregation(BaseModel): @@ -149,12 +150,13 @@ class Aggregation(BaseModel): operation: str = Field( description="The aggregation performed", ) - parameters: Parameters = Field( - description="Parameters for this realization", - ) realization_ids: list[int] = Field( description="Array of realization ids included in this aggregation" ) + parameters: Optional[Parameters] = Field( + default=None, + description="Parameters for this realization", + ) class Workflow(BaseModel): @@ -171,7 +173,6 @@ class User(BaseModel): class Case(BaseModel): - description: Optional[list[str]] = None name: str = Field( description="The case name", examples=["MyCaseName"], @@ -182,6 +183,9 @@ class Case(BaseModel): uuid: UUID = Field( examples=["15ce3b84-766f-4c93-9050-b154861f9100"], ) + description: Optional[list[str]] = Field( + default=None, + ) class Iteration(BaseModel): @@ -194,7 +198,10 @@ class Iteration(BaseModel): description="The convential name of this iteration, e.g. iter-0 or pred", examples=["iter-0"], ) - restart_from: UUID | None = Field( + uuid: UUID = Field( + examples=["15ce3b84-766f-4c93-9050-b154861f9100"], + ) + restart_from: Optional[UUID] = Field( default=None, description=( "A uuid reference to another iteration that this " @@ -203,10 +210,6 @@ class Iteration(BaseModel): examples=["15ce3b84-766f-4c93-9050-b154861f9100"], ) - uuid: UUID = Field( - examples=["15ce3b84-766f-4c93-9050-b154861f9100"], - ) - class Model(BaseModel): description: Optional[list[str]] = Field( @@ -255,20 +258,20 @@ class Realization(BaseModel): id: int = Field( description="The unique number of this realization as used in FMU", ) - jobs: Optional[RealizationJobs] = Field( - default=None, - description=( - "Content directly taken from the ERT jobs.json file for this realization" - ), - ) name: str = Field( description="The convential name of this iteration, e.g. iter-0 or pred", examples=["iter-0"], ) - parameters: Parameters | None = Field( + parameters: Optional[Parameters] = Field( default=None, description="Parameters for this realization", ) + jobs: Optional[RealizationJobs] = Field( + default=None, + description=( + "Content directly taken from the ERT jobs.json file for this realization" + ), + ) uuid: UUID = Field(examples=["15ce3b84-766f-4c93-9050-b154861f9100"]) @@ -332,29 +335,16 @@ class TracklogEvent(BaseModel): class FMUCase(BaseModel): - case: Case - model: Model - - -class FMUDataObj(FMUCase): - iteration: Optional[Iteration] = None - workflow: Optional[Workflow] = None - - -class FMUAggregation(FMUDataObj): """ The FMU block records properties that are specific to FMU """ - aggregation: Aggregation - - -class FMURealization(FMUDataObj): - """ - The FMU block records properties that are specific to FMU - """ - - realization: Realization + case: Case + model: Model + iteration: Optional[Iteration] = Field(default=None) + workflow: Optional[Workflow] = Field(default=None) + aggregation: Optional[Aggregation] = Field(default=None) + realization: Optional[Realization] = Field(default=None) class ClassMeta(BaseModel): @@ -397,14 +387,7 @@ class FMUDataClassMeta(ClassMeta): # FMUObj it is. The fmu_discriminator will inspects # the obj. and returns a tag that tells pydantic # what model to use. - fmu: Annotated[ - Union[ - Annotated[FMUAggregation, Tag("FMUAggregation")], - Annotated[FMURealization, Tag("FMURealization")], - Annotated[FMUCase, Tag("FMUCase")], - ], - Discriminator(discriminator.fmu_discriminator), - ] + fmu: FMUCase access: SsdlAccess data: content.AnyContent file: File diff --git a/tests/conftest.py b/tests/conftest.py index 345c0f0ac..c478901d1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -350,8 +350,7 @@ def fixture_schema_080(): return _parse_json(ROOTPWD / "schema/definitions/0.8.0/schema/fmu_results.json") -@pytest.fixture(name="metadata_examples", scope="session") -def fixture_metadata_examples(): +def metadata_examples(): """Parse all metadata examples. Returns: @@ -366,6 +365,17 @@ def fixture_metadata_examples(): } +@pytest.fixture(name="metadata_examples", scope="session") +def fixture_metadata_examples(): + """Parse all metadata examples. + + Returns: + Dict: Dictionary with filename as key, file contents as value. + + """ + return metadata_examples() + + # ====================================================================================== # Various objects # ====================================================================================== diff --git a/tests/test_schema/test_schema_logic_pydantic.py b/tests/test_schema/test_schema_logic_pydantic.py index 1d628beb1..b35751f59 100644 --- a/tests/test_schema/test_schema_logic_pydantic.py +++ b/tests/test_schema/test_schema_logic_pydantic.py @@ -4,8 +4,10 @@ import jsonschema import pytest +from conftest import metadata_examples from fmu.dataio._definitions import ALLOWED_CONTENTS from fmu.dataio.models.meta import dump +from fmu.dataio.models.meta.enums import ContentEnum # pylint: disable=no-member @@ -17,20 +19,10 @@ def pydantic_schema(): return dump() -def test_schema_basic_json_syntax(pydantic_schema): - """Confirm that schemas are valid JSON.""" - - assert "$schema" in pydantic_schema - - -def test_schema_example_filenames(metadata_examples): +@pytest.mark.parametrize("file, example", metadata_examples().items()) +def test_schema_example_filenames(file, example): """Assert that all examples are .yml, not .yaml""" - - # check that examples are there - assert len(metadata_examples) > 0 - - for filename in metadata_examples: - assert filename.endswith(".yml"), filename + assert file.endswith(".yml") # ====================================================================================== @@ -38,11 +30,10 @@ def test_schema_example_filenames(metadata_examples): # ====================================================================================== -def test_pydantic_schema_validate_examples_as_is(pydantic_schema, metadata_examples): +@pytest.mark.parametrize("file, example", metadata_examples().items()) +def test_pydantic_schema_validate_examples_as_is(pydantic_schema, file, example): """Confirm that examples are valid against the schema""" - - for i, (name, metadata) in enumerate(metadata_examples.items()): - jsonschema.validate(instance=metadata, schema=pydantic_schema) + jsonschema.validate(instance=example, schema=pydantic_schema) def test_pydantic_schema_file_block(pydantic_schema, metadata_examples): @@ -51,6 +42,7 @@ def test_pydantic_schema_file_block(pydantic_schema, metadata_examples): # get a specific example example = metadata_examples["surface_depth.yml"] + # Root.model_validate(example) # shall validate as-is jsonschema.validate(instance=example, schema=pydantic_schema) @@ -64,6 +56,8 @@ def test_pydantic_schema_file_block(pydantic_schema, metadata_examples): with pytest.raises(jsonschema.exceptions.ValidationError): jsonschema.validate(instance=_example, schema=pydantic_schema) + return + # shall not validate without checksum_md5 del _example["file"]["checksum_md5"] with pytest.raises(jsonschema.exceptions.ValidationError): @@ -392,15 +386,11 @@ def test_schema_logic_content_whitelist(pydantic_schema, metadata_examples): jsonschema.validate(instance=example_surface, schema=pydantic_schema) -def test_schema_content_synch_with_code(pydantic_schema): +def test_schema_content_synch_with_code(): """Currently, the whitelist for content is maintained both in the schema and in the code. This test asserts that list used in the code is in synch with schema. Note! This is one-way, and will not fail if additional elements are added to the schema only.""" - schema_allowed = pydantic_schema["$defs"]["data"]["properties"]["content"]["enum"] for allowed_content in ALLOWED_CONTENTS: - if allowed_content not in schema_allowed: - raise ValueError( - f"content '{allowed_content}' allowed in code, but not schema." - ) + assert allowed_content in {v.name for v in ContentEnum} From d6b2293d51d425559f125ecb2d800bb7598583b1 Mon Sep 17 00:00:00 2001 From: JB Lovland Date: Wed, 24 Jan 2024 17:01:35 +0100 Subject: [PATCH 19/25] Set allow_inf_nan=False --- src/fmu/dataio/models/meta/content.py | 35 ++++++++-- src/fmu/dataio/models/meta/model.py | 94 +++++++++++++++++++++------ 2 files changed, 104 insertions(+), 25 deletions(-) diff --git a/src/fmu/dataio/models/meta/content.py b/src/fmu/dataio/models/meta/content.py index ce5fe256e..872199539 100644 --- a/src/fmu/dataio/models/meta/content.py +++ b/src/fmu/dataio/models/meta/content.py @@ -42,9 +42,11 @@ class Seismic(BaseModel): examples=["mean"], ) filter_size: Optional[float] = Field( + allow_inf_nan=False, default=None, ) scaling_factor: Optional[float] = Field( + allow_inf_nan=False, default=None, ) stacking_offset: Optional[str] = Field( @@ -52,6 +54,7 @@ class Seismic(BaseModel): examples=["0-15"], ) zrange: Optional[float] = Field( + allow_inf_nan=False, default=None, ) @@ -98,6 +101,7 @@ class Layer(BaseModel): examples=["VIKING GP. Top"], ) offset: float = Field( + allow_inf_nan=False, default=0, ) stratigraphic: bool = Field( @@ -109,12 +113,30 @@ class Layer(BaseModel): class BoundingBox(BaseModel): - xmin: float = Field(description="Minimum x-coordinate") - xmax: float = Field(description="Maximum x-coordinate") - ymin: float = Field(description="Minimum y-coordinate") - ymax: float = Field(description="Maximum y-coordinate") - zmin: float = Field(description="Minimum z-coordinate") - zmax: float = Field(description="Maximum z-coordinate") + xmin: float = Field( + description="Minimum x-coordinate", + allow_inf_nan=False, + ) + xmax: float = Field( + description="Maximum x-coordinate", + allow_inf_nan=False, + ) + ymin: float = Field( + description="Minimum y-coordinate", + allow_inf_nan=False, + ) + ymax: float = Field( + description="Maximum y-coordinate", + allow_inf_nan=False, + ) + zmin: float = Field( + description="Minimum z-coordinate", + allow_inf_nan=False, + ) + zmax: float = Field( + description="Maximum z-coordinate", + allow_inf_nan=False, + ) class Content(BaseModel): @@ -152,6 +174,7 @@ class Content(BaseModel): ) offset: float = Field( default=0.0, + allow_inf_nan=False, ) # spec: Optional[TableSpec | CPGridSpec, ...] = None stratigraphic_alias: Optional[list[str]] = Field(default=None) diff --git a/src/fmu/dataio/models/meta/model.py b/src/fmu/dataio/models/meta/model.py index 98dfae5f1..f9ab70d6c 100644 --- a/src/fmu/dataio/models/meta/model.py +++ b/src/fmu/dataio/models/meta/model.py @@ -44,12 +44,29 @@ class CaseSpec(BaseModel): class SurfaceSpec(Shape): - rotation: float = Field(description="Rotation angle") - undef: float = Field(description="Value representing undefined data") - xinc: float = Field(description="Increment along the x-axis") - xori: float = Field(description="Origin along the x-axis") - yflip: enums.AxisOrientation = Field(description="Flip along the y-axis, -1 or 1") - yori: float = Field(description="Origin along the y-axis") + rotation: float = Field( + description="Rotation angle", + allow_inf_nan=False, + ) + undef: float = Field( + description="Value representing undefined data", + allow_inf_nan=False, + ) + xinc: float = Field( + description="Increment along the x-axis", + allow_inf_nan=False, + ) + xori: float = Field( + description="Origin along the x-axis", + allow_inf_nan=False, + ) + yflip: enums.AxisOrientation = Field( + description="Flip along the y-axis, -1 or 1", + ) + yori: float = Field( + description="Origin along the y-axis", + allow_inf_nan=False, + ) class TableSpec(BaseModel): @@ -65,13 +82,31 @@ class TableSpec(BaseModel): class CPGridSpec(Shape): """Corner point grid""" - xshift: float = Field(description="Shift along the x-axis") - yshift: float = Field(description="Shift along the y-axis") - zshift: float = Field(description="Shift along the z-axis") + xshift: float = Field( + description="Shift along the x-axis", + allow_inf_nan=False, + ) + yshift: float = Field( + description="Shift along the y-axis", + allow_inf_nan=False, + ) + zshift: float = Field( + description="Shift along the z-axis", + allow_inf_nan=False, + ) - xscale: float = Field(description="Scaling factor for the x-axis") - yscale: float = Field(description="Scaling factor for the y-axis") - zscale: float = Field(description="Scaling factor for the z-axis") + xscale: float = Field( + description="Scaling factor for the x-axis", + allow_inf_nan=False, + ) + yscale: float = Field( + description="Scaling factor for the y-axis", + allow_inf_nan=False, + ) + zscale: float = Field( + description="Scaling factor for the z-axis", + allow_inf_nan=False, + ) class CPGridPropertySpec(Shape): @@ -86,19 +121,40 @@ class PolygonsSpec(BaseModel): class CubeSpec(SurfaceSpec): # Increment - xinc: float = Field(description="Increment along the x-axis") - yinc: float = Field(description="Increment along the y-axis") - zinc: float = Field(description="Increment along the z-axis") + xinc: float = Field( + description="Increment along the x-axis", + allow_inf_nan=False, + ) + yinc: float = Field( + description="Increment along the y-axis", + allow_inf_nan=False, + ) + zinc: float = Field( + description="Increment along the z-axis", + allow_inf_nan=False, + ) # Origin - xori: float = Field(description="Origin along the x-axis") - yori: float = Field(description="Origin along the y-axis") - zori: float = Field(description="Origin along the z-axis") + xori: float = Field( + description="Origin along the x-axis", + allow_inf_nan=False, + ) + yori: float = Field( + description="Origin along the y-axis", + allow_inf_nan=False, + ) + zori: float = Field( + description="Origin along the z-axis", + allow_inf_nan=False, + ) # Miscellaneous yflip: enums.AxisOrientation = Field(description="Flip along the y-axis, -1 or 1") zflip: enums.AxisOrientation = Field(description="Flip along the z-axis, -1 or 1") - rotation: float = Field(description="Rotation angle") + rotation: float = Field( + description="Rotation angle", + allow_inf_nan=False, + ) undef: float = Field(description="Value representing undefined data") From dbb3d3cc498928f28cec9798af6ddbb9adf28af8 Mon Sep 17 00:00:00 2001 From: JB Lovland Date: Wed, 24 Jan 2024 17:28:12 +0100 Subject: [PATCH 20/25] add specification module --- src/fmu/dataio/models/meta/content.py | 4 +- src/fmu/dataio/models/meta/model.py | 137 ----------------- src/fmu/dataio/models/meta/specification.py | 155 ++++++++++++++++++++ 3 files changed, 157 insertions(+), 139 deletions(-) create mode 100644 src/fmu/dataio/models/meta/specification.py diff --git a/src/fmu/dataio/models/meta/content.py b/src/fmu/dataio/models/meta/content.py index 872199539..d5e30850a 100644 --- a/src/fmu/dataio/models/meta/content.py +++ b/src/fmu/dataio/models/meta/content.py @@ -5,7 +5,7 @@ from pydantic import BaseModel, Field from typing_extensions import Annotated -from . import enums +from . import enums, specification class FMUTimeObject(BaseModel): @@ -176,7 +176,7 @@ class Content(BaseModel): default=0.0, allow_inf_nan=False, ) - # spec: Optional[TableSpec | CPGridSpec, ...] = None + spec: Optional[specification.AnySpecification] = Field(default=None) stratigraphic_alias: Optional[list[str]] = Field(default=None) stratigraphic: bool = Field( description=( diff --git a/src/fmu/dataio/models/meta/model.py b/src/fmu/dataio/models/meta/model.py index f9ab70d6c..67b75fe02 100644 --- a/src/fmu/dataio/models/meta/model.py +++ b/src/fmu/dataio/models/meta/model.py @@ -33,143 +33,6 @@ class SsdlAccess(Access): ssdl: Ssdl -class Shape(BaseModel): - nrow: int = Field(description="The number of rows") - ncol: int = Field(description="The number of columns") - nlay: int = Field(description="The number of layers") - - -class CaseSpec(BaseModel): - ... - - -class SurfaceSpec(Shape): - rotation: float = Field( - description="Rotation angle", - allow_inf_nan=False, - ) - undef: float = Field( - description="Value representing undefined data", - allow_inf_nan=False, - ) - xinc: float = Field( - description="Increment along the x-axis", - allow_inf_nan=False, - ) - xori: float = Field( - description="Origin along the x-axis", - allow_inf_nan=False, - ) - yflip: enums.AxisOrientation = Field( - description="Flip along the y-axis, -1 or 1", - ) - yori: float = Field( - description="Origin along the y-axis", - allow_inf_nan=False, - ) - - -class TableSpec(BaseModel): - columns: list[str] = Field( - description="List of columns present in a table.", - ) - size: int = Field( - description="Size of data object.", - examples=[1, 9999], - ) - - -class CPGridSpec(Shape): - """Corner point grid""" - - xshift: float = Field( - description="Shift along the x-axis", - allow_inf_nan=False, - ) - yshift: float = Field( - description="Shift along the y-axis", - allow_inf_nan=False, - ) - zshift: float = Field( - description="Shift along the z-axis", - allow_inf_nan=False, - ) - - xscale: float = Field( - description="Scaling factor for the x-axis", - allow_inf_nan=False, - ) - yscale: float = Field( - description="Scaling factor for the y-axis", - allow_inf_nan=False, - ) - zscale: float = Field( - description="Scaling factor for the z-axis", - allow_inf_nan=False, - ) - - -class CPGridPropertySpec(Shape): - ... - - -class PolygonsSpec(BaseModel): - npolys: int = Field( - description="The number of individual polygons in the data object", - ) - - -class CubeSpec(SurfaceSpec): - # Increment - xinc: float = Field( - description="Increment along the x-axis", - allow_inf_nan=False, - ) - yinc: float = Field( - description="Increment along the y-axis", - allow_inf_nan=False, - ) - zinc: float = Field( - description="Increment along the z-axis", - allow_inf_nan=False, - ) - - # Origin - xori: float = Field( - description="Origin along the x-axis", - allow_inf_nan=False, - ) - yori: float = Field( - description="Origin along the y-axis", - allow_inf_nan=False, - ) - zori: float = Field( - description="Origin along the z-axis", - allow_inf_nan=False, - ) - - # Miscellaneous - yflip: enums.AxisOrientation = Field(description="Flip along the y-axis, -1 or 1") - zflip: enums.AxisOrientation = Field(description="Flip along the z-axis, -1 or 1") - rotation: float = Field( - description="Rotation angle", - allow_inf_nan=False, - ) - undef: float = Field(description="Value representing undefined data") - - -class WellSpec(BaseModel): - ... - - -class PointsSpec(BaseModel): - ... - - -class DictionarySpec(BaseModel): - ... - - class File(BaseModel): """ Block describing the file as the data appear in FMU context diff --git a/src/fmu/dataio/models/meta/specification.py b/src/fmu/dataio/models/meta/specification.py new file mode 100644 index 000000000..fa28d0821 --- /dev/null +++ b/src/fmu/dataio/models/meta/specification.py @@ -0,0 +1,155 @@ +from __future__ import annotations + +from typing import Union + +from pydantic import BaseModel, Field + +from . import enums + + +class Shape(BaseModel): + nrow: int = Field( + description="The number of rows", + ) + ncol: int = Field( + description="The number of columns", + ) + nlay: int = Field( + description="The number of layers", + ) + + +class SurfaceSpecification(Shape): + rotation: float = Field( + description="Rotation angle", + allow_inf_nan=False, + ) + undef: float = Field( + description="Value representing undefined data", + allow_inf_nan=False, + ) + xinc: float = Field( + description="Increment along the x-axis", + allow_inf_nan=False, + ) + xori: float = Field( + description="Origin along the x-axis", + allow_inf_nan=False, + ) + yflip: enums.AxisOrientation = Field( + description="Flip along the y-axis, -1 or 1", + ) + yori: float = Field( + description="Origin along the y-axis", + allow_inf_nan=False, + ) + + +class TableSpecification(BaseModel): + columns: list[str] = Field( + description="List of columns present in a table.", + ) + size: int = Field( + description="Size of data object.", + examples=[1, 9999], + ) + + +class CPGridSpecification(Shape): + """Corner point grid""" + + xshift: float = Field( + description="Shift along the x-axis", + allow_inf_nan=False, + ) + yshift: float = Field( + description="Shift along the y-axis", + allow_inf_nan=False, + ) + zshift: float = Field( + description="Shift along the z-axis", + allow_inf_nan=False, + ) + + xscale: float = Field( + description="Scaling factor for the x-axis", + allow_inf_nan=False, + ) + yscale: float = Field( + description="Scaling factor for the y-axis", + allow_inf_nan=False, + ) + zscale: float = Field( + description="Scaling factor for the z-axis", + allow_inf_nan=False, + ) + + +class CPGridPropertySpecification(Shape): + ... + + +class PolygonsSpecification(BaseModel): + npolys: int = Field( + description="The number of individual polygons in the data object", + ) + + +class CubeSpecification(SurfaceSpecification): + # Increment + xinc: float = Field( + description="Increment along the x-axis", + allow_inf_nan=False, + ) + yinc: float = Field( + description="Increment along the y-axis", + allow_inf_nan=False, + ) + zinc: float = Field( + description="Increment along the z-axis", + allow_inf_nan=False, + ) + + # Origin + xori: float = Field( + description="Origin along the x-axis", + allow_inf_nan=False, + ) + yori: float = Field( + description="Origin along the y-axis", + allow_inf_nan=False, + ) + zori: float = Field( + description="Origin along the z-axis", + allow_inf_nan=False, + ) + + # Miscellaneous + yflip: enums.AxisOrientation = Field( + description="Flip along the y-axis, -1 or 1", + ) + zflip: enums.AxisOrientation = Field( + description="Flip along the z-axis, -1 or 1", + ) + rotation: float = Field( + description="Rotation angle", + allow_inf_nan=False, + ) + undef: float = Field( + description="Value representing undefined data", + ) + + +class WellPointsDictionaryCaseSpecification(BaseModel): + ... + + +AnySpecification = Union[ + CPGridPropertySpecification, + CPGridSpecification, + CubeSpecification, + PolygonsSpecification, + SurfaceSpecification, + TableSpecification, + WellPointsDictionaryCaseSpecification, +] From 73e1a3f548d21425ffe327857681c2d40042c338 Mon Sep 17 00:00:00 2001 From: JB Lovland Date: Thu, 25 Jan 2024 10:48:53 +0100 Subject: [PATCH 21/25] Grooming --- tests/test_pydantic_vs_jsonschema.py | 33 +++++++++++++++++++ gen.sh => tools/gen.sh | 0 ex.py => tools/schema-examples-validate.py | 1 + .../sumo-explorer-model-validate.py | 0 tools/validate-schema-example.py | 24 ++++++++++++++ 5 files changed, 58 insertions(+) create mode 100644 tests/test_pydantic_vs_jsonschema.py rename gen.sh => tools/gen.sh (100%) rename ex.py => tools/schema-examples-validate.py (83%) rename boom.py => tools/sumo-explorer-model-validate.py (100%) create mode 100644 tools/validate-schema-example.py diff --git a/tests/test_pydantic_vs_jsonschema.py b/tests/test_pydantic_vs_jsonschema.py new file mode 100644 index 000000000..3523398d3 --- /dev/null +++ b/tests/test_pydantic_vs_jsonschema.py @@ -0,0 +1,33 @@ +import pytest +from fmu.dataio.models.meta import Root +from jsonschema import validate +from jsonschema.exceptions import ValidationError as JSValidationError +from polyfactory.factories.pydantic_factory import ModelFactory +from pydantic import ValidationError as PDValidationError + + +class RootFactory(ModelFactory[Root]): + ... + + +def pydantic_valid(obj): + try: + Root.model_validate(obj) + except PDValidationError: + return False + return True + + +def json_schema_valid(obj, schema_080): + try: + validate(instance=obj, schema=schema_080) + except JSValidationError: + return False + return True + + +@pytest.mark.parametrize("n", list(range(100))) +def test_pydantic_vs_jsonschema(n, schema_080): + obj = RootFactory.build().model_dump_json() + print(obj) + assert pydantic_valid(obj) == json_schema_valid(obj, schema_080) diff --git a/gen.sh b/tools/gen.sh similarity index 100% rename from gen.sh rename to tools/gen.sh diff --git a/ex.py b/tools/schema-examples-validate.py similarity index 83% rename from ex.py rename to tools/schema-examples-validate.py index 94e98c351..b0d35619f 100644 --- a/ex.py +++ b/tools/schema-examples-validate.py @@ -1,4 +1,5 @@ # type: ignore +# Ex. usage: time (find schema -name "*.yml" | python3 tools/validate-schema-example.py) import sys diff --git a/boom.py b/tools/sumo-explorer-model-validate.py similarity index 100% rename from boom.py rename to tools/sumo-explorer-model-validate.py diff --git a/tools/validate-schema-example.py b/tools/validate-schema-example.py new file mode 100644 index 000000000..cfcc00224 --- /dev/null +++ b/tools/validate-schema-example.py @@ -0,0 +1,24 @@ +# type: ignore +# Ex. usage: time (find schema -name "*.yml" | python3 tools/schema-example-validate.py) + +import sys + +from fmu.dataio.models.meta.model import Root +from orjson import dumps +from yaml import safe_load + + +def read(file): + with open(file) as f: + return f.read() + + +for file in (f.strip() for f in sys.stdin.readlines()): + print(file) + try: + Root.model_validate_json(dumps(safe_load(read(file)))) + except ValueError: + from pprint import pp + + pp(safe_load(read(file))) + raise From f7e7ed0610e8067d975507c14196dc0f9e0805c8 Mon Sep 17 00:00:00 2001 From: JB Lovland Date: Thu, 25 Jan 2024 10:49:44 +0100 Subject: [PATCH 22/25] Remove tests/test_pydantic_vs_jsonschema.py --- tests/test_pydantic_vs_jsonschema.py | 33 ---------------------------- 1 file changed, 33 deletions(-) delete mode 100644 tests/test_pydantic_vs_jsonschema.py diff --git a/tests/test_pydantic_vs_jsonschema.py b/tests/test_pydantic_vs_jsonschema.py deleted file mode 100644 index 3523398d3..000000000 --- a/tests/test_pydantic_vs_jsonschema.py +++ /dev/null @@ -1,33 +0,0 @@ -import pytest -from fmu.dataio.models.meta import Root -from jsonschema import validate -from jsonschema.exceptions import ValidationError as JSValidationError -from polyfactory.factories.pydantic_factory import ModelFactory -from pydantic import ValidationError as PDValidationError - - -class RootFactory(ModelFactory[Root]): - ... - - -def pydantic_valid(obj): - try: - Root.model_validate(obj) - except PDValidationError: - return False - return True - - -def json_schema_valid(obj, schema_080): - try: - validate(instance=obj, schema=schema_080) - except JSValidationError: - return False - return True - - -@pytest.mark.parametrize("n", list(range(100))) -def test_pydantic_vs_jsonschema(n, schema_080): - obj = RootFactory.build().model_dump_json() - print(obj) - assert pydantic_valid(obj) == json_schema_valid(obj, schema_080) From df93648b10291c9f210c0316dc30c77559199cb5 Mon Sep 17 00:00:00 2001 From: JB Lovland Date: Thu, 25 Jan 2024 13:33:11 +0100 Subject: [PATCH 23/25] All tests pass --- mypy.ini | 2 +- src/fmu/dataio/models/meta/content.py | 105 ++++++++++------ src/fmu/dataio/models/meta/enums.py | 4 +- src/fmu/dataio/models/meta/model.py | 114 ++++++++++++++---- src/fmu/dataio/models/meta/specification.py | 4 +- .../test_schema/test_schema_logic_pydantic.py | 14 ++- 6 files changed, 171 insertions(+), 72 deletions(-) diff --git a/mypy.ini b/mypy.ini index 2a56bf1a7..b8fc97559 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,7 +1,7 @@ [mypy] plugins = pydantic.mypy disallow_untyped_defs = True -exclude = ^((tests|docs|examples|build)/|conftest.py?) +exclude = ^((tests|docs|examples|build|tools)/|conftest.py?) extra_checks = True ignore_missing_imports = True python_version = 3.8 diff --git a/src/fmu/dataio/models/meta/content.py b/src/fmu/dataio/models/meta/content.py index d5e30850a..e59ad07d3 100644 --- a/src/fmu/dataio/models/meta/content.py +++ b/src/fmu/dataio/models/meta/content.py @@ -1,8 +1,9 @@ from __future__ import annotations -from typing import Literal, Optional, Union +from typing import Any, Dict, List, Literal, Optional, Union -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, GetJsonSchemaHandler, RootModel, model_validator +from pydantic_core import CoreSchema from typing_extensions import Annotated from . import enums, specification @@ -142,12 +143,12 @@ class BoundingBox(BaseModel): class Content(BaseModel): content: enums.ContentEnum = Field(description="The contents of this data object") - alias: Optional[list[str]] = Field(default=None) + alias: Optional[List[str]] = Field(default=None) # Only valid for cooridate based meta. bbox: Optional[BoundingBox] = Field(default=None) - description: Optional[list[str]] = Field( + description: Optional[List[str]] = Field( default=None, ) format: str = Field( @@ -177,7 +178,7 @@ class Content(BaseModel): allow_inf_nan=False, ) spec: Optional[specification.AnySpecification] = Field(default=None) - stratigraphic_alias: Optional[list[str]] = Field(default=None) + stratigraphic_alias: Optional[List[str]] = Field(default=None) stratigraphic: bool = Field( description=( "True if data object represents an entity in the stratigraphic column" @@ -190,9 +191,6 @@ class Content(BaseModel): ) time: Optional[Time] = Field(default=None) - base: Optional[Layer] = None - top: Optional[Layer] = None - undef_is_zero: Optional[bool] = Field( default=None, description="Flag if undefined values are to be interpreted as zero", @@ -206,6 +204,10 @@ class Content(BaseModel): examples=["depth"], ) + # Both must be set, or none. + base: Optional[Layer] = None + top: Optional[Layer] = None + class DepthContent(Content): content: Literal[enums.ContentEnum.depth] @@ -320,32 +322,61 @@ class WellPicksContent(Content): content: Literal[enums.ContentEnum.wellpicks] -AnyContent = Annotated[ - Union[ - DepthContent, - FaultLinesContent, - FieldOutlineContent, - FieldRegionContent, - FluidContactContent, - InplaceVolumesContent, - KPProductContent, - LiftCurvesContent, - ParametersContent, - PinchoutContent, - PropertyContent, - PTVContent, - RegionsContent, - RelpermContent, - RFTContent, - SeismicContent, - SubcropContent, - ThicknessContent, - TimeContent, - TimeSeriesContent, - VelocityContent, - VolumesContent, - VolumetricsContent, - WellPicksContent, - ], - Field(discriminator="content"), -] +class AnyContent(RootModel): + root: Annotated[ + Union[ + DepthContent, + FaultLinesContent, + FieldOutlineContent, + FieldRegionContent, + FluidContactContent, + InplaceVolumesContent, + KPProductContent, + LiftCurvesContent, + ParametersContent, + PinchoutContent, + PropertyContent, + PTVContent, + RegionsContent, + RelpermContent, + RFTContent, + SeismicContent, + SubcropContent, + ThicknessContent, + TimeContent, + TimeSeriesContent, + VelocityContent, + VolumesContent, + VolumetricsContent, + WellPicksContent, + ], + Field(discriminator="content"), + ] + + @model_validator(mode="before") + @classmethod + def _top_and_base_(cls, values: Dict) -> Dict: + top, base = values.get("top"), values.get("base") + if top is None and base is None: + return values + if top is not None and base is not None: + return values + raise ValueError("Both 'top' and 'base' must be set together or both be unset") + + @classmethod + def __get_pydantic_json_schema__( + cls, + core_schema: CoreSchema, + handler: GetJsonSchemaHandler, + ) -> Dict[str, Any]: + json_schema = super().__get_pydantic_json_schema__(core_schema, handler) + json_schema = handler.resolve_ref_schema(json_schema) + json_schema.update( + { + "dependencies": { + "top": {"required": ["base"]}, + "base": {"required": ["top"]}, + } + } + ) + return json_schema diff --git a/src/fmu/dataio/models/meta/enums.py b/src/fmu/dataio/models/meta/enums.py index 221f0efe1..d930bf967 100644 --- a/src/fmu/dataio/models/meta/enums.py +++ b/src/fmu/dataio/models/meta/enums.py @@ -1,6 +1,6 @@ from __future__ import annotations -from enum import Enum +from enum import Enum, IntEnum class ContentEnum(str, Enum): @@ -50,6 +50,6 @@ class AccessLevel(str, Enum): restricted = "restricted" -class AxisOrientation(int, Enum): +class AxisOrientation(IntEnum): normal = 1 flipped = -1 diff --git a/src/fmu/dataio/models/meta/model.py b/src/fmu/dataio/models/meta/model.py index 67b75fe02..12d004bc3 100644 --- a/src/fmu/dataio/models/meta/model.py +++ b/src/fmu/dataio/models/meta/model.py @@ -2,10 +2,18 @@ from collections import ChainMap from pathlib import Path -from typing import Dict, Literal, Optional, Union +from typing import Dict, List, Literal, Optional, Union from uuid import UUID -from pydantic import BaseModel, Field, NaiveDatetime, RootModel +from pydantic import ( + BaseModel, + Field, + GetJsonSchemaHandler, + NaiveDatetime, + RootModel, + model_validator, +) +from pydantic_core import CoreSchema from typing_extensions import Annotated from . import content, enums @@ -58,7 +66,7 @@ class File(BaseModel): class Parameters(RootModel): - root: Dict[str, Union[Parameters, int, float, str, Parameters]] + root: Dict[str, Union[Parameters, int, float, str]] class Aggregation(BaseModel): @@ -69,7 +77,7 @@ class Aggregation(BaseModel): operation: str = Field( description="The aggregation performed", ) - realization_ids: list[int] = Field( + realization_ids: List[int] = Field( description="Array of realization ids included in this aggregation" ) parameters: Optional[Parameters] = Field( @@ -91,7 +99,7 @@ class User(BaseModel): ) -class Case(BaseModel): +class FMUCase(BaseModel): name: str = Field( description="The case name", examples=["MyCaseName"], @@ -102,7 +110,7 @@ class Case(BaseModel): uuid: UUID = Field( examples=["15ce3b84-766f-4c93-9050-b154861f9100"], ) - description: Optional[list[str]] = Field( + description: Optional[List[str]] = Field( default=None, ) @@ -130,8 +138,8 @@ class Iteration(BaseModel): ) -class Model(BaseModel): - description: Optional[list[str]] = Field( +class FMUModel(BaseModel): + description: Optional[List[str]] = Field( default=None, description="This is a free text description of the model setup", ) @@ -146,8 +154,8 @@ class Model(BaseModel): class RealizationJobListing(BaseModel): - arg_types: list[str] - argList: list[Path] + arg_types: List[str] + argList: List[Path] error_file: Optional[Path] executable: Path license_path: Optional[Path] @@ -166,9 +174,9 @@ class RealizationJobListing(BaseModel): class RealizationJobs(BaseModel): data_root: Path = Field(alias="DATA_ROOT") ert_pid: str - global_environment: dict[str, str] + global_environment: Dict[str, str] global_update_path: dict - job_list: list[RealizationJobListing] = Field(alias="jobList") + job_list: List[RealizationJobListing] = Field(alias="jobList") run_id: str umask: str @@ -231,9 +239,9 @@ class StratigraphicColumn(BaseModel): class Smda(BaseModel): coordinate_system: CoordinateSystem - country: list[CountryItem] - discovery: list[DiscoveryItem] - field: list[FieldItem] + country: List[CountryItem] + discovery: List[DiscoveryItem] + field: List[FieldItem] stratigraphic_column: StratigraphicColumn @@ -253,18 +261,47 @@ class TracklogEvent(BaseModel): user: User -class FMUCase(BaseModel): +class FMU(BaseModel): """ The FMU block records properties that are specific to FMU """ - case: Case - model: Model + case: FMUCase + model: FMUModel iteration: Optional[Iteration] = Field(default=None) workflow: Optional[Workflow] = Field(default=None) aggregation: Optional[Aggregation] = Field(default=None) realization: Optional[Realization] = Field(default=None) + @model_validator(mode="before") + @classmethod + def _dependencies_aggregation_realization(cls, values: Dict) -> Dict: + aggregation, realization = values.get("aggregation"), values.get("realization") + if aggregation and realization: + raise ValueError( + "Both 'aggregation' and 'realization' cannot be set " + "at the same time. Please set only one." + ) + return values + + @classmethod + def __get_pydantic_json_schema__( + cls, + core_schema: CoreSchema, + handler: GetJsonSchemaHandler, + ) -> Dict[str, object]: + json_schema = super().__get_pydantic_json_schema__(core_schema, handler) + json_schema = handler.resolve_ref_schema(json_schema) + json_schema.update( + { + "dependencies": { + "aggregation": {"not": {"required": ["realization"]}}, + "realization": {"not": {"required": ["aggregation"]}}, + } + } + ) + return json_schema + class ClassMeta(BaseModel): class_: enums.FMUClassEnum = Field( @@ -272,7 +309,7 @@ class ClassMeta(BaseModel): title="Metadata class", ) masterdata: Masterdata - tracklog: list[TracklogEvent] + tracklog: List[TracklogEvent] source: Literal["fmu"] = Field(description="Data source (FMU)") version: Literal["0.8.0"] = Field(title="FMU results metadata version") @@ -282,7 +319,7 @@ class FMUCaseClassMeta(ClassMeta): alias="class", title="Metadata class", ) - fmu: FMUCase + fmu: FMU access: Access @@ -306,7 +343,7 @@ class FMUDataClassMeta(ClassMeta): # FMUObj it is. The fmu_discriminator will inspects # the obj. and returns a tag that tells pydantic # what model to use. - fmu: FMUCase + fmu: FMU access: SsdlAccess data: content.AnyContent file: File @@ -323,7 +360,34 @@ class Root( ] ] ): - ... + @model_validator(mode="before") + @classmethod + def _check_class_data_spec(cls, values: Dict) -> Dict: + class_ = values.get("class_") + data = values.get("data") + + if class_ in ["table", "surface"] and (data is None or "spec" not in data): + raise ValueError( + "When 'class' is 'table' or 'surface', " + "'data' must contain the 'spec' field." + ) + return values + + @classmethod + def __get_pydantic_json_schema__( + cls, + core_schema: CoreSchema, + handler: GetJsonSchemaHandler, + ) -> Dict[str, object]: + json_schema = super().__get_pydantic_json_schema__(core_schema, handler) + json_schema = handler.resolve_ref_schema(json_schema) + json_schema.update( + { + "if": {"properties": {"class": {"enum": ["table", "surface"]}}}, + "then": {"properties": {"data": {"required": ["spec"]}}}, + } + ) + return json_schema def dump() -> dict: @@ -368,9 +432,9 @@ def dump() -> dict: "file.checksum_md5", "file.size_bytes", ], + # schema must be present for "dependencies" key to work. + "$schema": "http://json-schema.org/draft-07/schema", }, - Root.model_json_schema( - by_alias=True, - ), + Root.model_json_schema(), ) ) diff --git a/src/fmu/dataio/models/meta/specification.py b/src/fmu/dataio/models/meta/specification.py index fa28d0821..dbb681ee9 100644 --- a/src/fmu/dataio/models/meta/specification.py +++ b/src/fmu/dataio/models/meta/specification.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Union +from typing import List, Union from pydantic import BaseModel, Field @@ -46,7 +46,7 @@ class SurfaceSpecification(Shape): class TableSpecification(BaseModel): - columns: list[str] = Field( + columns: List[str] = Field( description="List of columns present in a table.", ) size: int = Field( diff --git a/tests/test_schema/test_schema_logic_pydantic.py b/tests/test_schema/test_schema_logic_pydantic.py index b35751f59..f218313af 100644 --- a/tests/test_schema/test_schema_logic_pydantic.py +++ b/tests/test_schema/test_schema_logic_pydantic.py @@ -6,7 +6,7 @@ import pytest from conftest import metadata_examples from fmu.dataio._definitions import ALLOWED_CONTENTS -from fmu.dataio.models.meta import dump +from fmu.dataio.models.meta import Root, dump from fmu.dataio.models.meta.enums import ContentEnum # pylint: disable=no-member @@ -14,7 +14,7 @@ logger = logging.getLogger(__name__) -@pytest.fixture +@pytest.fixture(scope="session") def pydantic_schema(): return dump() @@ -31,11 +31,17 @@ def test_schema_example_filenames(file, example): @pytest.mark.parametrize("file, example", metadata_examples().items()) -def test_pydantic_schema_validate_examples_as_is(pydantic_schema, file, example): +def test_jsonschema_validate(pydantic_schema, file, example): """Confirm that examples are valid against the schema""" jsonschema.validate(instance=example, schema=pydantic_schema) +@pytest.mark.parametrize("file, example", metadata_examples().items()) +def test_pydantic_model_validate(pydantic_schema, file, example): + """Confirm that examples are valid against the schema""" + Root.model_validate(example) + + def test_pydantic_schema_file_block(pydantic_schema, metadata_examples): """Test variations on the file block.""" @@ -56,8 +62,6 @@ def test_pydantic_schema_file_block(pydantic_schema, metadata_examples): with pytest.raises(jsonschema.exceptions.ValidationError): jsonschema.validate(instance=_example, schema=pydantic_schema) - return - # shall not validate without checksum_md5 del _example["file"]["checksum_md5"] with pytest.raises(jsonschema.exceptions.ValidationError): From 3deb9b8a134707f001e6848928da57018eb3e895 Mon Sep 17 00:00:00 2001 From: JB Lovland Date: Thu, 25 Jan 2024 16:10:23 +0100 Subject: [PATCH 24/25] Rename models folder -> datastructure --- src/fmu/dataio/{models => datastructure}/meta/__init__.py | 0 src/fmu/dataio/{models => datastructure}/meta/__main__.py | 0 src/fmu/dataio/{models => datastructure}/meta/content.py | 0 src/fmu/dataio/{models => datastructure}/meta/enums.py | 0 src/fmu/dataio/{models => datastructure}/meta/model.py | 0 src/fmu/dataio/{models => datastructure}/meta/specification.py | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename src/fmu/dataio/{models => datastructure}/meta/__init__.py (100%) rename src/fmu/dataio/{models => datastructure}/meta/__main__.py (100%) rename src/fmu/dataio/{models => datastructure}/meta/content.py (100%) rename src/fmu/dataio/{models => datastructure}/meta/enums.py (100%) rename src/fmu/dataio/{models => datastructure}/meta/model.py (100%) rename src/fmu/dataio/{models => datastructure}/meta/specification.py (100%) diff --git a/src/fmu/dataio/models/meta/__init__.py b/src/fmu/dataio/datastructure/meta/__init__.py similarity index 100% rename from src/fmu/dataio/models/meta/__init__.py rename to src/fmu/dataio/datastructure/meta/__init__.py diff --git a/src/fmu/dataio/models/meta/__main__.py b/src/fmu/dataio/datastructure/meta/__main__.py similarity index 100% rename from src/fmu/dataio/models/meta/__main__.py rename to src/fmu/dataio/datastructure/meta/__main__.py diff --git a/src/fmu/dataio/models/meta/content.py b/src/fmu/dataio/datastructure/meta/content.py similarity index 100% rename from src/fmu/dataio/models/meta/content.py rename to src/fmu/dataio/datastructure/meta/content.py diff --git a/src/fmu/dataio/models/meta/enums.py b/src/fmu/dataio/datastructure/meta/enums.py similarity index 100% rename from src/fmu/dataio/models/meta/enums.py rename to src/fmu/dataio/datastructure/meta/enums.py diff --git a/src/fmu/dataio/models/meta/model.py b/src/fmu/dataio/datastructure/meta/model.py similarity index 100% rename from src/fmu/dataio/models/meta/model.py rename to src/fmu/dataio/datastructure/meta/model.py diff --git a/src/fmu/dataio/models/meta/specification.py b/src/fmu/dataio/datastructure/meta/specification.py similarity index 100% rename from src/fmu/dataio/models/meta/specification.py rename to src/fmu/dataio/datastructure/meta/specification.py From 54f275f567247d14cc78cfaf0ad91df577a9b493 Mon Sep 17 00:00:00 2001 From: JB Lovland Date: Thu, 25 Jan 2024 16:14:46 +0100 Subject: [PATCH 25/25] Rename models.py -> meta.py --- src/fmu/dataio/datastructure/meta/__init__.py | 2 +- src/fmu/dataio/datastructure/meta/content.py | 4 +- .../datastructure/meta/{model.py => meta.py} | 5 ++- .../test_schema/test_schema_logic_pydantic.py | 4 +- tools/sumo-explorer-model-validate.py | 37 ++++++++++++------- 5 files changed, 32 insertions(+), 20 deletions(-) rename src/fmu/dataio/datastructure/meta/{model.py => meta.py} (98%) diff --git a/src/fmu/dataio/datastructure/meta/__init__.py b/src/fmu/dataio/datastructure/meta/__init__.py index e3ebe81f8..1e35c2d3a 100644 --- a/src/fmu/dataio/datastructure/meta/__init__.py +++ b/src/fmu/dataio/datastructure/meta/__init__.py @@ -1,4 +1,4 @@ -from .model import Root, dump +from .meta import Root, dump __all__ = [ "dump", diff --git a/src/fmu/dataio/datastructure/meta/content.py b/src/fmu/dataio/datastructure/meta/content.py index e59ad07d3..329e2d744 100644 --- a/src/fmu/dataio/datastructure/meta/content.py +++ b/src/fmu/dataio/datastructure/meta/content.py @@ -263,7 +263,7 @@ class PropertyContent(Content): content: Literal[enums.ContentEnum.property] -class PTVContent(Content): +class PVTContent(Content): content: Literal[enums.ContentEnum.pvt] @@ -336,7 +336,7 @@ class AnyContent(RootModel): ParametersContent, PinchoutContent, PropertyContent, - PTVContent, + PVTContent, RegionsContent, RelpermContent, RFTContent, diff --git a/src/fmu/dataio/datastructure/meta/model.py b/src/fmu/dataio/datastructure/meta/meta.py similarity index 98% rename from src/fmu/dataio/datastructure/meta/model.py rename to src/fmu/dataio/datastructure/meta/meta.py index 12d004bc3..72614968c 100644 --- a/src/fmu/dataio/datastructure/meta/model.py +++ b/src/fmu/dataio/datastructure/meta/meta.py @@ -6,6 +6,7 @@ from uuid import UUID from pydantic import ( + AwareDatetime, BaseModel, Field, GetJsonSchemaHandler, @@ -252,7 +253,9 @@ class Masterdata(BaseModel): class TracklogEvent(BaseModel): # TODO: Update ex. to inc. timezone # update NaiveDatetime -> AwareDatetime - datetime: NaiveDatetime = Field( + # On upload, sumo adds timezone if its lacking. + # For roundtripping i need an Union here. + datetime: Union[NaiveDatetime, AwareDatetime] = Field( examples=["2020-10-28T14:28:02"], ) event: str = Field( diff --git a/tests/test_schema/test_schema_logic_pydantic.py b/tests/test_schema/test_schema_logic_pydantic.py index f218313af..b546d8a3c 100644 --- a/tests/test_schema/test_schema_logic_pydantic.py +++ b/tests/test_schema/test_schema_logic_pydantic.py @@ -6,8 +6,8 @@ import pytest from conftest import metadata_examples from fmu.dataio._definitions import ALLOWED_CONTENTS -from fmu.dataio.models.meta import Root, dump -from fmu.dataio.models.meta.enums import ContentEnum +from fmu.dataio.datastructure.meta import Root, dump +from fmu.dataio.datastructure.meta.enums import ContentEnum # pylint: disable=no-member diff --git a/tools/sumo-explorer-model-validate.py b/tools/sumo-explorer-model-validate.py index 0a5360e7d..b914f4753 100644 --- a/tools/sumo-explorer-model-validate.py +++ b/tools/sumo-explorer-model-validate.py @@ -2,10 +2,12 @@ from __future__ import annotations +from collections import Counter from contextlib import suppress +from pprint import pp from random import sample -from fmu.dataio.models.meta.model import Root +from fmu.dataio.datastructure.meta import Root from fmu.sumo.explorer import Explorer from tqdm import tqdm @@ -23,7 +25,7 @@ def lazy_sampler(x, lenx, k=100): def gen(): e = Explorer(env="dev") - for c in tqdm(sample(tuple(e.cases), 3), ascii=True, position=0): + for c in sample(tuple(e.cases), 25): yield c.metadata for cube in lazy_sampler(c.cubes, len(c.cubes)): @@ -43,15 +45,22 @@ def gen(): if __name__ == "__main__": - root_classes = set() - for m in tqdm(gen(), ascii=True, position=1): - try: - root_classes.add(Root.model_validate(m)) - except Exception: - from pprint import pp - - pp(m) - raise - - print("-->>> All good in da hood.") - print(root_classes) + tally = Counter() + with tqdm(ascii=True, position=1) as pbar: + for m in gen(): + pbar.update() + try: + parsed = Root.model_validate(m) + content = ( + parsed.root.data.root.content + if hasattr(parsed.root, "data") + else None + ) + tally.update([(parsed.root.class_, content)]) + if sum(tally.values()) % 25 == 0: + pbar.write("-" * 100) + pbar.write("\n".join(str(v) for v in tally.items())) + except Exception as e: + print(str(e)) + pp(m) + raise