diff --git a/src/ert/dark_storage/app.py b/src/ert/dark_storage/app.py index d093856470b..3fd3273563e 100644 --- a/src/ert/dark_storage/app.py +++ b/src/ert/dark_storage/app.py @@ -6,10 +6,11 @@ from fastapi.responses import HTMLResponse, RedirectResponse from ert.dark_storage.endpoints import router as endpoints_router +from ert.shared import __version__ app = FastAPI( - title=ert_storage_app.title, - version=ert_storage_app.version, + title="ERT Storage API (dark storage)", + version=__version__, debug=True, default_response_class=JSONResponse, # Disable documentation so we can replace it with ERT Storage's later @@ -51,23 +52,6 @@ async def not_implemented_handler( return JSONResponse({}, status_code=status.HTTP_501_NOT_IMPLEMENTED) -@app.get("/openapi.json", include_in_schema=False) -async def get_openapi() -> JSONResponse: - return JSONResponse(ert_storage_app.openapi()) - - -@app.get("/docs", include_in_schema=False) -async def get_swagger(req: Request) -> HTMLResponse: - return get_swagger_ui_html( - openapi_url="/openapi.json", title=f"{app.title} - Swagger UI" - ) - - -@app.get("/redoc", include_in_schema=False) -async def get_redoc(req: Request) -> HTMLResponse: - return get_redoc_html(openapi_url="/openapi.json", title=f"{app.title} - Redoc") - - @app.get("/") async def root() -> RedirectResponse: return RedirectResponse("/docs") diff --git a/src/ert/dark_storage/compute/__init__.py b/src/ert/dark_storage/compute/__init__.py new file mode 100644 index 00000000000..80c0c5f8cf7 --- /dev/null +++ b/src/ert/dark_storage/compute/__init__.py @@ -0,0 +1 @@ +from .misfits import calculate_misfits_from_pandas diff --git a/src/ert/dark_storage/compute/misfits.py b/src/ert/dark_storage/compute/misfits.py new file mode 100644 index 00000000000..2285391e0ec --- /dev/null +++ b/src/ert/dark_storage/compute/misfits.py @@ -0,0 +1,36 @@ +from typing import Any, List, Mapping, Optional +from uuid import UUID + +import numpy as np +import pandas as pd + + +def _calculate_misfit( + obs_value: np.ndarray, response_value: np.ndarray, obs_std: np.ndarray +) -> List[float]: + difference = response_value - obs_value + misfit = (difference / obs_std) ** 2 + return (misfit * np.sign(difference)).tolist() + + +def calculate_misfits_from_pandas( + reponses_dict: Mapping[int, pd.DataFrame], + observation: pd.DataFrame, + summary_misfits: bool = False, +) -> pd.DataFrame: + """ + Compute misfits from reponses_dict (real_id, values in dataframe) + and observation + """ + misfits_dict = {} + for realization_index in reponses_dict: + misfits_dict[realization_index] = _calculate_misfit( + observation["values"], + reponses_dict[realization_index].loc[:, observation.index].values.flatten(), + observation["errors"], + ) + + df = pd.DataFrame(data=misfits_dict, index=observation.index) + if summary_misfits: + df = pd.DataFrame([df.abs().sum(axis=0)], columns=df.columns, index=[0]) + return df.T diff --git a/src/ert/dark_storage/exceptions.py b/src/ert/dark_storage/exceptions.py new file mode 100644 index 00000000000..dff48ae43e9 --- /dev/null +++ b/src/ert/dark_storage/exceptions.py @@ -0,0 +1,30 @@ +from typing import Any + +from fastapi import status + + +class ErtStorageError(RuntimeError): + """ + Base error class for all the rest of errors + """ + + __status_code__ = status.HTTP_200_OK + + def __init__(self, message: str, **kwargs: Any): + super().__init__(message, kwargs) + + +class NotFoundError(ErtStorageError): + __status_code__ = status.HTTP_404_NOT_FOUND + + +class ConflictError(ErtStorageError): + __status_code__ = status.HTTP_409_CONFLICT + + +class ExpectationError(ErtStorageError): + __status_code__ = status.HTTP_417_EXPECTATION_FAILED + + +class UnprocessableError(ErtStorageError): + __status_code__ = status.HTTP_422_UNPROCESSABLE_ENTITY diff --git a/src/ert/dark_storage/json_schema/__init__.py b/src/ert/dark_storage/json_schema/__init__.py new file mode 100644 index 00000000000..8692ecf41f2 --- /dev/null +++ b/src/ert/dark_storage/json_schema/__init__.py @@ -0,0 +1,11 @@ +from .ensemble import EnsembleIn, EnsembleOut +from .experiment import ExperimentIn, ExperimentOut +from .observation import ( + ObservationIn, + ObservationOut, + ObservationTransformationIn, + ObservationTransformationOut, +) +from .prior import Prior +from .record import RecordOut +from .update import UpdateIn, UpdateOut diff --git a/src/ert/dark_storage/json_schema/ensemble.py b/src/ert/dark_storage/json_schema/ensemble.py new file mode 100644 index 00000000000..347b34c28db --- /dev/null +++ b/src/ert/dark_storage/json_schema/ensemble.py @@ -0,0 +1,37 @@ +from typing import Any, List, Mapping, Optional +from uuid import UUID + +from pydantic import BaseModel, Field, root_validator + + +class _Ensemble(BaseModel): + size: int + parameter_names: List[str] + response_names: List[str] + active_realizations: List[int] = [] + + +class EnsembleIn(_Ensemble): + update_id: Optional[UUID] = None + userdata: Mapping[str, Any] = {} + + @root_validator + def _check_names_no_overlap(cls, values: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Verify that `parameter_names` and `response_names` don't overlap. Ie, no + record can be both a parameter and a response. + """ + if not set(values["parameter_names"]).isdisjoint(set(values["response_names"])): + raise ValueError("parameters and responses cannot have a name in common") + return values + + +class EnsembleOut(_Ensemble): + id: UUID + children: List[UUID] = Field(alias="child_ensemble_ids") + parent: Optional[UUID] = Field(alias="parent_ensemble_id") + experiment_id: Optional[UUID] = None + userdata: Mapping[str, Any] + + class Config: + orm_mode = True diff --git a/src/ert/dark_storage/json_schema/experiment.py b/src/ert/dark_storage/json_schema/experiment.py new file mode 100644 index 00000000000..106e134e667 --- /dev/null +++ b/src/ert/dark_storage/json_schema/experiment.py @@ -0,0 +1,24 @@ +from typing import Any, List, Mapping +from uuid import UUID + +from pydantic import BaseModel + +from .prior import Prior + + +class _Experiment(BaseModel): + name: str + + +class ExperimentIn(_Experiment): + priors: Mapping[str, Prior] = {} + + +class ExperimentOut(_Experiment): + id: UUID + ensemble_ids: List[UUID] + priors: Mapping[str, dict] + userdata: Mapping[str, Any] + + class Config: + orm_mode = True diff --git a/src/ert/dark_storage/json_schema/observation.py b/src/ert/dark_storage/json_schema/observation.py new file mode 100644 index 00000000000..699a56d2d33 --- /dev/null +++ b/src/ert/dark_storage/json_schema/observation.py @@ -0,0 +1,43 @@ +from typing import Any, List, Mapping, Optional +from uuid import UUID + +from pydantic import BaseModel + + +class _ObservationTransformation(BaseModel): + name: str + active: List[bool] + scale: List[float] + observation_id: UUID + + +class ObservationTransformationIn(_ObservationTransformation): + pass + + +class ObservationTransformationOut(_ObservationTransformation): + id: UUID + + class Config: + orm_mode = True + + +class _Observation(BaseModel): + name: str + errors: List[float] + values: List[float] + x_axis: List[Any] + records: Optional[List[UUID]] = None + + +class ObservationIn(_Observation): + pass + + +class ObservationOut(_Observation): + id: UUID + transformation: Optional[ObservationTransformationOut] = None + userdata: Mapping[str, Any] = {} + + class Config: + orm_mode = True diff --git a/src/ert/dark_storage/json_schema/prior.py b/src/ert/dark_storage/json_schema/prior.py new file mode 100644 index 00000000000..b22bfc5b735 --- /dev/null +++ b/src/ert/dark_storage/json_schema/prior.py @@ -0,0 +1,152 @@ +import sys +from typing import Union + +from pydantic import BaseModel + +if sys.version_info < (3, 8): + from typing_extensions import Literal +else: + from typing import Literal + + +class PriorConst(BaseModel): + """ + Constant parameter prior + """ + + function: Literal["const"] = "const" + value: float + + +class PriorTrig(BaseModel): + """ + Triangular distribution parameter prior + """ + + function: Literal["trig"] = "trig" + min: float + max: float + mode: float + + +class PriorNormal(BaseModel): + """ + Normal distribution parameter prior + """ + + function: Literal["normal"] = "normal" + mean: float + std: float + + +class PriorLogNormal(BaseModel): + """ + Log-normal distribution parameter prior + """ + + function: Literal["lognormal"] = "lognormal" + mean: float + std: float + + +class PriorErtTruncNormal(BaseModel): + """ + ERT Truncated normal distribution parameter prior + + ERT differs from the usual distribution by that it simply clamps on `min` + and `max`, which gives a bias towards the extremes. + + """ + + function: Literal["ert_truncnormal"] = "ert_truncnormal" + mean: float + std: float + min: float + max: float + + +class PriorStdNormal(BaseModel): + """ + Standard normal distribution parameter prior + + Normal distribution with mean of 0 and standard deviation of 1 + """ + + function: Literal["stdnormal"] = "stdnormal" + + +class PriorUniform(BaseModel): + """ + Uniform distribution parameter prior + """ + + function: Literal["uniform"] = "uniform" + min: float + max: float + + +class PriorErtDUniform(BaseModel): + """ + ERT Discrete uniform distribution parameter prior + + This discrete uniform distribution differs from the standard by using the + `bins` parameter. Normally, `a`, and `b` are integers, and the sample space + are the integers between. ERT allows `a` and `b` to be arbitrary floats, + where the sample space is binned. + + """ + + function: Literal["ert_duniform"] = "ert_duniform" + bins: int + min: float + max: float + + +class PriorLogUniform(BaseModel): + """ + Logarithmic uniform distribution parameter prior + """ + + function: Literal["loguniform"] = "loguniform" + min: float + max: float + + +class PriorErtErf(BaseModel): + """ + ERT Error function distribution parameter prior + """ + + function: Literal["ert_erf"] = "ert_erf" + min: float + max: float + skewness: float + width: float + + +class PriorErtDErf(BaseModel): + """ + ERT Discrete error function distribution parameter prior + """ + + function: Literal["ert_derf"] = "ert_derf" + bins: int + min: float + max: float + skewness: float + width: float + + +Prior = Union[ + PriorConst, + PriorTrig, + PriorNormal, + PriorLogNormal, + PriorErtTruncNormal, + PriorStdNormal, + PriorUniform, + PriorErtDUniform, + PriorLogUniform, + PriorErtErf, + PriorErtDErf, +] diff --git a/src/ert/dark_storage/json_schema/record.py b/src/ert/dark_storage/json_schema/record.py new file mode 100644 index 00000000000..2df045abaa5 --- /dev/null +++ b/src/ert/dark_storage/json_schema/record.py @@ -0,0 +1,18 @@ +from typing import Any, Mapping, Optional +from uuid import UUID + +from pydantic import BaseModel, Field + + +class _Record(BaseModel): + pass + + +class RecordOut(_Record): + id: UUID + name: str + userdata: Mapping[str, Any] + has_observations: Optional[bool] + + class Config: + orm_mode = True diff --git a/src/ert/dark_storage/json_schema/update.py b/src/ert/dark_storage/json_schema/update.py new file mode 100644 index 00000000000..e3230312fcb --- /dev/null +++ b/src/ert/dark_storage/json_schema/update.py @@ -0,0 +1,24 @@ +from typing import List, Optional, Union +from uuid import UUID + +from pydantic import BaseModel + +from .observation import ObservationTransformationIn + + +class _Update(BaseModel): + algorithm: str + ensemble_result_id: Union[UUID, None] + ensemble_reference_id: Union[UUID, None] + + +class UpdateIn(_Update): + observation_transformations: Optional[List[ObservationTransformationIn]] = None + + +class UpdateOut(_Update): + id: UUID + experiment_id: UUID + + class Config: + orm_mode = True diff --git a/src/ert/dark_storage/security.py b/src/ert/dark_storage/security.py new file mode 100644 index 00000000000..f410b8047d9 --- /dev/null +++ b/src/ert/dark_storage/security.py @@ -0,0 +1,25 @@ +import os +from typing import Optional + +from fastapi import HTTPException, Security, status +from fastapi.security import APIKeyHeader + +DEFAULT_TOKEN = "hunter2" +_security_header = APIKeyHeader(name="Token", auto_error=False) + + +async def security(*, token: Optional[str] = Security(_security_header)) -> None: + if os.getenv("ERT_STORAGE_NO_TOKEN"): + return + if not token: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, detail="Not authenticated" + ) + real_token = os.getenv("ERT_STORAGE_TOKEN", DEFAULT_TOKEN) + if token == real_token: + # Success + return + + # HTTP 403 is when the user has authorized themselves, but aren't allowed to + # access this resource + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Invalid token")