Skip to content

Commit

Permalink
Disregard expected_objectives from OptimalResults
Browse files Browse the repository at this point in the history
the calculation within seba does not make sense
  • Loading branch information
yngve-sk committed Nov 14, 2024
1 parent d32d419 commit a1a028c
Show file tree
Hide file tree
Showing 11 changed files with 31 additions and 47 deletions.
27 changes: 24 additions & 3 deletions src/ert/run_models/everest_run_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import shutil
import threading
import time
from dataclasses import dataclass
from pathlib import Path
from types import TracebackType
from typing import (
Expand Down Expand Up @@ -290,6 +291,24 @@ class OptimizerCallback(Protocol):
def __call__(self) -> str | None: ...


@dataclass
class OptimalResult:
batch: int
controls: List[Any]
total_objective: float

@staticmethod
def from_seba_optimal_result(
o: Optional[seba_sqlite.sqlite_storage.OptimalResult] = None,
) -> "OptimalResult" | None:
if o is None:
return None

return OptimalResult(
batch=o.batch, controls=o.controls, total_objective=o.total_objective
)


class EverestRunModel(BaseRunModel):
def __init__(
self,
Expand Down Expand Up @@ -318,7 +337,7 @@ def __init__(
else (everest_config.simulator.delete_run_path or False)
)
self._display_all_jobs = display_all_jobs
self._result: Optional[seba_sqlite.sqlite_storage.OptimalResult] = None
self._result: Optional[OptimalResult] = None
self._exit_code: Optional[
Literal["max_batch_num_reached"] | OptimizerExitCode
] = None
Expand Down Expand Up @@ -415,7 +434,9 @@ def run_experiment(
optimizer_exit_code = optimizer.run().exit_code

# Extract the best result from the storage.
self._result = seba_storage.get_optimal_result() # type: ignore
self._result = OptimalResult.from_seba_optimal_result(
seba_storage.get_optimal_result() # type: ignore
)

if self._monitor_thread is not None:
self._monitor_thread.stop()
Expand Down Expand Up @@ -565,7 +586,7 @@ def exit_code(
return self._exit_code

@property
def result(self) -> Optional[seba_sqlite.sqlite_storage.OptimalResult]:
def result(self) -> Optional[OptimalResult]:
return self._result

def __repr__(self) -> str:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -381,9 +381,6 @@
"point_x-1": 0.00635029,
"point_x-2": 0.38836369
},
"total_objective": -1.52561897,
"expected_objectives": {
"distance": -1.52561897
}
"total_objective": -1.52561897
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -192,9 +192,6 @@
"point_y": 0.21233962,
"point_z": 0.49289096
},
"total_objective": -0.159768,
"expected_objectives": {
"distance": -0.159768
}
"total_objective": -0.159768
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -155,9 +155,6 @@
"point_y": 0.49610079,
"point_z": 0.50412446
},
"total_objective": -0.00003229,
"expected_objectives": {
"distance": 0.0
}
"total_objective": -0.00003229
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -103,9 +103,6 @@
"point_x": 3.0,
"point_y": 7.0
},
"total_objective": 7.0,
"expected_objectives": {
"func": 7.0
}
"total_objective": 7.0
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -126,9 +126,6 @@
"point_y": 0.50022835,
"point_z": 0.49962361
},
"total_objective": -2.2e-7,
"expected_objectives": {
"distance": -2.2e-7
}
"total_objective": -2.2e-7
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -151,10 +151,6 @@
"point_y": -0.00564829,
"point_z": 0.49992811
},
"total_objective": -2.00004834,
"expected_objectives": {
"distance_p": -0.50778502,
"distance_q": -4.47678995
}
"total_objective": -2.00004834
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -162,10 +162,6 @@
"point_y": 0.50220467,
"point_z": 0.48265145
},
"total_objective": -0.00022499,
"expected_objectives": {
"distance": 0.0,
"stddev": 0.0
}
"total_objective": -0.00022499
}
}
1 change: 0 additions & 1 deletion tests/everest/test_api_snapshots.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,6 @@ def test_api_snapshots(
"batch": optimal_result.batch,
"controls": optimal_result.controls,
"total_objective": optimal_result.total_objective,
"expected_objectives": optimal_result.expected_objectives,
}

api = EverestDataAPI(config)
Expand Down
8 changes: 0 additions & 8 deletions tests/everest/test_math_func.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,6 @@ def test_math_func_multiobj(
assert y == pytest.approx(0.0, abs=0.05)
assert z == pytest.approx(0.5, abs=0.05)

# Check the optimum values for each object.
optim_p = run_model.result.expected_objectives["distance_p"]
optim_q = run_model.result.expected_objectives["distance_q"]
assert optim_p == pytest.approx(-0.5, abs=0.05)
assert optim_q == pytest.approx(-4.5, abs=0.05)

# The overall optimum is a weighted average of the objectives
assert run_model.result.total_objective == pytest.approx(
(-0.5 * (2.0 / 3.0) * 1.5) + (-4.5 * (1.0 / 3.0) * 1.0), abs=0.01
Expand Down Expand Up @@ -73,8 +67,6 @@ def test_math_func_multiobj(
assert best["point_x"] == pytest.approx(x)
assert best["point_y"] == pytest.approx(y)
assert best["point_z"] == pytest.approx(z)
assert best["distance_p"] == pytest.approx(optim_p)
assert best["distance_q"] == pytest.approx(optim_q)
assert best["sim_avg_obj"] == pytest.approx(run_model.result.total_objective)

test_space = itertools.product(
Expand Down
5 changes: 0 additions & 5 deletions tests/everest/test_multiobjective.py
Original file line number Diff line number Diff line change
Expand Up @@ -105,8 +105,3 @@ def test_multi_objectives_run(
run_model = EverestRunModel.create(config)
evaluator_server_config = evaluator_server_config_generator(run_model)
run_model.run_experiment(evaluator_server_config)

# Loop through objective functions in config and ensure they are in the
# result object
for obj in config.objective_functions:
assert obj.name in run_model.result.expected_objectives

0 comments on commit a1a028c

Please sign in to comment.