From bc1e1e7d6b3c45b6e7e05ad420fba115a7b594cf Mon Sep 17 00:00:00 2001 From: DanSava Date: Tue, 8 Oct 2024 13:43:52 +0300 Subject: [PATCH 01/11] Allow non zero iteration number when creating experiment-ensemble pair using CreateExperimentDialog --- .../ertwidgets/create_experiment_dialog.py | 33 +++++++++++++++---- .../manage_experiments/storage_widget.py | 1 + src/ert/validation/__init__.py | 3 +- tests/ert/ui_tests/gui/test_main_window.py | 7 ++++ 4 files changed, 37 insertions(+), 7 deletions(-) diff --git a/src/ert/gui/ertwidgets/create_experiment_dialog.py b/src/ert/gui/ertwidgets/create_experiment_dialog.py index 110b3a43a3b..0eca7d8f5fe 100644 --- a/src/ert/gui/ertwidgets/create_experiment_dialog.py +++ b/src/ert/gui/ertwidgets/create_experiment_dialog.py @@ -13,15 +13,16 @@ ) from ert.gui.ertnotifier import ErtNotifier -from ert.gui.ertwidgets import StringBox, TextModel -from ert.validation.proper_name_argument import ( +from ert.gui.ertwidgets import StringBox, TextModel, ValueModel +from ert.validation import ( ExperimentValidation, + IntegerArgument, ProperNameArgument, ) class CreateExperimentDialog(QDialog): - onDone = Signal(str, str) + onDone = Signal(str, str, int) def __init__( self, @@ -54,6 +55,15 @@ def __init__( ) self._ensemble_edit.setValidator(ProperNameArgument()) + iteration_label = QLabel("Ensemble iteration:") + self._iterations_model = ValueModel(0) # type: ignore + self._iterations_field = StringBox( + self._iterations_model, # type: ignore + "0", + minimum_width=200, + ) + self._iterations_field.setValidator(IntegerArgument(from_value=0)) + self._iterations_field.setObjectName("iterations_field_ced") buttons = QDialogButtonBox( QDialogButtonBox.Ok | QDialogButtonBox.Cancel, Qt.Orientation.Horizontal, @@ -67,7 +77,7 @@ def __init__( self._ok_button.clicked.connect( lambda: self.onDone.emit( - self._experiment_edit.get_text, self._ensemble_edit.get_text + self.experiment_name, self.ensemble_name, self.iteration ) ) @@ -76,12 +86,15 @@ def enableOkButton() -> None: self._experiment_edit.textChanged.connect(enableOkButton) self._ensemble_edit.textChanged.connect(enableOkButton) + self._iterations_field.textChanged.connect(enableOkButton) layout.addWidget(experiment_label, 0, 0) layout.addWidget(self._experiment_edit, 0, 1) layout.addWidget(ensemble_label, 1, 0) layout.addWidget(self._ensemble_edit, 1, 1) - layout.addWidget(buttons, 2, 1) + layout.addWidget(iteration_label, 2, 0) + layout.addWidget(self._iterations_field, 2, 1) + layout.addWidget(buttons, 3, 1) self.setLayout(layout) @@ -103,5 +116,13 @@ def experiment_name(self) -> str: def ensemble_name(self) -> str: return self._ensemble_edit.get_text + @property + def iteration(self) -> int: + return int(self._iterations_field.get_text) + def isConfigurationValid(self) -> bool: - return self._experiment_edit.isValid() and self._ensemble_edit.isValid() + return ( + self._experiment_edit.isValid() + and self._ensemble_edit.isValid() + and self._iterations_field.isValid() + ) diff --git a/src/ert/gui/tools/manage_experiments/storage_widget.py b/src/ert/gui/tools/manage_experiments/storage_widget.py index a13e9dcc45c..3d419ecc92b 100644 --- a/src/ert/gui/tools/manage_experiments/storage_widget.py +++ b/src/ert/gui/tools/manage_experiments/storage_widget.py @@ -156,6 +156,7 @@ def _addItem(self) -> None: ).create_ensemble( name=create_experiment_dialog.ensemble_name, ensemble_size=self._ensemble_size, + iteration=create_experiment_dialog.iteration, ) self._notifier.set_current_ensemble(ensemble) self._notifier.ertChanged.emit() diff --git a/src/ert/validation/__init__.py b/src/ert/validation/__init__.py index 7b8ffa7cc01..ed54976b727 100644 --- a/src/ert/validation/__init__.py +++ b/src/ert/validation/__init__.py @@ -3,7 +3,7 @@ from .ensemble_realizations_argument import EnsembleRealizationsArgument from .integer_argument import IntegerArgument from .number_list_string_argument import NumberListStringArgument -from .proper_name_argument import ProperNameArgument +from .proper_name_argument import ExperimentValidation, ProperNameArgument from .proper_name_format_argument import ProperNameFormatArgument from .range_string_argument import RangeStringArgument from .rangestring import mask_to_rangestring, rangestring_to_list, rangestring_to_mask @@ -13,6 +13,7 @@ __all__ = [ "ActiveRange", "ArgumentDefinition", + "ExperimentValidation", "EnsembleRealizationsArgument", "IntegerArgument", "NumberListStringArgument", diff --git a/tests/ert/ui_tests/gui/test_main_window.py b/tests/ert/ui_tests/gui/test_main_window.py index 96dad43de8e..ae36dcea8de 100644 --- a/tests/ert/ui_tests/gui/test_main_window.py +++ b/tests/ert/ui_tests/gui/test_main_window.py @@ -391,12 +391,19 @@ def handle_add_dialog(): dialog._ensemble_edit.setText("_new_ensemble_") assert dialog._ok_button.isEnabled() + dialog._iterations_field.setText("a") + assert not dialog._ok_button.isEnabled() + dialog._iterations_field.setText("42") + assert dialog._ok_button.isEnabled() + qtbot.mouseClick(dialog._ok_button, Qt.MouseButton.LeftButton) QTimer.singleShot(1000, handle_add_dialog) create_widget = get_child(storage_widget, AddWidget) qtbot.mouseClick(create_widget.addButton, Qt.LeftButton) + assert experiments_panel.notifier.current_ensemble.iteration == 42 + # Go to the "initialize from scratch" panel experiments_panel.setCurrentIndex(1) current_tab = experiments_panel.currentWidget() From d3cff8129964fb998423a194d232b0ccfec44826 Mon Sep 17 00:00:00 2001 From: DanSava Date: Mon, 16 Sep 2024 09:27:57 +0300 Subject: [PATCH 02/11] Enable load results manually from any available iteration --- src/ert/callbacks.py | 80 ++++++++++--- src/ert/gui/ertwidgets/__init__.py | 2 + src/ert/gui/ertwidgets/textbox.py | 110 ++++++++++++++++++ .../tools/load_results/load_results_panel.py | 81 ++++--------- src/ert/libres_facade.py | 68 +++++------ src/ert/validation/__init__.py | 2 + src/ert/validation/string_definition.py | 39 +++++++ src/everest/bin/everload_script.py | 2 +- .../performance_tests/enkf/test_load_state.py | 4 +- .../cli/test_parameter_sample_types.py | 2 +- .../gui/test_load_results_manually.py | 14 ++- .../scenarios/test_summary_response.py | 4 +- .../ert/unit_tests/storage/create_runpath.py | 2 +- .../ert/unit_tests/test_load_forward_model.py | 67 +++++++++-- tests/ert/unit_tests/test_summary_response.py | 2 +- 15 files changed, 341 insertions(+), 138 deletions(-) create mode 100644 src/ert/gui/ertwidgets/textbox.py create mode 100644 src/ert/validation/string_definition.py diff --git a/src/ert/callbacks.py b/src/ert/callbacks.py index 1511dfe48f1..1db8cb6b6db 100644 --- a/src/ert/callbacks.py +++ b/src/ert/callbacks.py @@ -4,10 +4,10 @@ import logging import time from pathlib import Path -from typing import Iterable -from ert.config import InvalidResponseFile, ParameterConfig, ResponseConfig +from ert.config import InvalidResponseFile from ert.run_arg import RunArg +from ert.storage import Ensemble from ert.storage.realization_storage_state import RealizationStorageState from .load_status import LoadResult, LoadStatus @@ -16,24 +16,27 @@ async def _read_parameters( - run_arg: RunArg, parameter_configuration: Iterable[ParameterConfig] + run_path: str, + realization: int, + ensemble: Ensemble, ) -> LoadResult: result = LoadResult(LoadStatus.LOAD_SUCCESSFUL, "") error_msg = "" + parameter_configuration = ensemble.experiment.parameter_configuration.values() for config in parameter_configuration: if not config.forward_init: continue try: start_time = time.perf_counter() logger.debug(f"Starting to load parameter: {config.name}") - ds = config.read_from_runpath(Path(run_arg.runpath), run_arg.iens) + ds = config.read_from_runpath(Path(run_path), realization) await asyncio.sleep(0) logger.debug( f"Loaded {config.name}", extra={"Time": f"{(time.perf_counter() - start_time):.4f}s"}, ) start_time = time.perf_counter() - run_arg.ensemble_storage.save_parameters(config.name, run_arg.iens, ds) + ensemble.save_parameters(config.name, realization, ds) await asyncio.sleep(0) logger.debug( f"Saved {config.name} to storage", @@ -42,23 +45,26 @@ async def _read_parameters( except Exception as err: error_msg += str(err) result = LoadResult(LoadStatus.LOAD_FAILURE, error_msg) - logger.warning(f"Failed to load: {run_arg.iens}", exc_info=err) + logger.warning(f"Failed to load: {realization}", exc_info=err) return result async def _write_responses_to_storage( - run_arg: RunArg, response_configs: Iterable[ResponseConfig] + run_path: str, + realization: int, + ensemble: Ensemble, ) -> LoadResult: errors = [] + response_configs = ensemble.experiment.response_configuration.values() for config in response_configs: try: start_time = time.perf_counter() logger.debug(f"Starting to load response: {config.response_type}") try: - ds = config.read_from_file(run_arg.runpath, run_arg.iens) + ds = config.read_from_file(run_path, realization) except (FileNotFoundError, InvalidResponseFile) as err: errors.append(str(err)) - logger.warning(f"Failed to write: {run_arg.iens}: {err}") + logger.warning(f"Failed to write: {realization}: {err}") continue await asyncio.sleep(0) logger.debug( @@ -66,9 +72,7 @@ async def _write_responses_to_storage( extra={"Time": f"{(time.perf_counter() - start_time):.4f}s"}, ) start_time = time.perf_counter() - run_arg.ensemble_storage.save_response( - config.response_type, ds, run_arg.iens - ) + ensemble.save_response(config.response_type, ds, realization) await asyncio.sleep(0) logger.debug( f"Saved {config.response_type} to storage", @@ -77,7 +81,7 @@ async def _write_responses_to_storage( except Exception as err: errors.append(str(err)) logger.exception( - f"Unexpected exception while writing response to storage {run_arg.iens}", + f"Unexpected exception while writing response to storage {realization}", exc_info=err, ) continue @@ -97,14 +101,16 @@ async def forward_model_ok( # handles parameters if run_arg.itr == 0: parameters_result = await _read_parameters( - run_arg, - run_arg.ensemble_storage.experiment.parameter_configuration.values(), + run_arg.runpath, + run_arg.iens, + run_arg.ensemble_storage, ) if parameters_result.status == LoadStatus.LOAD_SUCCESSFUL: response_result = await _write_responses_to_storage( - run_arg, - run_arg.ensemble_storage.experiment.response_configuration.values(), + run_arg.runpath, + run_arg.iens, + run_arg.ensemble_storage, ) except Exception as err: @@ -128,3 +134,43 @@ async def forward_model_ok( run_arg.ensemble_storage.unset_failure(run_arg.iens) return final_result + + +async def load_realization( + run_path: str, + realization: int, + ensemble: Ensemble, +) -> LoadResult: + response_result = LoadResult(LoadStatus.LOAD_SUCCESSFUL, "") + try: + parameters_result = await _read_parameters( + run_path, + realization, + ensemble, + ) + + if parameters_result.status == LoadStatus.LOAD_SUCCESSFUL: + response_result = await _write_responses_to_storage( + run_path, + realization, + ensemble, + ) + + except Exception as err: + logger.exception(f"Failed to load results for realization {realization}") + parameters_result = LoadResult( + LoadStatus.LOAD_FAILURE, + "Failed to load results for realization " + f"{realization}, failed with: {err}", + ) + + final_result = parameters_result + if response_result.status != LoadStatus.LOAD_SUCCESSFUL: + final_result = response_result + ensemble.set_failure( + realization, RealizationStorageState.LOAD_FAILURE, final_result.message + ) + elif ensemble.has_failure(realization): + ensemble.unset_failure(realization) + + return final_result diff --git a/src/ert/gui/ertwidgets/__init__.py b/src/ert/gui/ertwidgets/__init__.py index 3d0c18867b5..15a7bf738b7 100644 --- a/src/ert/gui/ertwidgets/__init__.py +++ b/src/ert/gui/ertwidgets/__init__.py @@ -25,6 +25,7 @@ def wrapper(*arg: Any) -> Any: from .ensembleselector import EnsembleSelector from .checklist import CheckList from .stringbox import StringBox +from .textbox import TextBox from .multilinestringbox import MultiLineStringBox from .listeditbox import ListEditBox from .customdialog import CustomDialog @@ -59,6 +60,7 @@ def wrapper(*arg: Any) -> Any: "SelectableListModel", "StringBox", "TargetEnsembleModel", + "TextBox", "TextModel", "ValueModel", "showWaitCursorWhileWaiting", diff --git a/src/ert/gui/ertwidgets/textbox.py b/src/ert/gui/ertwidgets/textbox.py new file mode 100644 index 00000000000..190be153502 --- /dev/null +++ b/src/ert/gui/ertwidgets/textbox.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Optional + +from qtpy.QtGui import QPalette +from qtpy.QtWidgets import QTextEdit + +from .validationsupport import ValidationSupport + +if TYPE_CHECKING: + from ert.validation import StringDefinition + + from .models import TextModel + + +class TextBox(QTextEdit): + """StringBox shows a string. The data structure expected and sent to the + getter and setter is a string.""" + + def __init__( + self, + model: TextModel, + default_string: str = "", + placeholder_text: str = "", + minimum_width: int = 250, + ): + """ + :type model: ert.gui.ertwidgets.models.valuemodel.ValueModel + :type help_link: str + :type default_string: str + """ + QTextEdit.__init__(self) + self.setMinimumWidth(minimum_width) + self._validation = ValidationSupport(self) + self._validator: Optional[StringDefinition] = None + self._model = model + self._enable_validation = True + + if placeholder_text: + self.setPlaceholderText(placeholder_text) + + self.textChanged.connect(self.textBoxChanged) + self.textChanged.connect(self.validateString) + + self._valid_color = self.palette().color(self.backgroundRole()) + self.setText(default_string) + + self._model.valueChanged.connect(self.modelChanged) + self.modelChanged() + + def validateString(self) -> None: + if self._enable_validation: + string_to_validate = self.get_text + if self._validator is not None: + status = self._validator.validate(string_to_validate) + + palette = QPalette() + if not status: + palette.setColor( + self.backgroundRole(), ValidationSupport.ERROR_COLOR + ) + self.setPalette(palette) + self._validation.setValidationMessage( + str(status), ValidationSupport.EXCLAMATION + ) + else: + palette.setColor(self.backgroundRole(), self._valid_color) + self.setPalette(palette) + self._validation.setValidationMessage("") + + def emitChange(self, q_string: Any) -> None: + self.textChanged.emit(str(q_string)) + + def textBoxChanged(self) -> None: + """Called whenever the contents of the textbox changes.""" + text: Optional[str] = self.toPlainText() + if not text: + text = None + + self._model.setValue(text) + + def modelChanged(self) -> None: + """Retrieves data from the model and inserts it into the textbox""" + text = self._model.getValue() + if text is None: + text = "" + # If model and view has same text, return + if text == self.toPlainText(): + return + self.setText(str(text)) + + @property + def model(self) -> TextModel: + return self._model + + def setValidator(self, validator: StringDefinition) -> None: + self._validator = validator + + def getValidationSupport(self) -> ValidationSupport: + return self._validation + + def isValid(self) -> bool: + return self._validation.isValid() + + @property + def get_text(self) -> str: + return self.toPlainText() if self.toPlainText() else self.placeholderText() + + def enable_validation(self, enabled: bool) -> None: + self._enable_validation = enabled diff --git a/src/ert/gui/tools/load_results/load_results_panel.py b/src/ert/gui/tools/load_results/load_results_panel.py index 5612362edff..84671a297f5 100644 --- a/src/ert/gui/tools/load_results/load_results_panel.py +++ b/src/ert/gui/tools/load_results/load_results_panel.py @@ -11,11 +11,12 @@ MultiLineStringBox, QApplication, StringBox, - ValueModel, + TextBox, + TextModel, ) from ert.libres_facade import LibresFacade from ert.run_models.base_run_model import captured_logs -from ert.validation import IntegerArgument, RangeStringArgument, RunPathArgument +from ert.validation import RangeStringArgument, StringDefinition, RunPathArgument class LoadResultsPanel(QWidget): @@ -35,58 +36,34 @@ def __init__(self, facade: LibresFacade, notifier: ErtNotifier): layout = QFormLayout() - self.run_path_text_model = ValueModel() - self._run_path_field = MultiLineStringBox( - self.run_path_text_model, # type: ignore - default_string="", - readonly=True, + self._run_path_text = TextBox(TextModel(self.readCurrentRunPath())) + self._run_path_text.setFixedHeight(80) + self._run_path_text.setValidator(StringDefinition(required=[""])) + self._run_path_text.setObjectName("run_path_edit_lrm") + self._run_path_text.getValidationSupport().validationChanged.connect( + self.panelConfigurationChanged ) - self._run_path_field.setValidator(RunPathArgument()) - self._run_path_field.setObjectName("run_path_field_lrm") - self._run_path_field.setFixedHeight(80) - self._run_path_field.setText(self.readCurrentRunPath()) - layout.addRow("Load data from current run path: ", self._run_path_field) + layout.addRow("Load data from run path: ", self._run_path_text) ensemble_selector = EnsembleSelector(self._notifier) layout.addRow("Load into ensemble:", ensemble_selector) self._ensemble_selector = ensemble_selector - self._active_realizations_model = ActiveRealizationsModel( - self._facade.get_ensemble_size() - ) + ensemble_size = self._facade.get_ensemble_size() + self._active_realizations_model = ActiveRealizationsModel(ensemble_size) self._active_realizations_field = StringBox( self._active_realizations_model, # type: ignore "load_results_manually/Realizations", ) - self._active_realizations_field.setValidator( - RangeStringArgument(self._facade.get_ensemble_size()), - ) + self._active_realizations_field.setValidator(RangeStringArgument(ensemble_size)) self._active_realizations_field.setObjectName("active_realizations_lrm") layout.addRow("Realizations to load:", self._active_realizations_field) - self._iterations_model = ValueModel(0) # type: ignore - self._iterations_field = StringBox( - self._iterations_model, # type: ignore - "load_results_manually/iterations", - ) - self._iterations_field.setValidator(IntegerArgument(from_value=0)) - self._iterations_field.setObjectName("iterations_field_lrm") - layout.addRow("Iteration to load:", self._iterations_field) - self._run_path_field.getValidationSupport().validationChanged.connect( - self.panelConfigurationChanged - ) self._active_realizations_field.getValidationSupport().validationChanged.connect( self.panelConfigurationChanged ) - self._iterations_field.getValidationSupport().validationChanged.connect( - self.panelConfigurationChanged - ) - self.setLayout(layout) - def refresh(self) -> None: - self._run_path_field.refresh() - def readCurrentRunPath(self) -> str: current_ensemble = self._notifier.current_ensemble_name run_path = self._facade.run_path @@ -96,37 +73,21 @@ def readCurrentRunPath(self) -> str: def isConfigurationValid(self) -> bool: return ( - self._run_path_field.isValid() - and self._active_realizations_field.isValid() - and self._iterations_field.isValid() + self._active_realizations_field.isValid() and self._run_path_text.isValid() ) def load(self) -> int: - selected_ensemble = self._notifier.current_ensemble realizations = self._active_realizations_model.getActiveRealizationsMask() - iteration = self._iterations_model.getValue() - try: - if iteration is None: - iteration = "" - iteration_int = int(iteration) - except ValueError: - QMessageBox.warning( - self, - "Warning", - ( - "Expected an integer number in iteration field, " - f'got "{iteration}"' - ), - ) - return False - + active_realizations = [ + iens for iens, active in enumerate(realizations) if active + ] QApplication.setOverrideCursor(Qt.CursorShape.WaitCursor) messages: list[str] = [] with captured_logs(messages): - loaded = self._facade.load_from_forward_model( - selected_ensemble, # type: ignore - realizations, # type: ignore - iteration_int, + loaded = self._facade.load_from_run_path( + run_path_format=self._run_path_text.get_text, + ensemble=self._notifier.current_ensemble, # type: ignore + active_realizations=active_realizations, ) QApplication.restoreOverrideCursor() diff --git a/src/ert/libres_facade.py b/src/ert/libres_facade.py index c4bb0aec4b7..40f9e4577ed 100644 --- a/src/ert/libres_facade.py +++ b/src/ert/libres_facade.py @@ -5,6 +5,7 @@ import time import warnings from multiprocessing.pool import ThreadPool +from pathlib import Path from typing import ( TYPE_CHECKING, Any, @@ -20,7 +21,7 @@ from pandas import DataFrame from ert.analysis import AnalysisEvent, SmootherSnapshot, smoother_update -from ert.callbacks import forward_model_ok +from ert.callbacks import load_realization from ert.config import ( EnkfObservationImplementationType, ErtConfig, @@ -29,10 +30,8 @@ from ert.data import MeasuredData from ert.data._measured_data import ObservationError, ResponseError from ert.load_status import LoadResult, LoadStatus -from ert.run_arg import create_run_arguments from .plugins import ErtPluginContext -from .runpaths import Runpaths _logger = logging.getLogger(__name__) @@ -43,16 +42,16 @@ EnkfObs, WorkflowJob, ) - from ert.run_arg import RunArg from ert.storage import Ensemble, Storage -def _load_realization( - realisation: int, - run_args: List[RunArg], +def _load_realization_from_run_path( + run_path: str, + realization: int, + ensemble: Ensemble, ) -> Tuple[LoadResult, int]: - result = asyncio.run(forward_model_ok(run_args[realisation])) - return result, realisation + result = asyncio.run(load_realization(run_path, realization, ensemble)) + return result, realization class LibresFacade: @@ -122,34 +121,20 @@ def get_ensemble_size(self) -> int: def run_path(self) -> str: return self.config.model_config.runpath_format_string + @property + def resolved_run_path(self) -> str: + return str(Path(self.config.model_config.runpath_format_string).resolve()) + def load_from_forward_model( self, ensemble: Ensemble, realisations: npt.NDArray[np.bool_], - iteration: Optional[int] = None, ) -> int: - if iteration is not None: - warnings.warn( - "The iteration argument has no effect, iteration is read from ensemble", - DeprecationWarning, - stacklevel=1, - ) t = time.perf_counter() - run_args = create_run_arguments( - Runpaths( - jobname_format=self.config.model_config.jobname_format_string, - runpath_format=self.config.model_config.runpath_format_string, - filename=str(self.config.runpath_file), - substitutions=self.config.substitutions, - eclbase=self.config.model_config.eclbase_format_string, - ), - realisations, - ensemble=ensemble, - ) - nr_loaded = self._load_from_run_path( - self.config.model_config.num_realizations, - run_args, - realisations, + nr_loaded = self.load_from_run_path( + self.resolved_run_path, + ensemble, + [r for r, active in enumerate(realisations) if active], ) _logger.debug( f"load_from_forward_model() time_used {(time.perf_counter() - t):.4f}s" @@ -157,21 +142,26 @@ def load_from_forward_model( return nr_loaded @staticmethod - def _load_from_run_path( - ensemble_size: int, - run_args: List[RunArg], - active_realizations: npt.NDArray[np.bool_], + def load_from_run_path( + run_path_format: str, + ensemble: Ensemble, + active_realizations: List[int], ) -> int: """Returns the number of loaded realizations""" pool = ThreadPool(processes=8) async_result = [ pool.apply_async( - _load_realization, - (iens, run_args), + _load_realization_from_run_path, + ( + run_path_format.replace("", str(realization)).replace( + "", "0" + ), + realization, + ensemble, + ), ) - for iens in range(ensemble_size) - if active_realizations[iens] + for realization in active_realizations ] loaded = 0 diff --git a/src/ert/validation/__init__.py b/src/ert/validation/__init__.py index ed54976b727..3547923b68d 100644 --- a/src/ert/validation/__init__.py +++ b/src/ert/validation/__init__.py @@ -7,6 +7,7 @@ from .proper_name_format_argument import ProperNameFormatArgument from .range_string_argument import RangeStringArgument from .rangestring import mask_to_rangestring, rangestring_to_list, rangestring_to_mask +from .string_definition import StringDefinition from .runpath_argument import RunPathArgument from .validation_status import ValidationStatus @@ -20,6 +21,7 @@ "ProperNameArgument", "ProperNameFormatArgument", "RangeStringArgument", + "StringDefinition", "RunPathArgument", "ValidationStatus", "mask_to_rangestring", diff --git a/src/ert/validation/string_definition.py b/src/ert/validation/string_definition.py new file mode 100644 index 00000000000..9547fe7dd34 --- /dev/null +++ b/src/ert/validation/string_definition.py @@ -0,0 +1,39 @@ +from typing import List, Optional + +from .validation_status import ValidationStatus + + +class StringDefinition: + MISSING_TOKEN = "Missing required %s!" + INVALID_TOKEN = "Contains invalid string %s!" + + def __init__( + self, + optional: bool = False, + required: Optional[List[str]] = None, + invalid: Optional[List[str]] = None, + ) -> None: + super().__init__() + self.__optional = optional + self._required_tokens = required or [] + self._invalid_tokens = invalid or [] + + def isOptional(self) -> bool: + return self.__optional + + def validate(self, value: str) -> ValidationStatus: + vs = ValidationStatus() + required = [token for token in self._required_tokens if token not in value] + invalid = [token for token in self._invalid_tokens if token in value] + + if not self.isOptional() and any(required): + vs.setFailed() + for token in required: + vs.addToMessage(StringDefinition.MISSING_TOKEN % token) + + if not self.isOptional() and any(invalid): + vs.setFailed() + for token in invalid: + vs.addToMessage(StringDefinition.INVALID_TOKEN % token) + + return vs diff --git a/src/everest/bin/everload_script.py b/src/everest/bin/everload_script.py index 129779626e8..804a40c616c 100755 --- a/src/everest/bin/everload_script.py +++ b/src/everest/bin/everload_script.py @@ -189,7 +189,7 @@ def _internalize_batch(ert_config, batch_id, batch_data): realizations = [True] * batch_size + [False] * ( facade.get_ensemble_size() - batch_size ) - facade.load_from_forward_model(ensemble, realizations, 0) + facade.load_from_forward_model(ensemble, realizations) if __name__ == "__main__": diff --git a/tests/ert/performance_tests/enkf/test_load_state.py b/tests/ert/performance_tests/enkf/test_load_state.py index e5c481841dc..51442345d5e 100644 --- a/tests/ert/performance_tests/enkf/test_load_state.py +++ b/tests/ert/performance_tests/enkf/test_load_state.py @@ -15,7 +15,7 @@ def test_load_from_context(benchmark, template_config): expected_reals = template_config["reals"] realisations = [True] * expected_reals loaded_reals = benchmark( - facade.load_from_forward_model, load_into, realisations, 0 + facade.load_from_forward_model, load_into, realisations ) assert loaded_reals == expected_reals @@ -30,6 +30,6 @@ def test_load_from_fs(benchmark, template_config): expected_reals = template_config["reals"] realisations = [True] * expected_reals loaded_reals = benchmark( - facade.load_from_forward_model, load_from, realisations, 0 + facade.load_from_forward_model, load_from, realisations ) assert loaded_reals == expected_reals diff --git a/tests/ert/ui_tests/cli/test_parameter_sample_types.py b/tests/ert/ui_tests/cli/test_parameter_sample_types.py index 4ffef35f05f..2e921e421d6 100644 --- a/tests/ert/ui_tests/cli/test_parameter_sample_types.py +++ b/tests/ert/ui_tests/cli/test_parameter_sample_types.py @@ -18,7 +18,7 @@ def load_from_forward_model(ert_config, ensemble): facade = LibresFacade.from_config_file(ert_config) realizations = [True] * facade.get_ensemble_size() - return facade.load_from_forward_model(ensemble, realizations, 0) + return facade.load_from_forward_model(ensemble, realizations) @pytest.mark.usefixtures("set_site_config") diff --git a/tests/ert/ui_tests/gui/test_load_results_manually.py b/tests/ert/ui_tests/gui/test_load_results_manually.py index a28b5380456..e7ecc60619f 100644 --- a/tests/ert/ui_tests/gui/test_load_results_manually.py +++ b/tests/ert/ui_tests/gui/test_load_results_manually.py @@ -1,7 +1,7 @@ from qtpy.QtCore import Qt, QTimer from qtpy.QtWidgets import QPushButton -from ert.gui.ertwidgets import ClosableDialog, StringBox +from ert.gui.ertwidgets import ClosableDialog, StringBox, TextBox from ert.gui.ertwidgets.ensembleselector import EnsembleSelector from ert.gui.tools.load_results import LoadResultsPanel @@ -25,6 +25,11 @@ def handle_load_results_dialog(): load_button = get_child(panel.parent(), QPushButton, name="Load") + run_path_edit = get_child(panel, TextBox, name="run_path_edit_lrm") + assert run_path_edit.isEnabled() + valid_text = run_path_edit.get_text + assert "" in valid_text + active_realizations = get_child( panel, StringBox, name="active_realizations_lrm" ) @@ -37,12 +42,9 @@ def handle_load_results_dialog(): active_realizations.setText(default_value_active_reals) assert load_button.isEnabled() - iterations_field = get_child(panel, StringBox, name="iterations_field_lrm") - default_value_iteration = iterations_field.get_text - iterations_field.setText("-10") - + run_path_edit.setText(valid_text.replace("", "")) assert not load_button.isEnabled() - iterations_field.setText(default_value_iteration) + run_path_edit.setText(valid_text) assert load_button.isEnabled() dialog.close() diff --git a/tests/ert/unit_tests/scenarios/test_summary_response.py b/tests/ert/unit_tests/scenarios/test_summary_response.py index bb07dddaf74..0c93300e9bc 100644 --- a/tests/ert/unit_tests/scenarios/test_summary_response.py +++ b/tests/ert/unit_tests/scenarios/test_summary_response.py @@ -76,9 +76,7 @@ def create_responses(config_file, prior_ensemble, response_times): run_sim(response_time, rng.standard_normal(), fname=f"ECLIPSE_CASE_{i}") os.chdir(cwd) facade = LibresFacade.from_config_file(config_file) - facade.load_from_forward_model( - prior_ensemble, [True] * facade.get_ensemble_size(), 0 - ) + facade.load_from_forward_model(prior_ensemble, [True] * facade.get_ensemble_size()) def test_that_reading_matching_time_is_ok(ert_config, storage, prior_ensemble): diff --git a/tests/ert/unit_tests/storage/create_runpath.py b/tests/ert/unit_tests/storage/create_runpath.py index 1dc2b372098..dbf45a7a5f5 100644 --- a/tests/ert/unit_tests/storage/create_runpath.py +++ b/tests/ert/unit_tests/storage/create_runpath.py @@ -61,4 +61,4 @@ def create_runpath( def load_from_forward_model(ert_config, ensemble): facade = LibresFacade.from_config_file(ert_config) realizations = [True] * facade.get_ensemble_size() - return facade.load_from_forward_model(ensemble, realizations, 0) + return facade.load_from_forward_model(ensemble, realizations) diff --git a/tests/ert/unit_tests/test_load_forward_model.py b/tests/ert/unit_tests/test_load_forward_model.py index ffd775daa96..a08322057dd 100644 --- a/tests/ert/unit_tests/test_load_forward_model.py +++ b/tests/ert/unit_tests/test_load_forward_model.py @@ -11,6 +11,7 @@ from ert.config import ErtConfig from ert.enkf_main import create_run_path from ert.libres_facade import LibresFacade +from ert.run_arg import create_run_arguments from ert.storage import open_storage @@ -82,7 +83,7 @@ def test_load_forward_model(snake_oil_default_storage): experiment = storage.get_experiment_by_name("ensemble-experiment") default = experiment.get_ensemble_by_name("default_0") - loaded = facade.load_from_forward_model(default, realizations, 0) + loaded = facade.load_from_forward_model(default, realizations) assert loaded == 1 assert default.get_realization_mask_with_responses()[ realisation_number @@ -151,7 +152,7 @@ def test_load_forward_model_summary( ) facade = LibresFacade(ert_config) with caplog.at_level(logging.ERROR): - loaded = facade.load_from_forward_model(prior_ensemble, [True], 0) + loaded = facade.load_from_forward_model(prior_ensemble, [True]) expected_loaded, expected_log_message = expected assert loaded == expected_loaded if expected_log_message: @@ -176,7 +177,7 @@ def test_load_forward_model_gen_data(setup_case): fout.write("\n".join(["1", "0", "1"])) facade = LibresFacade(config) - facade.load_from_forward_model(prior_ensemble, [True], 0) + facade.load_from_forward_model(prior_ensemble, [True]) df = prior_ensemble.load_responses("gen_data", (0,)) filter_cond = polars.col("report_step").eq(0), polars.col("values").is_not_nan() assert df.filter(filter_cond)["values"].to_list() == [1.0, 3.0] @@ -198,7 +199,7 @@ def test_single_valued_gen_data_with_active_info_is_loaded(setup_case): fout.write("\n".join(["1"])) facade = LibresFacade(config) - facade.load_from_forward_model(prior_ensemble, [True], 0) + facade.load_from_forward_model(prior_ensemble, [True]) df = prior_ensemble.load_responses("RESPONSE", (0,)) assert df["values"].to_list() == [1.0] @@ -219,7 +220,7 @@ def test_that_all_deactivated_values_are_loaded(setup_case): fout.write("\n".join(["0"])) facade = LibresFacade(config) - facade.load_from_forward_model(prior_ensemble, [True], 0) + facade.load_from_forward_model(prior_ensemble, [True]) response = prior_ensemble.load_responses("RESPONSE", (0,)) assert np.isnan(response[0]["values"].to_list()) assert len(response) == 1 @@ -262,7 +263,7 @@ def test_loading_gen_data_without_restart(storage, run_paths, run_args): fout.write("\n".join(["1", "0", "1"])) facade = LibresFacade.from_config_file("config.ert") - facade.load_from_forward_model(prior_ensemble, [True], 0) + facade.load_from_forward_model(prior_ensemble, [True]) df = prior_ensemble.load_responses("RESPONSE", (0,)) df_no_nans = df.filter(polars.col("values").is_not_nan()) assert df_no_nans["values"].to_list() == [1.0, 3.0] @@ -284,6 +285,58 @@ def test_that_the_states_are_set_correctly(): new_ensemble = storage.create_ensemble( experiment=ensemble.experiment, ensemble_size=ensemble_size ) - facade.load_from_forward_model(new_ensemble, realizations, 0) + facade.load_from_forward_model(new_ensemble, realizations) assert not new_ensemble.is_initalized() assert new_ensemble.has_data() + + +@pytest.mark.parametrize("iter", [None, 0, 1, 2, 3]) +@pytest.mark.usefixtures("use_tmpdir") +def test_loading_from_any_available_iter(storage, run_paths, run_args, iter): + config_text = dedent( + """ + NUM_REALIZATIONS 1 + GEN_DATA RESPONSE RESULT_FILE:response.out INPUT_FORMAT:ASCII + """ + ) + Path("config.ert").write_text(config_text, encoding="utf-8") + + ert_config = ErtConfig.from_file("config.ert") + prior_ensemble = storage.create_ensemble( + storage.create_experiment( + responses=ert_config.ensemble_config.response_configuration + ), + name="prior", + ensemble_size=ert_config.model_config.num_realizations, + iteration=iter if iter is not None else 0, + ) + + run_args = create_run_arguments( + run_paths(ert_config), + [True] * ert_config.model_config.num_realizations, + prior_ensemble, + ) + create_run_path( + run_args, + prior_ensemble, + ert_config, + run_paths(ert_config), + ) + run_path = Path( + f"simulations/realization-0/iter-{iter if iter is not None else 0}/" + ) + with open(run_path / "response.out", "w", encoding="utf-8") as fout: + fout.write("\n".join(["1", "2", "3"])) + with open(run_path / "response.out_active", "w", encoding="utf-8") as fout: + fout.write("\n".join(["1", "0", "1"])) + + facade = LibresFacade.from_config_file("config.ert") + run_path_format = str( + Path( + f"simulations/realization-/iter-{iter if iter is not None else 0}" + ).resolve() + ) + facade.load_from_run_path(run_path_format, prior_ensemble, [0]) + df = prior_ensemble.load_responses("RESPONSE", (0,)) + df_no_nans = df.filter(polars.col("values").is_not_nan()) + assert df_no_nans["values"].to_list() == [1.0, 3.0] diff --git a/tests/ert/unit_tests/test_summary_response.py b/tests/ert/unit_tests/test_summary_response.py index 0439bb271c7..606c6713811 100644 --- a/tests/ert/unit_tests/test_summary_response.py +++ b/tests/ert/unit_tests/test_summary_response.py @@ -54,7 +54,7 @@ def test_load_summary_response_restart_not_zero( shutil.copy(test_path / "PRED_RUN.UNSMRY", sim_path / "PRED_RUN.UNSMRY") facade = LibresFacade.from_config_file("config.ert") - facade.load_from_forward_model(ensemble, [True], 0) + facade.load_from_forward_model(ensemble, [True]) df = ensemble.load_responses("summary", (0,)) df = df.pivot(on="response_key", values="values") From 862bcbff92328e1d85c4d7213e41cb8c12fec59f Mon Sep 17 00:00:00 2001 From: DanSava Date: Mon, 28 Oct 2024 10:25:58 +0200 Subject: [PATCH 03/11] Remove duplicate functionality in multilinestringbox.py --- src/ert/gui/ertwidgets/__init__.py | 2 - src/ert/gui/ertwidgets/multilinestringbox.py | 112 ------------------ .../tools/load_results/load_results_panel.py | 1 - 3 files changed, 115 deletions(-) delete mode 100644 src/ert/gui/ertwidgets/multilinestringbox.py diff --git a/src/ert/gui/ertwidgets/__init__.py b/src/ert/gui/ertwidgets/__init__.py index 15a7bf738b7..00253078b04 100644 --- a/src/ert/gui/ertwidgets/__init__.py +++ b/src/ert/gui/ertwidgets/__init__.py @@ -26,7 +26,6 @@ def wrapper(*arg: Any) -> Any: from .checklist import CheckList from .stringbox import StringBox from .textbox import TextBox -from .multilinestringbox import MultiLineStringBox from .listeditbox import ListEditBox from .customdialog import CustomDialog from .pathchooser import PathChooser @@ -53,7 +52,6 @@ def wrapper(*arg: Any) -> Any: "EnsembleSelector", "ErtMessageBox", "ListEditBox", - "MultiLineStringBox", "PathChooser", "PathModel", "SearchBox", diff --git a/src/ert/gui/ertwidgets/multilinestringbox.py b/src/ert/gui/ertwidgets/multilinestringbox.py deleted file mode 100644 index 306b945638f..00000000000 --- a/src/ert/gui/ertwidgets/multilinestringbox.py +++ /dev/null @@ -1,112 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING, Any, Optional - -from qtpy.QtGui import QPalette -from qtpy.QtWidgets import QTextEdit - -from .validationsupport import ValidationSupport - -if TYPE_CHECKING: - from ert.validation import ArgumentDefinition - - from .models import TextModel - - -class MultiLineStringBox(QTextEdit): - """MultiLineStringBox shows a multiline string. The data structure expected and sent to the - getter and setter is a multiline string.""" - - def __init__( - self, - model: TextModel, - default_string: str = "", - placeholder_text: str = "", - minimum_width: int = 250, - readonly: bool = False, - ): - QTextEdit.__init__(self) - self.setMinimumWidth(minimum_width) - self._validation = ValidationSupport(self) - self._validator: Optional[ArgumentDefinition] = None - self._model = model - self._enable_validation = True - - if placeholder_text: - self.setPlaceholderText(placeholder_text) - self.textChanged.connect(self.stringBoxChanged) - - self.textChanged.connect(self.validateString) - - self._valid_color = self.palette().color(self.backgroundRole()) - self.setText(default_string) - - self._model.valueChanged.connect(self.modelChanged) - self.modelChanged() - self.setReadOnly(readonly) - - def validateString(self) -> None: - if not self._enable_validation or self._validator is None: - return - - string_to_validate = self.toPlainText() - if not string_to_validate and self.placeholderText(): - string_to_validate = self.placeholderText() - - validation_success = self._validator.validate(string_to_validate) - - palette = self.palette() - if not validation_success: - palette.setColor(QPalette.ColorRole.Base, ValidationSupport.ERROR_COLOR) - self.setPalette(palette) - self._validation.setValidationMessage( - str(validation_success), ValidationSupport.EXCLAMATION - ) - else: - palette.setColor(QPalette.ColorRole.Base, self._valid_color) - self.setPalette(palette) - self._validation.setValidationMessage("") - - def emitChange(self, q_string: Any) -> None: - self.textChanged.emit(str(q_string)) - - def stringBoxChanged(self) -> None: - """Called whenever the contents of the textedit changes.""" - text: Optional[str] = self.get_text - if not text: - text = None - - self._model.setValue(text) - - def modelChanged(self) -> None: - """Retrieves data from the model and inserts it into the textedit""" - text = self._model.getValue() - if text is None: - text = "" - # If model and view has same text, return - if text == self.toPlainText(): - return - self.setText(str(text)) - - @property - def model(self) -> TextModel: - return self._model - - def setValidator(self, validator: ArgumentDefinition) -> None: - self._validator = validator - - def getValidationSupport(self) -> ValidationSupport: - return self._validation - - def isValid(self) -> bool: - return self._validation.isValid() - - @property - def get_text(self) -> str: - return self.toPlainText() if self.toPlainText() else self.placeholderText() - - def enable_validation(self, enabled: bool) -> None: - self._enable_validation = enabled - - def refresh(self) -> None: - self.validateString() diff --git a/src/ert/gui/tools/load_results/load_results_panel.py b/src/ert/gui/tools/load_results/load_results_panel.py index 84671a297f5..8545f570c73 100644 --- a/src/ert/gui/tools/load_results/load_results_panel.py +++ b/src/ert/gui/tools/load_results/load_results_panel.py @@ -8,7 +8,6 @@ ActiveRealizationsModel, EnsembleSelector, ErtMessageBox, - MultiLineStringBox, QApplication, StringBox, TextBox, From 83806f1ec6782791f624770a25bd026e0f49384b Mon Sep 17 00:00:00 2001 From: DanSava Date: Mon, 28 Oct 2024 10:33:11 +0200 Subject: [PATCH 04/11] Add exc_info when load_realization fails. --- src/ert/callbacks.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/ert/callbacks.py b/src/ert/callbacks.py index 1db8cb6b6db..141a941de3f 100644 --- a/src/ert/callbacks.py +++ b/src/ert/callbacks.py @@ -157,7 +157,10 @@ async def load_realization( ) except Exception as err: - logger.exception(f"Failed to load results for realization {realization}") + logger.exception( + f"Failed to load results for realization {realization}", + exc_info=err, + ) parameters_result = LoadResult( LoadStatus.LOAD_FAILURE, "Failed to load results for realization " From a84d5fd2eec2fcd1bda73c50e7f1bd6f86512831 Mon Sep 17 00:00:00 2001 From: DanSava Date: Mon, 28 Oct 2024 10:35:34 +0200 Subject: [PATCH 05/11] Style fix --- src/ert/gui/tools/load_results/load_results_panel.py | 2 +- src/ert/validation/__init__.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ert/gui/tools/load_results/load_results_panel.py b/src/ert/gui/tools/load_results/load_results_panel.py index 8545f570c73..35c3b633220 100644 --- a/src/ert/gui/tools/load_results/load_results_panel.py +++ b/src/ert/gui/tools/load_results/load_results_panel.py @@ -15,7 +15,7 @@ ) from ert.libres_facade import LibresFacade from ert.run_models.base_run_model import captured_logs -from ert.validation import RangeStringArgument, StringDefinition, RunPathArgument +from ert.validation import RangeStringArgument, StringDefinition class LoadResultsPanel(QWidget): diff --git a/src/ert/validation/__init__.py b/src/ert/validation/__init__.py index 3547923b68d..a33dc1d4ffe 100644 --- a/src/ert/validation/__init__.py +++ b/src/ert/validation/__init__.py @@ -7,22 +7,22 @@ from .proper_name_format_argument import ProperNameFormatArgument from .range_string_argument import RangeStringArgument from .rangestring import mask_to_rangestring, rangestring_to_list, rangestring_to_mask -from .string_definition import StringDefinition from .runpath_argument import RunPathArgument +from .string_definition import StringDefinition from .validation_status import ValidationStatus __all__ = [ "ActiveRange", "ArgumentDefinition", - "ExperimentValidation", "EnsembleRealizationsArgument", + "ExperimentValidation", "IntegerArgument", "NumberListStringArgument", "ProperNameArgument", "ProperNameFormatArgument", "RangeStringArgument", - "StringDefinition", "RunPathArgument", + "StringDefinition", "ValidationStatus", "mask_to_rangestring", "rangestring_to_list", From af173cb276f516ae49cfd4c97931ce756035d162 Mon Sep 17 00:00:00 2001 From: DanSava Date: Mon, 28 Oct 2024 11:16:53 +0200 Subject: [PATCH 06/11] Add refresh functionality --- src/ert/gui/ertwidgets/textbox.py | 3 +++ src/ert/gui/tools/load_results/load_results_panel.py | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/ert/gui/ertwidgets/textbox.py b/src/ert/gui/ertwidgets/textbox.py index 190be153502..2ae01d1a483 100644 --- a/src/ert/gui/ertwidgets/textbox.py +++ b/src/ert/gui/ertwidgets/textbox.py @@ -108,3 +108,6 @@ def get_text(self) -> str: def enable_validation(self, enabled: bool) -> None: self._enable_validation = enabled + + def refresh(self) -> None: + self.validateString() diff --git a/src/ert/gui/tools/load_results/load_results_panel.py b/src/ert/gui/tools/load_results/load_results_panel.py index 35c3b633220..b99ff9a2077 100644 --- a/src/ert/gui/tools/load_results/load_results_panel.py +++ b/src/ert/gui/tools/load_results/load_results_panel.py @@ -65,7 +65,7 @@ def __init__(self, facade: LibresFacade, notifier: ErtNotifier): def readCurrentRunPath(self) -> str: current_ensemble = self._notifier.current_ensemble_name - run_path = self._facade.run_path + run_path = self._facade.resolved_run_path run_path = run_path.replace("", current_ensemble) run_path = run_path.replace("", current_ensemble) return run_path @@ -103,3 +103,7 @@ def load(self) -> int: msg = ErtMessageBox("No realizations loaded", "\n".join(messages)) msg.exec_() return loaded + + def refresh(self) -> None: + self._run_path_text.setText(self.readCurrentRunPath()) + self._run_path_text.refresh() From cf9e39b8c300cb0a119dd171366bb8da5f756cd9 Mon Sep 17 00:00:00 2001 From: DanSava Date: Mon, 28 Oct 2024 11:40:38 +0200 Subject: [PATCH 07/11] Rename function name load_realization -> load_run_path_realization --- src/ert/callbacks.py | 2 +- src/ert/libres_facade.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ert/callbacks.py b/src/ert/callbacks.py index 141a941de3f..8c14abcce5c 100644 --- a/src/ert/callbacks.py +++ b/src/ert/callbacks.py @@ -136,7 +136,7 @@ async def forward_model_ok( return final_result -async def load_realization( +async def load_run_path_realization( run_path: str, realization: int, ensemble: Ensemble, diff --git a/src/ert/libres_facade.py b/src/ert/libres_facade.py index 40f9e4577ed..551ff5187ea 100644 --- a/src/ert/libres_facade.py +++ b/src/ert/libres_facade.py @@ -21,7 +21,7 @@ from pandas import DataFrame from ert.analysis import AnalysisEvent, SmootherSnapshot, smoother_update -from ert.callbacks import load_realization +from ert.callbacks import load_run_path_realization from ert.config import ( EnkfObservationImplementationType, ErtConfig, @@ -50,7 +50,7 @@ def _load_realization_from_run_path( realization: int, ensemble: Ensemble, ) -> Tuple[LoadResult, int]: - result = asyncio.run(load_realization(run_path, realization, ensemble)) + result = asyncio.run(load_run_path_realization(run_path, realization, ensemble)) return result, realization From b5708bbdcd0e960f18d278ff0d89c54a53c7cc8d Mon Sep 17 00:00:00 2001 From: DanSava Date: Mon, 28 Oct 2024 11:41:00 +0200 Subject: [PATCH 08/11] Cleanup comments in textbox --- src/ert/gui/ertwidgets/textbox.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/ert/gui/ertwidgets/textbox.py b/src/ert/gui/ertwidgets/textbox.py index 2ae01d1a483..b8b0837fb1e 100644 --- a/src/ert/gui/ertwidgets/textbox.py +++ b/src/ert/gui/ertwidgets/textbox.py @@ -14,8 +14,8 @@ class TextBox(QTextEdit): - """StringBox shows a string. The data structure expected and sent to the - getter and setter is a string.""" + """TextBox shows a multi line string. The data structure expected and sent to the + getter and setter is a multi line string.""" def __init__( self, @@ -24,11 +24,6 @@ def __init__( placeholder_text: str = "", minimum_width: int = 250, ): - """ - :type model: ert.gui.ertwidgets.models.valuemodel.ValueModel - :type help_link: str - :type default_string: str - """ QTextEdit.__init__(self) self.setMinimumWidth(minimum_width) self._validation = ValidationSupport(self) From 57d7bf1231ddec252ac0ed4f79ea15fd20e6f715 Mon Sep 17 00:00:00 2001 From: DanSava Date: Mon, 28 Oct 2024 11:41:27 +0200 Subject: [PATCH 09/11] Add testing for StringDefinition validator --- .../ide/test_string_definition_argument.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 tests/ert/unit_tests/gui/ide/test_string_definition_argument.py diff --git a/tests/ert/unit_tests/gui/ide/test_string_definition_argument.py b/tests/ert/unit_tests/gui/ide/test_string_definition_argument.py new file mode 100644 index 00000000000..109d5f76e3b --- /dev/null +++ b/tests/ert/unit_tests/gui/ide/test_string_definition_argument.py @@ -0,0 +1,29 @@ +from ert.validation import StringDefinition + + +def test_validate_success_with_all_required_tokens(): + string_def = StringDefinition(required=["token1", "token2"], invalid=["invalid1"]) + validation_status = string_def.validate("This is a string with token1 and token2") + assert bool(validation_status) is True + assert not validation_status.message() + + +def test_validate_success_with_required_tokens(): + string_def = StringDefinition(required=["token1", "token2"], invalid=["invalid1"]) + validation_status = string_def.validate("This is a string with token1 and token2") + assert bool(validation_status) is True + assert not validation_status.message() + + +def test_validate_failure_with_empty_required_tokens(): + string_def = StringDefinition(optional=False, required=[], invalid=["invalid1"]) + validation_status = string_def.validate("This is a string with invalid1") + assert bool(validation_status) is False + assert validation_status.message() == "Contains invalid string invalid1!" + + +def test_validate_empty_string(): + string_def = StringDefinition(required=["token1"], invalid=["invalid1"]) + validation_status = string_def.validate("") + assert bool(validation_status) is False + assert "Missing required token1!" in validation_status.message() From cbc105309783046c04f9daa582640cec19938e85 Mon Sep 17 00:00:00 2001 From: DanSava Date: Fri, 1 Nov 2024 11:38:41 +0200 Subject: [PATCH 10/11] Add helpful text in load_results_panel --- .../gui/tools/load_results/load_results_panel.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/ert/gui/tools/load_results/load_results_panel.py b/src/ert/gui/tools/load_results/load_results_panel.py index b99ff9a2077..a0cb44e0936 100644 --- a/src/ert/gui/tools/load_results/load_results_panel.py +++ b/src/ert/gui/tools/load_results/load_results_panel.py @@ -1,7 +1,7 @@ from __future__ import annotations from qtpy.QtCore import Qt, Signal -from qtpy.QtWidgets import QFormLayout, QMessageBox, QWidget +from qtpy.QtWidgets import QFormLayout, QLabel, QMessageBox, QWidget from ert.gui.ertnotifier import ErtNotifier from ert.gui.ertwidgets import ( @@ -42,9 +42,14 @@ def __init__(self, facade: LibresFacade, notifier: ErtNotifier): self._run_path_text.getValidationSupport().validationChanged.connect( self.panelConfigurationChanged ) + self._run_path_text.textChanged.connect(self.text_change) + self.help_iter_lbl = QLabel(" will be replace by: 0") + self.help_iens_lbl = QLabel(" will be replace by %") layout.addRow("Load data from run path: ", self._run_path_text) ensemble_selector = EnsembleSelector(self._notifier) + layout.addRow("", self.help_iens_lbl) + layout.addRow("", self.help_iter_lbl) layout.addRow("Load into ensemble:", ensemble_selector) self._ensemble_selector = ensemble_selector @@ -54,8 +59,12 @@ def __init__(self, facade: LibresFacade, notifier: ErtNotifier): self._active_realizations_model, # type: ignore "load_results_manually/Realizations", ) + self._active_realizations_field.textChanged.connect(self.text_change) self._active_realizations_field.setValidator(RangeStringArgument(ensemble_size)) self._active_realizations_field.setObjectName("active_realizations_lrm") + self.help_iens_lbl.setText( + f" will be replace by {self._active_realizations_field.get_text}" + ) layout.addRow("Realizations to load:", self._active_realizations_field) self._active_realizations_field.getValidationSupport().validationChanged.connect( @@ -63,6 +72,11 @@ def __init__(self, facade: LibresFacade, notifier: ErtNotifier): ) self.setLayout(layout) + def text_change(self) -> None: + active_realizations = self._active_realizations_field.get_text + self.help_iens_lbl.setText(f" will be replace by {active_realizations}") + self.help_iter_lbl.setVisible("" in self._run_path_text.get_text) + def readCurrentRunPath(self) -> str: current_ensemble = self._notifier.current_ensemble_name run_path = self._facade.resolved_run_path From af611f7f5b184476abfd9cc566c22cd7121b5e5d Mon Sep 17 00:00:00 2001 From: DanSava Date: Mon, 4 Nov 2024 10:23:56 +0200 Subject: [PATCH 11/11] Merge new functionality from load_run_path_realization into forward_model_ok --- src/ert/callbacks.py | 57 +++---------------- src/ert/libres_facade.py | 4 +- src/ert/scheduler/job.py | 7 ++- .../unit_tests/ensemble_evaluator/conftest.py | 2 +- tests/ert/unit_tests/scheduler/test_job.py | 2 +- .../ert/unit_tests/test_load_forward_model.py | 31 +++++----- 6 files changed, 32 insertions(+), 71 deletions(-) diff --git a/src/ert/callbacks.py b/src/ert/callbacks.py index 8c14abcce5c..8189dbfab3e 100644 --- a/src/ert/callbacks.py +++ b/src/ert/callbacks.py @@ -6,7 +6,6 @@ from pathlib import Path from ert.config import InvalidResponseFile -from ert.run_arg import RunArg from ert.storage import Ensemble from ert.storage.realization_storage_state import RealizationStorageState @@ -92,63 +91,23 @@ async def _write_responses_to_storage( async def forward_model_ok( - run_arg: RunArg, + run_path: str, + realization: int, + iter: int, + ensemble: Ensemble, ) -> LoadResult: parameters_result = LoadResult(LoadStatus.LOAD_SUCCESSFUL, "") response_result = LoadResult(LoadStatus.LOAD_SUCCESSFUL, "") try: # We only read parameters after the prior, after that, ERT # handles parameters - if run_arg.itr == 0: + if iter == 0: parameters_result = await _read_parameters( - run_arg.runpath, - run_arg.iens, - run_arg.ensemble_storage, - ) - - if parameters_result.status == LoadStatus.LOAD_SUCCESSFUL: - response_result = await _write_responses_to_storage( - run_arg.runpath, - run_arg.iens, - run_arg.ensemble_storage, + run_path, + realization, + ensemble, ) - except Exception as err: - logger.exception( - f"Failed to load results for realization {run_arg.iens}", - exc_info=err, - ) - parameters_result = LoadResult( - LoadStatus.LOAD_FAILURE, - "Failed to load results for realization " - f"{run_arg.iens}, failed with: {err}", - ) - - final_result = parameters_result - if response_result.status != LoadStatus.LOAD_SUCCESSFUL: - final_result = response_result - run_arg.ensemble_storage.set_failure( - run_arg.iens, RealizationStorageState.LOAD_FAILURE, final_result.message - ) - elif run_arg.ensemble_storage.has_failure(run_arg.iens): - run_arg.ensemble_storage.unset_failure(run_arg.iens) - - return final_result - - -async def load_run_path_realization( - run_path: str, - realization: int, - ensemble: Ensemble, -) -> LoadResult: - response_result = LoadResult(LoadStatus.LOAD_SUCCESSFUL, "") - try: - parameters_result = await _read_parameters( - run_path, - realization, - ensemble, - ) - if parameters_result.status == LoadStatus.LOAD_SUCCESSFUL: response_result = await _write_responses_to_storage( run_path, diff --git a/src/ert/libres_facade.py b/src/ert/libres_facade.py index 551ff5187ea..bc516f04455 100644 --- a/src/ert/libres_facade.py +++ b/src/ert/libres_facade.py @@ -21,7 +21,7 @@ from pandas import DataFrame from ert.analysis import AnalysisEvent, SmootherSnapshot, smoother_update -from ert.callbacks import load_run_path_realization +from ert.callbacks import forward_model_ok from ert.config import ( EnkfObservationImplementationType, ErtConfig, @@ -50,7 +50,7 @@ def _load_realization_from_run_path( realization: int, ensemble: Ensemble, ) -> Tuple[LoadResult, int]: - result = asyncio.run(load_run_path_realization(run_path, realization, ensemble)) + result = asyncio.run(forward_model_ok(run_path, realization, 0, ensemble)) return result, realization diff --git a/src/ert/scheduler/job.py b/src/ert/scheduler/job.py index a3055f83d1a..cba6e9144ca 100644 --- a/src/ert/scheduler/job.py +++ b/src/ert/scheduler/job.py @@ -240,7 +240,12 @@ async def _verify_checksum( logger.error(f"Disk synchronization failed for {file_path}") async def _handle_finished_forward_model(self) -> None: - callback_status, status_msg = await forward_model_ok(self.real.run_arg) + callback_status, status_msg = await forward_model_ok( + run_path=self.real.run_arg.runpath, + realization=self.real.run_arg.iens, + iter=self.real.run_arg.itr, + ensemble=self.real.run_arg.ensemble_storage, + ) if self._message: self._message = status_msg else: diff --git a/tests/ert/unit_tests/ensemble_evaluator/conftest.py b/tests/ert/unit_tests/ensemble_evaluator/conftest.py index e996d8a299b..eda4a55b27a 100644 --- a/tests/ert/unit_tests/ensemble_evaluator/conftest.py +++ b/tests/ert/unit_tests/ensemble_evaluator/conftest.py @@ -64,7 +64,7 @@ def queue_config_fixture(): @pytest.fixture def make_ensemble(queue_config): def _make_ensemble_builder(monkeypatch, tmpdir, num_reals, num_jobs, job_sleep=0): - async def load_successful(_): + async def load_successful(**_): return (LoadStatus.LOAD_SUCCESSFUL, "") monkeypatch.setattr(ert.scheduler.job, "forward_model_ok", load_successful) diff --git a/tests/ert/unit_tests/scheduler/test_job.py b/tests/ert/unit_tests/scheduler/test_job.py index 775e68a0317..ef3e0307394 100644 --- a/tests/ert/unit_tests/scheduler/test_job.py +++ b/tests/ert/unit_tests/scheduler/test_job.py @@ -119,7 +119,7 @@ async def test_job_run_sends_expected_events( realization: Realization, monkeypatch, ): - async def load_result(_): + async def load_result(**_): return (forward_model_ok_result, "") monkeypatch.setattr(ert.scheduler.job, "forward_model_ok", load_result) diff --git a/tests/ert/unit_tests/test_load_forward_model.py b/tests/ert/unit_tests/test_load_forward_model.py index a08322057dd..ba6a5eaa599 100644 --- a/tests/ert/unit_tests/test_load_forward_model.py +++ b/tests/ert/unit_tests/test_load_forward_model.py @@ -11,7 +11,6 @@ from ert.config import ErtConfig from ert.enkf_main import create_run_path from ert.libres_facade import LibresFacade -from ert.run_arg import create_run_arguments from ert.storage import open_storage @@ -290,9 +289,9 @@ def test_that_the_states_are_set_correctly(): assert new_ensemble.has_data() -@pytest.mark.parametrize("iter", [None, 0, 1, 2, 3]) +@pytest.mark.parametrize("itr", [None, 0, 1, 2, 3]) @pytest.mark.usefixtures("use_tmpdir") -def test_loading_from_any_available_iter(storage, run_paths, run_args, iter): +def test_loading_from_any_available_iter(storage, run_paths, run_args, itr): config_text = dedent( """ NUM_REALIZATIONS 1 @@ -308,23 +307,21 @@ def test_loading_from_any_available_iter(storage, run_paths, run_args, iter): ), name="prior", ensemble_size=ert_config.model_config.num_realizations, - iteration=iter if iter is not None else 0, + iteration=itr if itr is not None else 0, ) - run_args = create_run_arguments( - run_paths(ert_config), - [True] * ert_config.model_config.num_realizations, - prior_ensemble, - ) create_run_path( - run_args, - prior_ensemble, - ert_config, - run_paths(ert_config), - ) - run_path = Path( - f"simulations/realization-0/iter-{iter if iter is not None else 0}/" + run_args=run_args(ert_config, prior_ensemble), + ensemble=prior_ensemble, + user_config_file=ert_config.user_config_file, + env_vars=ert_config.env_vars, + forward_model_steps=ert_config.forward_model_steps, + substitutions=ert_config.substitutions, + templates=ert_config.ert_templates, + model_config=ert_config.model_config, + runpaths=run_paths(ert_config), ) + run_path = Path(f"simulations/realization-0/iter-{itr if itr is not None else 0}/") with open(run_path / "response.out", "w", encoding="utf-8") as fout: fout.write("\n".join(["1", "2", "3"])) with open(run_path / "response.out_active", "w", encoding="utf-8") as fout: @@ -333,7 +330,7 @@ def test_loading_from_any_available_iter(storage, run_paths, run_args, iter): facade = LibresFacade.from_config_file("config.ert") run_path_format = str( Path( - f"simulations/realization-/iter-{iter if iter is not None else 0}" + f"simulations/realization-/iter-{itr if itr is not None else 0}" ).resolve() ) facade.load_from_run_path(run_path_format, prior_ensemble, [0])