diff --git a/src/ert/cli/workflow.py b/src/ert/cli/workflow.py index 8cdb93e8a4c..c41806dda56 100644 --- a/src/ert/cli/workflow.py +++ b/src/ert/cli/workflow.py @@ -20,7 +20,7 @@ def execute_workflow( msg = "Workflow {} is not in the list of available workflows" logger.error(msg.format(workflow_name)) return - runner = WorkflowRunner(workflow, storage, ert_config=ert_config) + runner = WorkflowRunner(workflow=workflow, storage=storage, ert_config=ert_config) runner.run_blocking() if not all(v["completed"] for v in runner.workflowReport().values()): logger.error(f"Workflow {workflow_name} failed!") diff --git a/src/ert/config/ert_config.py b/src/ert/config/ert_config.py index 99f012041f4..4c3241be5cb 100644 --- a/src/ert/config/ert_config.py +++ b/src/ert/config/ert_config.py @@ -83,6 +83,164 @@ def site_config_location() -> str: ) +def create_forward_model_json( + context: Substitutions, + forward_model_steps: List[ForwardModelStep], + run_id: Optional[str], + iens: int = 0, + itr: int = 0, + user_config_file: Optional[str] = "", + env_vars: Optional[Dict[str, str]] = None, + skip_pre_experiment_validation: bool = False, +) -> Dict[str, Any]: + if env_vars is None: + env_vars = {} + + class Substituter: + def __init__(self, fm_step): + fm_step_args = ",".join( + [f"{key}={value}" for key, value in fm_step.private_args.items()] + ) + fm_step_description = f"{fm_step.name}({fm_step_args})" + self.substitution_context_hint = ( + f"parsing forward model step `FORWARD_MODEL {fm_step_description}` - " + "reconstructed, with defines applied during parsing" + ) + self.copy_private_args = Substitutions() + for key, val in fm_step.private_args.items(): + self.copy_private_args[key] = context.substitute_real_iter( + val, iens, itr + ) + + @overload + def substitute(self, string: str) -> str: ... + + @overload + def substitute(self, string: None) -> None: ... + + def substitute(self, string): + if string is None: + return string + string = self.copy_private_args.substitute( + string, self.substitution_context_hint, 1, warn_max_iter=False + ) + return context.substitute_real_iter(string, iens, itr) + + def filter_env_dict(self, d): + result = {} + for key, value in d.items(): + new_key = self.substitute(key) + new_value = self.substitute(value) + if new_value is None: + result[new_key] = None + elif not (new_value[0] == "<" and new_value[-1] == ">"): + # Remove values containing "". These are expected to be + # replaced by substitute, but were not. + result[new_key] = new_value + else: + logger.warning( + f"Environment variable {new_key} skipped due to" + f" unmatched define {new_value}", + ) + # Its expected that empty dicts be replaced with "null" + # in jobs.json + if not result: + return None + return result + + def handle_default(fm_step: ForwardModelStep, arg: str) -> str: + return fm_step.default_mapping.get(arg, arg) + + for fm_step in forward_model_steps: + for key, val in fm_step.private_args.items(): + if key in context and key != val and context[key] != val: + logger.info( + f"Private arg '{key}':'{val}' chosen over" + f" global '{context[key]}' in forward model step {fm_step.name}" + ) + config_file_path = Path(user_config_file) if user_config_file is not None else None + config_path = str(config_file_path.parent) if config_file_path else "" + config_file = str(config_file_path.name) if config_file_path else "" + + job_list_errors = [] + job_list: List[ForwardModelStepJSON] = [] + for idx, fm_step in enumerate(forward_model_steps): + substituter = Substituter(fm_step) + fm_step_json = { + "name": substituter.substitute(fm_step.name), + "executable": substituter.substitute(fm_step.executable), + "target_file": substituter.substitute(fm_step.target_file), + "error_file": substituter.substitute(fm_step.error_file), + "start_file": substituter.substitute(fm_step.start_file), + "stdout": ( + substituter.substitute(fm_step.stdout_file) + f".{idx}" + if fm_step.stdout_file + else None + ), + "stderr": ( + substituter.substitute(fm_step.stderr_file) + f".{idx}" + if fm_step.stderr_file + else None + ), + "stdin": substituter.substitute(fm_step.stdin_file), + "argList": [ + handle_default(fm_step, substituter.substitute(arg)) + for arg in fm_step.arglist + ], + "environment": substituter.filter_env_dict(fm_step.environment), + "exec_env": substituter.filter_env_dict(fm_step.exec_env), + "max_running_minutes": fm_step.max_running_minutes, + } + + try: + if not skip_pre_experiment_validation: + fm_step_json = fm_step.validate_pre_realization_run(fm_step_json) + except ForwardModelStepValidationError as exc: + job_list_errors.append( + ErrorInfo( + message=f"Validation failed for " + f"forward model step {fm_step.name}: {exc!s}" + ).set_context(fm_step.name) + ) + + job_list.append(fm_step_json) + + if job_list_errors: + raise ConfigValidationError.from_collected(job_list_errors) + + return { + "global_environment": env_vars, + "config_path": config_path, + "config_file": config_file, + "jobList": job_list, + "run_id": run_id, + "ert_pid": str(os.getpid()), + } + + +def forward_model_data_to_json( + substitutions: Substitutions, + forward_model_steps: List[ForwardModelStep], + env_vars: Dict[str, str], + user_config_file: Optional[str] = "", + run_id: Optional[str] = None, + iens: int = 0, + itr: int = 0, + context_env: Optional[Dict[str, str]] = None, +): + if context_env is None: + context_env = {} + return create_forward_model_json( + context=substitutions, + forward_model_steps=forward_model_steps, + user_config_file=user_config_file, + env_vars={**env_vars, **context_env}, + run_id=run_id, + iens=iens, + itr=itr, + ) + + @dataclass class ErtConfig: DEFAULT_ENSPATH: ClassVar[str] = "storage" @@ -615,7 +773,7 @@ def _create_list_of_forward_model_steps_to_run( for fm_step in fm_steps: if fm_step.name in cls.PREINSTALLED_FORWARD_MODEL_STEPS: try: - substituted_json = cls._create_forward_model_json( + substituted_json = create_forward_model_json( run_id=None, context=substitutions, forward_model_steps=[fm_step], @@ -644,163 +802,6 @@ def _create_list_of_forward_model_steps_to_run( def forward_model_step_name_list(self) -> List[str]: return [j.name for j in self.forward_model_steps] - def forward_model_data_to_json( - self, - run_id: Optional[str] = None, - iens: int = 0, - itr: int = 0, - context_env: Optional[Dict[str, str]] = None, - ): - if context_env is None: - context_env = {} - return self._create_forward_model_json( - context=self.substitutions, - forward_model_steps=self.forward_model_steps, - user_config_file=self.user_config_file, - env_vars={**self.env_vars, **context_env}, - run_id=run_id, - iens=iens, - itr=itr, - ) - - @classmethod - def _create_forward_model_json( - cls, - context: Substitutions, - forward_model_steps: List[ForwardModelStep], - run_id: Optional[str], - iens: int = 0, - itr: int = 0, - user_config_file: Optional[str] = "", - env_vars: Optional[Dict[str, str]] = None, - skip_pre_experiment_validation: bool = False, - ) -> Dict[str, Any]: - if env_vars is None: - env_vars = {} - - class Substituter: - def __init__(self, fm_step): - fm_step_args = ",".join( - [f"{key}={value}" for key, value in fm_step.private_args.items()] - ) - fm_step_description = f"{fm_step.name}({fm_step_args})" - self.substitution_context_hint = ( - f"parsing forward model step `FORWARD_MODEL {fm_step_description}` - " - "reconstructed, with defines applied during parsing" - ) - self.copy_private_args = Substitutions() - for key, val in fm_step.private_args.items(): - self.copy_private_args[key] = context.substitute_real_iter( - val, iens, itr - ) - - @overload - def substitute(self, string: str) -> str: ... - - @overload - def substitute(self, string: None) -> None: ... - - def substitute(self, string): - if string is None: - return string - string = self.copy_private_args.substitute( - string, self.substitution_context_hint, 1, warn_max_iter=False - ) - return context.substitute_real_iter(string, iens, itr) - - def filter_env_dict(self, d): - result = {} - for key, value in d.items(): - new_key = self.substitute(key) - new_value = self.substitute(value) - if new_value is None: - result[new_key] = None - elif not (new_value[0] == "<" and new_value[-1] == ">"): - # Remove values containing "". These are expected to be - # replaced by substitute, but were not. - result[new_key] = new_value - else: - logger.warning( - f"Environment variable {new_key} skipped due to" - f" unmatched define {new_value}", - ) - # Its expected that empty dicts be replaced with "null" - # in jobs.json - if not result: - return None - return result - - def handle_default(fm_step: ForwardModelStep, arg: str) -> str: - return fm_step.default_mapping.get(arg, arg) - - for fm_step in forward_model_steps: - for key, val in fm_step.private_args.items(): - if key in context and key != val and context[key] != val: - logger.info( - f"Private arg '{key}':'{val}' chosen over" - f" global '{context[key]}' in forward model step {fm_step.name}" - ) - config_file_path = ( - Path(user_config_file) if user_config_file is not None else None - ) - config_path = str(config_file_path.parent) if config_file_path else "" - config_file = str(config_file_path.name) if config_file_path else "" - - job_list_errors = [] - job_list: List[ForwardModelStepJSON] = [] - for idx, fm_step in enumerate(forward_model_steps): - substituter = Substituter(fm_step) - fm_step_json = { - "name": substituter.substitute(fm_step.name), - "executable": substituter.substitute(fm_step.executable), - "target_file": substituter.substitute(fm_step.target_file), - "error_file": substituter.substitute(fm_step.error_file), - "start_file": substituter.substitute(fm_step.start_file), - "stdout": ( - substituter.substitute(fm_step.stdout_file) + f".{idx}" - if fm_step.stdout_file - else None - ), - "stderr": ( - substituter.substitute(fm_step.stderr_file) + f".{idx}" - if fm_step.stderr_file - else None - ), - "stdin": substituter.substitute(fm_step.stdin_file), - "argList": [ - handle_default(fm_step, substituter.substitute(arg)) - for arg in fm_step.arglist - ], - "environment": substituter.filter_env_dict(fm_step.environment), - "exec_env": substituter.filter_env_dict(fm_step.exec_env), - "max_running_minutes": fm_step.max_running_minutes, - } - - try: - if not skip_pre_experiment_validation: - fm_step_json = fm_step.validate_pre_realization_run(fm_step_json) - except ForwardModelStepValidationError as exc: - job_list_errors.append( - ErrorInfo( - message=f"Validation failed for " - f"forward model step {fm_step.name}: {exc!s}" - ).set_context(fm_step.name) - ) - - job_list.append(fm_step_json) - - if job_list_errors: - raise ConfigValidationError.from_collected(job_list_errors) - - return { - "global_environment": env_vars, - "config_path": config_path, - "config_file": config_file, - "jobList": job_list, - "run_id": run_id, - "ert_pid": str(os.getpid()), - } - @classmethod def _workflows_from_dict( cls, diff --git a/src/ert/enkf_main.py b/src/ert/enkf_main.py index c2fd4d3bbcf..5e776a548d9 100644 --- a/src/ert/enkf_main.py +++ b/src/ert/enkf_main.py @@ -6,11 +6,26 @@ import time from datetime import datetime from pathlib import Path -from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Mapping, Optional, Union +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Iterable, + List, + Mapping, + Optional, + Tuple, + Union, +) import orjson from numpy.random import SeedSequence +from ert.config.ert_config import forward_model_data_to_json +from ert.config.forward_model_step import ForwardModelStep +from ert.config.model_config import ModelConfig +from ert.substitutions import Substitutions + from .config import ( ExtParamConfig, Field, @@ -22,7 +37,6 @@ from .runpaths import Runpaths if TYPE_CHECKING: - from .config import ErtConfig from .storage import Ensemble logger = logging.getLogger(__name__) @@ -183,20 +197,24 @@ def sample_prior( def create_run_path( run_args: List[RunArg], ensemble: Ensemble, - ert_config: ErtConfig, + user_config_file: str, + env_vars: Dict[str, str], + forward_model_steps: List[ForwardModelStep], + substitutions: Substitutions, + templates: List[Tuple[str, str]], + model_config: ModelConfig, runpaths: Runpaths, context_env: Optional[Dict[str, str]] = None, ) -> None: if context_env is None: context_env = {} t = time.perf_counter() - substitutions = ert_config.substitutions runpaths.set_ert_ensemble(ensemble.name) for run_arg in run_args: run_path = Path(run_arg.runpath) if run_arg.active: run_path.mkdir(parents=True, exist_ok=True) - for source_file, target_file in ert_config.ert_templates: + for source_file, target_file in templates: target_file = substitutions.substitute_real_iter( target_file, run_arg.iens, ensemble.iteration ) @@ -220,7 +238,6 @@ def create_run_path( ) target.write_text(result) - model_config = ert_config.model_config _generate_parameter_files( ensemble.experiment.parameter_configuration.values(), model_config.gen_kw_export_name, @@ -232,8 +249,16 @@ def create_run_path( path = run_path / "jobs.json" _backup_if_existing(path) - forward_model_output = ert_config.forward_model_data_to_json( - run_arg.run_id, run_arg.iens, ensemble.iteration, context_env + + forward_model_output = forward_model_data_to_json( + substitutions=substitutions, + forward_model_steps=forward_model_steps, + user_config_file=user_config_file, + env_vars=env_vars, + run_id=run_arg.run_id, + iens=run_arg.iens, + itr=ensemble.iteration, + context_env=context_env, ) with open(run_path / "jobs.json", mode="wb") as fptr: fptr.write( diff --git a/src/ert/gui/tools/workflows/run_workflow_widget.py b/src/ert/gui/tools/workflows/run_workflow_widget.py index c64f5e94143..a42c0936624 100644 --- a/src/ert/gui/tools/workflows/run_workflow_widget.py +++ b/src/ert/gui/tools/workflows/run_workflow_widget.py @@ -125,7 +125,6 @@ def startWorkflow(self) -> None: workflow, storage=self.storage, ensemble=self.source_ensemble_selector.currentData(), - ert_config=self.config, ) self._workflow_runner.run() diff --git a/src/ert/run_models/base_run_model.py b/src/ert/run_models/base_run_model.py index 5e1dfed32fc..b000d5fa491 100644 --- a/src/ert/run_models/base_run_model.py +++ b/src/ert/run_models/base_run_model.py @@ -662,9 +662,7 @@ def run_workflows( self, runtime: HookRuntime, storage: Storage, ensemble: Ensemble ) -> None: for workflow in self.ert_config.hooked_workflows[runtime]: - WorkflowRunner( - workflow, storage, ensemble, ert_config=self.ert_config - ).run_blocking() + WorkflowRunner(workflow, storage, ensemble).run_blocking() def _evaluate_and_postprocess( self, @@ -673,11 +671,16 @@ def _evaluate_and_postprocess( evaluator_server_config: EvaluatorServerConfig, ) -> int: create_run_path( - run_args, - ensemble, - self.ert_config, - self.run_paths, - self._context_env, + run_args=run_args, + ensemble=ensemble, + user_config_file=self.ert_config.user_config_file, + env_vars=self.ert_config.env_vars, + forward_model_steps=self.ert_config.forward_model_steps, + substitutions=self.ert_config.substitutions, + templates=self.ert_config.ert_templates, + model_config=self.ert_config.model_config, + runpaths=self.run_paths, + context_env=self._context_env, ) self.run_workflows(HookRuntime.PRE_SIMULATION, self._storage, ensemble) diff --git a/src/ert/simulator/batch_simulator.py b/src/ert/simulator/batch_simulator.py index 1ac749f24a4..517e7232267 100644 --- a/src/ert/simulator/batch_simulator.py +++ b/src/ert/simulator/batch_simulator.py @@ -14,7 +14,15 @@ import numpy as np -from ert.config import ErtConfig, ExtParamConfig +from ert.config import ExtParamConfig +from ert.config.analysis_config import AnalysisConfig +from ert.config.forward_model_step import ForwardModelStep +from ert.config.model_config import ModelConfig +from ert.config.parameter_config import ParameterConfig +from ert.config.parsing.hook_runtime import HookRuntime +from ert.config.queue_config import QueueConfig +from ert.config.workflow import Workflow +from ert.substitutions import Substitutions from .batch_simulator_context import BatchContext @@ -25,7 +33,18 @@ class BatchSimulator: def __init__( self, - ert_config: ErtConfig, + perferred_num_cpu: int, + runpath_file: str, + user_config_file: str, + env_vars: Dict[str, str], + forward_model_steps: List[ForwardModelStep], + parameter_configurations: Dict[str, ParameterConfig], + queue_config: QueueConfig, + model_config: ModelConfig, + analysis_config: AnalysisConfig, + hooked_workflows: Dict[HookRuntime, List[Workflow]], + substitutions: Substitutions, + templates: List[Tuple[str, str]], experiment: Experiment, controls: Iterable[str], results: Iterable[str], @@ -95,10 +114,18 @@ def callback(*args, **kwargs): .... """ - if not isinstance(ert_config, ErtConfig): - raise ValueError("The first argument must be valid ErtConfig instance") - - self.ert_config = ert_config + self.preferred_num_cpu = perferred_num_cpu + self.user_config_file = user_config_file + self.env_vars = env_vars + self.forward_model_steps = forward_model_steps + self.runpath_file = runpath_file + self.queue_config = queue_config + self.model_config = model_config + self.analysis_config = analysis_config + self.hooked_workflows = hooked_workflows + self.substitutions = substitutions + self.templates = templates + self.parameter_configurations = parameter_configurations self.experiment = experiment self.control_keys = set(controls) self.result_keys = set(results) @@ -143,7 +170,10 @@ def _check_suffix( raise KeyError(err_msg) for control_name, control in controls.items(): - ext_config = self.ert_config.ensemble_config[control_name] + ext_config = self.parameter_configurations[control_name] + + # fix this + if isinstance(ext_config, ExtParamConfig): if len(ext_config) != len(control.keys()): raise KeyError( @@ -233,7 +263,22 @@ def start( itr = 0 mask = np.full(len(case_data), True, dtype=bool) sim_context = BatchContext( - self.result_keys, self.ert_config, ensemble, mask, itr, case_data + result_keys=self.result_keys, + ensemble=ensemble, + preferred_num_cpu=self.preferred_num_cpu, + user_config_file=self.user_config_file, + env_vars=self.env_vars, + forward_model_steps=self.forward_model_steps, + runpath_file=self.runpath_file, + queue_config=self.queue_config, + model_config=self.model_config, + analysis_config=self.analysis_config, + hooked_workflows=self.hooked_workflows, + substitutions=self.substitutions, + templates=self.templates, + mask=mask, + itr=itr, + case_data=case_data, ) if self.callback: diff --git a/src/ert/simulator/batch_simulator_context.py b/src/ert/simulator/batch_simulator_context.py index ccfea63d82f..23147381836 100644 --- a/src/ert/simulator/batch_simulator_context.py +++ b/src/ert/simulator/batch_simulator_context.py @@ -14,10 +14,16 @@ from _ert.threading import ErtThread from ert.config import HookRuntime +from ert.config.analysis_config import AnalysisConfig +from ert.config.forward_model_step import ForwardModelStep +from ert.config.model_config import ModelConfig +from ert.config.queue_config import QueueConfig +from ert.config.workflow import Workflow from ert.enkf_main import create_run_path from ert.ensemble_evaluator import Realization from ert.runpaths import Runpaths from ert.scheduler import JobState, Scheduler, create_driver +from ert.substitutions import Substitutions from ert.workflow_runner import WorkflowRunner from ..run_arg import RunArg, create_run_arguments @@ -28,7 +34,6 @@ import numpy.typing as npt - from ert.config import ErtConfig from ert.storage import Ensemble logger = logging.getLogger(__name__) @@ -71,20 +76,28 @@ def _slug(entity: str) -> str: def _run_forward_model( - ert_config: "ErtConfig", + prefered_num_cpu: int, + queue_config: QueueConfig, + analysis_config: AnalysisConfig, scheduler: Scheduler, run_args: List[RunArg], ) -> None: # run simplestep - asyncio.run(_submit_and_run_jobqueue(ert_config, scheduler, run_args)) + asyncio.run( + _submit_and_run_jobqueue( + prefered_num_cpu, queue_config, analysis_config, scheduler, run_args + ) + ) async def _submit_and_run_jobqueue( - ert_config: "ErtConfig", + preferred_num_cpu: int, + queue_config: QueueConfig, + analysis_config: AnalysisConfig, scheduler: Scheduler, run_args: List[RunArg], ) -> None: - max_runtime: Optional[int] = ert_config.analysis_config.max_runtime + max_runtime: Optional[int] = analysis_config.max_runtime if max_runtime == 0: max_runtime = None for run_arg in run_args: @@ -96,15 +109,15 @@ async def _submit_and_run_jobqueue( active=True, max_runtime=max_runtime, run_arg=run_arg, - num_cpu=ert_config.preferred_num_cpu, - job_script=ert_config.queue_config.job_script, - realization_memory=ert_config.queue_config.realization_memory, + num_cpu=preferred_num_cpu, + job_script=queue_config.job_script, + realization_memory=queue_config.realization_memory, ) scheduler.set_realization(realization) required_realizations = 0 - if ert_config.queue_config.stop_long_running: - required_realizations = ert_config.analysis_config.minimum_required_realizations + if queue_config.stop_long_running: + required_realizations = analysis_config.minimum_required_realizations with contextlib.suppress(asyncio.CancelledError): await scheduler.execute(required_realizations) @@ -112,7 +125,17 @@ async def _submit_and_run_jobqueue( @dataclass class BatchContext: result_keys: "Iterable[str]" - ert_config: "ErtConfig" + preferred_num_cpu: int + queue_config: QueueConfig + model_config: ModelConfig + analysis_config: AnalysisConfig + hooked_workflows: Dict[HookRuntime, List[Workflow]] + substitutions: Substitutions + templates: List[Tuple[str, str]] + user_config_file: str + env_vars: Dict[str, str] + forward_model_steps: List[ForwardModelStep] + runpath_file: str ensemble: Ensemble mask: npt.NDArray[np.bool_] itr: int @@ -122,24 +145,21 @@ def __post_init__(self) -> None: """ Handle which can be used to query status and results for batch simulation. """ - ert_config = self.ert_config - driver = create_driver(ert_config.queue_config) - self._scheduler = Scheduler( - driver, max_running=self.ert_config.queue_config.max_running - ) + driver = create_driver(self.queue_config) + self._scheduler = Scheduler(driver, max_running=self.queue_config.max_running) + # fill in the missing geo_id data - global_substitutions = self.ert_config.substitutions - global_substitutions[""] = _slug(self.ensemble.name) + self.substitutions[""] = _slug(self.ensemble.name) for sim_id, (geo_id, _) in enumerate(self.case_data): if self.mask[sim_id]: - global_substitutions[f""] = str(geo_id) + self.substitutions[f""] = str(geo_id) run_paths = Runpaths( - jobname_format=ert_config.model_config.jobname_format_string, - runpath_format=ert_config.model_config.runpath_format_string, - filename=str(ert_config.runpath_file), - substitutions=global_substitutions, - eclbase=ert_config.model_config.eclbase_format_string, + jobname_format=self.model_config.jobname_format_string, + runpath_format=self.model_config.runpath_format_string, + filename=str(self.runpath_file), + substitutions=self.substitutions, + eclbase=self.model_config.eclbase_format_string, ) self.run_args = create_run_arguments( run_paths, @@ -152,13 +172,18 @@ def __post_init__(self) -> None: "_ERT_SIMULATION_MODE": "batch_simulation", } create_run_path( - self.run_args, - self.ensemble, - ert_config, - run_paths, - context_env, + run_args=self.run_args, + ensemble=self.ensemble, + user_config_file=self.user_config_file, + env_vars=self.env_vars, + forward_model_steps=self.forward_model_steps, + substitutions=self.substitutions, + templates=self.templates, + model_config=self.model_config, + runpaths=run_paths, + context_env=context_env, ) - for workflow in ert_config.hooked_workflows[HookRuntime.PRE_SIMULATION]: + for workflow in self.hooked_workflows[HookRuntime.PRE_SIMULATION]: WorkflowRunner(workflow, None, self.ensemble).run_blocking() self._sim_thread = self._run_simulations_simple_step() @@ -176,7 +201,11 @@ def get_ensemble(self) -> Ensemble: def _run_simulations_simple_step(self) -> Thread: sim_thread = ErtThread( target=lambda: _run_forward_model( - self.ert_config, self._scheduler, self.run_args + prefered_num_cpu=self.preferred_num_cpu, + queue_config=self.queue_config, + analysis_config=self.analysis_config, + scheduler=self._scheduler, + run_args=self.run_args, ) ) sim_thread.start() diff --git a/src/ert/workflow_runner.py b/src/ert/workflow_runner.py index 900b07e5146..4e0ac43b76d 100644 --- a/src/ert/workflow_runner.py +++ b/src/ert/workflow_runner.py @@ -7,7 +7,8 @@ from typing_extensions import Self -from ert.config import ErtConfig, ErtScript, ExternalErtScript, Workflow, WorkflowJob +from ert.config import ErtScript, ExternalErtScript, Workflow, WorkflowJob +from ert.config.ert_config import ErtConfig if TYPE_CHECKING: from ert.storage import Ensemble, Storage diff --git a/src/everest/detached/__init__.py b/src/everest/detached/__init__.py index ff2d027618d..d6bed829a38 100644 --- a/src/everest/detached/__init__.py +++ b/src/everest/detached/__init__.py @@ -99,7 +99,24 @@ def start_server(config: EverestConfig, ert_config: ErtConfig, storage): responses=[], ) - _server = BatchSimulator(ert_config, experiment, {}, []) + _server = BatchSimulator( + experiment=experiment, + perferred_num_cpu=ert_config.preferred_num_cpu, + runpath_file=str(ert_config.runpath_file), + user_config_file=ert_config.user_config_file, + env_vars=ert_config.env_vars, + forward_model_steps=ert_config.forward_model_steps, + parameter_configurations=ert_config.ensemble_config.parameter_configs, + queue_config=ert_config.queue_config, + model_config=ert_config.model_config, + analysis_config=ert_config.analysis_config, + hooked_workflows=ert_config.hooked_workflows, + substitutions=ert_config.substitutions, + templates=ert_config.ert_templates, + controls={}, + results=[], + ) + _context = _server.start("dispatch_server", [(0, {})]) return _context diff --git a/src/everest/simulator/simulator.py b/src/everest/simulator/simulator.py index 52aac5486c2..84179f8cfb4 100644 --- a/src/everest/simulator/simulator.py +++ b/src/everest/simulator/simulator.py @@ -32,10 +32,21 @@ def __init__( ) super(Simulator, self).__init__( - ert_config, - experiment, - self._get_controls(ever_config), - self._get_results(ever_config), + experiment=experiment, + perferred_num_cpu=ert_config.preferred_num_cpu, + user_config_file=ert_config.user_config_file, + env_vars=ert_config.env_vars, + forward_model_steps=ert_config.forward_model_steps, + runpath_file=str(ert_config.runpath_file), + parameter_configurations=ert_config.ensemble_config.parameter_configs, + queue_config=ert_config.queue_config, + model_config=ert_config.model_config, + analysis_config=ert_config.analysis_config, + hooked_workflows=ert_config.hooked_workflows, + substitutions=ert_config.substitutions, + templates=ert_config.ert_templates, + controls=self._get_controls(ever_config), + results=self._get_results(ever_config), callback=callback, ) @@ -124,9 +135,11 @@ def __call__( # Pre-simulation workflows are run by sim_context, but # post-stimulation workflows are not, do it here: ensemble = sim_context.get_ensemble() - for workflow in self.ert_config.hooked_workflows[HookRuntime.POST_SIMULATION]: + for workflow in self.hooked_workflows[HookRuntime.POST_SIMULATION]: WorkflowRunner( - workflow, self.storage, ensemble, ert_config=self.ert_config + workflow, + self.storage, + ensemble, ).run_blocking() for fnc_name, alias in self._function_aliases.items(): diff --git a/tests/ert/unit_tests/config/test_ert_config.py b/tests/ert/unit_tests/config/test_ert_config.py index 30a1aeaaa7f..93e5af10a2a 100644 --- a/tests/ert/unit_tests/config/test_ert_config.py +++ b/tests/ert/unit_tests/config/test_ert_config.py @@ -13,7 +13,10 @@ from hypothesis import strategies as st from ert.config import AnalysisConfig, ConfigValidationError, ErtConfig, HookRuntime -from ert.config.ert_config import site_config_location +from ert.config.ert_config import ( + forward_model_data_to_json, + site_config_location, +) from ert.config.parsing import ConfigKeys, ConfigWarning from ert.config.parsing.context_values import ( ContextBool, @@ -1265,8 +1268,17 @@ def test_validate_no_logs_when_overwriting_with_same_value(caplog): fout.write("FORWARD_MODEL step_name(=10, =, =5)\n") with caplog.at_level(logging.INFO): - ert_conf = ErtConfig.from_file("config_file.ert") - ert_conf.forward_model_data_to_json("0", "0", 0) + ert_config = ErtConfig.from_file("config_file.ert") + forward_model_data_to_json( + substitutions=ert_config.substitutions, + forward_model_steps=ert_config.forward_model_steps, + env_vars=ert_config.env_vars, + user_config_file=ert_config.user_config_file, + run_id="0", + iens="0", + itr=0, + ) + assert ( "Private arg '':'5' chosen over global '55' in forward model step step_name" in caplog.text diff --git a/tests/ert/unit_tests/config/test_forward_model.py b/tests/ert/unit_tests/config/test_forward_model.py index 3e2f05e1690..d230e714fad 100644 --- a/tests/ert/unit_tests/config/test_forward_model.py +++ b/tests/ert/unit_tests/config/test_forward_model.py @@ -11,7 +11,10 @@ from hypothesis import given, settings from ert.config import ConfigValidationError, ConfigWarning, ErtConfig -from ert.config.ert_config import _forward_model_step_from_config_file +from ert.config.ert_config import ( + _forward_model_step_from_config_file, + forward_model_data_to_json, +) from ert.config.forward_model_step import ( ForwardModelStepJSON, ForwardModelStepPlugin, @@ -498,7 +501,15 @@ def test_that_forward_model_substitution_does_not_warn_about_reaching_max_iterat ert_config = ErtConfig.with_plugins().from_file(test_config_file_name) with caplog.at_level(logging.WARNING): - ert_config.forward_model_data_to_json(0, 0, 0) + forward_model_data_to_json( + substitutions=ert_config.substitutions, + forward_model_steps=ert_config.forward_model_steps, + env_vars=ert_config.env_vars, + user_config_file=ert_config.user_config_file, + iens=0, + itr=0, + ) + assert "Reached max iterations" not in caplog.text @@ -666,11 +677,11 @@ def validate_pre_realization_run( ) -> ForwardModelStepJSON: return fm_step_json - config = ErtConfig.with_plugins( + ert_config = ErtConfig.with_plugins( forward_model_step_classes=[PluginForwardModel] ).from_file(tmp_path / "test.ert") - first_fm = config.forward_model_steps[0] + first_fm = ert_config.forward_model_steps[0] expected_attrs = { "name": "PluginForwardModel", @@ -708,7 +719,16 @@ def validate_pre_realization_run( getattr(first_fm, a) == v ), f"Expected fm[{a}] to be {v} but was {getattr(first_fm,a)}" - fm_json = config.forward_model_data_to_json("some_id", 0, 0) + fm_json = forward_model_data_to_json( + substitutions=ert_config.substitutions, + forward_model_steps=ert_config.forward_model_steps, + env_vars=ert_config.env_vars, + user_config_file=ert_config.user_config_file, + run_id="some_id", + iens=0, + itr=0, + ) + assert len(fm_json["jobList"]) == 1 job_from_joblist = fm_json["jobList"][0] assert job_from_joblist["name"] == "PluginForwardModel" @@ -743,18 +763,26 @@ def validate_pre_realization_run( return fm_json - config = ErtConfig.with_plugins(forward_model_step_classes=[FM]).from_file( + ert_config = ErtConfig.with_plugins(forward_model_step_classes=[FM]).from_file( tmp_path / "test.ert" ) - first_fm = config.forward_model_steps[0] + first_fm = ert_config.forward_model_steps[0] with pytest.raises(ForwardModelStepValidationError, match="Oh no"): first_fm.validate_pre_realization_run({"argList": ["not hello"]}) with pytest.raises( ConfigValidationError, match="Validation failed for forward model step" ): - _ = config.forward_model_data_to_json("id", 0, 0) + forward_model_data_to_json( + substitutions=ert_config.substitutions, + forward_model_steps=ert_config.forward_model_steps, + env_vars=ert_config.env_vars, + user_config_file=ert_config.user_config_file, + run_id="id", + iens=0, + itr=0, + ) def test_that_plugin_forward_model_validation_accepts_valid_args(tmp_path): @@ -782,14 +810,22 @@ def validate_pre_realization_run( return fm_json - config = ErtConfig.with_plugins(forward_model_step_classes=[FM]).from_file( + ert_config = ErtConfig.with_plugins(forward_model_step_classes=[FM]).from_file( tmp_path / "test.ert" ) - first_fm = config.forward_model_steps[0] + first_fm = ert_config.forward_model_steps[0] first_fm.validate_pre_realization_run({"argList": ["never"]}) - _ = config.forward_model_data_to_json("id", 0, 0) + forward_model_data_to_json( + substitutions=ert_config.substitutions, + forward_model_steps=ert_config.forward_model_steps, + env_vars=ert_config.env_vars, + user_config_file=ert_config.user_config_file, + run_id="id", + iens=0, + itr=0, + ) def test_that_plugin_forward_model_raises_pre_realization_validation_error(tmp_path): @@ -845,7 +881,15 @@ def validate_pre_realization_run( ConfigValidationError, match=".*This is a bad forward model step, dont use it.*", ): - config.forward_model_data_to_json("id", 0, 0) + forward_model_data_to_json( + substitutions=config.substitutions, + forward_model_steps=config.forward_model_steps, + env_vars=config.env_vars, + user_config_file=config.user_config_file, + run_id="id", + iens=0, + itr=0, + ) def test_that_plugin_forward_model_raises_pre_experiment_validation_error_early( diff --git a/tests/ert/unit_tests/config/test_forward_model_data_to_json.py b/tests/ert/unit_tests/config/test_forward_model_data_to_json.py index 6b35219cdcd..dafdd1626b4 100644 --- a/tests/ert/unit_tests/config/test_forward_model_data_to_json.py +++ b/tests/ert/unit_tests/config/test_forward_model_data_to_json.py @@ -11,7 +11,10 @@ import pytest from ert.config import ErtConfig, ForwardModelStep -from ert.config.ert_config import _forward_model_step_from_config_file +from ert.config.ert_config import ( + _forward_model_step_from_config_file, + forward_model_data_to_json, +) from ert.constant_filenames import JOBS_FILE from ert.simulator.forward_model_status import ForwardModelStatus from ert.substitutions import Substitutions @@ -292,12 +295,17 @@ def verify_json_dump(fm_steplist, config, selected_steps, run_id): def test_config_path_and_file(context): run_id = "test_config_path_and_file_in_jobs_json" - steps_json = ErtConfig( + ert_config = ErtConfig( forward_model_steps=set_up_forward_model([]), substitutions=context, user_config_file="path_to_config_file/config.ert", - ).forward_model_data_to_json( - run_id, + ) + steps_json = forward_model_data_to_json( + substitutions=ert_config.substitutions, + forward_model_steps=ert_config.forward_model_steps, + env_vars=ert_config.env_vars, + user_config_file=ert_config.user_config_file, + run_id=run_id, ) assert "config_path" in steps_json assert "config_file" in steps_json @@ -309,12 +317,18 @@ def test_config_path_and_file(context): def test_no_steps(context): run_id = "test_no_jobs_id" - data = ErtConfig( + ert_config = ErtConfig( forward_model_steps=set_up_forward_model([]), substitutions=context, user_config_file="path_to_config_file/config.ert", - ).forward_model_data_to_json( - run_id, + ) + + data = forward_model_data_to_json( + substitutions=ert_config.substitutions, + forward_model_steps=ert_config.forward_model_steps, + env_vars=ert_config.env_vars, + user_config_file=ert_config.user_config_file, + run_id=run_id, ) verify_json_dump([], data, [], run_id) @@ -325,19 +339,36 @@ def test_one_step(fm_step_list, context): for i, step in enumerate(fm_step_list): run_id = "test_one_job" - data = ErtConfig( + ert_config = ErtConfig( forward_model_steps=set_up_forward_model([step]), substitutions=context, - ).forward_model_data_to_json(run_id) + ) + + data = forward_model_data_to_json( + substitutions=ert_config.substitutions, + forward_model_steps=ert_config.forward_model_steps, + env_vars=ert_config.env_vars, + user_config_file=ert_config.user_config_file, + run_id=run_id, + ) + verify_json_dump(fm_step_list, data, [i], run_id) def run_all(fm_steplist, context): run_id = "run_all" - data = ErtConfig( + ert_config = ErtConfig( forward_model_steps=set_up_forward_model(fm_steplist), substitutions=context, - ).forward_model_data_to_json(run_id) + ) + + data = forward_model_data_to_json( + substitutions=ert_config.substitutions, + forward_model_steps=ert_config.forward_model_steps, + env_vars=ert_config.env_vars, + user_config_file=ert_config.user_config_file, + run_id=run_id, + ) verify_json_dump(fm_steplist, data, range(len(fm_steplist)), run_id) @@ -367,12 +398,20 @@ def test_various_null_fields(fm_step_list, context): @pytest.mark.usefixtures("use_tmpdir") def test_status_file(fm_step_list, context): run_id = "test_no_jobs_id" + ert_config = ErtConfig( + forward_model_steps=set_up_forward_model(fm_step_list), + substitutions=context, + ) + data = forward_model_data_to_json( + substitutions=ert_config.substitutions, + forward_model_steps=ert_config.forward_model_steps, + env_vars=ert_config.env_vars, + user_config_file=ert_config.user_config_file, + run_id=run_id, + ) with open(JOBS_FILE, "w", encoding="utf-8") as fp: json.dump( - ErtConfig( - forward_model_steps=set_up_forward_model(fm_step_list), - substitutions=context, - ).forward_model_data_to_json(run_id), + data, fp, ) @@ -409,9 +448,17 @@ def test_that_values_with_brackets_are_ommitted(caplog, fm_step_list, context): forward_model_list[0].environment["ENV_VAR"] = "" run_id = "test_no_jobs_id" - data = ErtConfig( + ert_config = ErtConfig( forward_model_steps=forward_model_list, substitutions=context - ).forward_model_data_to_json(run_id) + ) + + data = forward_model_data_to_json( + substitutions=ert_config.substitutions, + forward_model_steps=ert_config.forward_model_steps, + env_vars=ert_config.env_vars, + user_config_file=ert_config.user_config_file, + run_id=run_id, + ) assert "Environment variable ENV_VAR skipped due to" in caplog.text assert "ENV_VAR" not in data["jobList"][0]["environment"] @@ -562,15 +609,16 @@ def test_forward_model_job(job, forward_model, expected_args): ert_config = ErtConfig.from_file("config_file.ert") forward_model = ert_config.forward_model_steps - assert len(forward_model) == 1 - assert ( - ert_config.forward_model_data_to_json( - "", - 0, - 0, - )["jobList"][0]["argList"] - == expected_args + + data = forward_model_data_to_json( + substitutions=ert_config.substitutions, + forward_model_steps=ert_config.forward_model_steps, + env_vars=ert_config.env_vars, + user_config_file=ert_config.user_config_file, + run_id="", ) + assert len(forward_model) == 1 + assert data["jobList"][0]["argList"] == expected_args @pytest.mark.usefixtures("use_tmpdir") @@ -593,12 +641,14 @@ def test_that_config_path_is_the_directory_of_the_main_ert_config(): fout.write("FORWARD_MODEL job_name") ert_config = ErtConfig.from_file("config_file.ert") - - assert ert_config.forward_model_data_to_json( - "", - 0, - 0, - )["jobList"][0]["argList"] == [os.getcwd()] + data = forward_model_data_to_json( + substitutions=ert_config.substitutions, + forward_model_steps=ert_config.forward_model_steps, + env_vars=ert_config.env_vars, + user_config_file=ert_config.user_config_file, + run_id="", + ) + assert data["jobList"][0]["argList"] == [os.getcwd()] @pytest.mark.usefixtures("use_tmpdir") @@ -663,12 +713,16 @@ def test_simulation_job(job, forward_model, expected_args): fout.write(forward_model) ert_config = ErtConfig.from_file("config_file.ert") + data = forward_model_data_to_json( + substitutions=ert_config.substitutions, + forward_model_steps=ert_config.forward_model_steps, + env_vars=ert_config.env_vars, + user_config_file=ert_config.user_config_file, + run_id="", + ) + assert len(ert_config.forward_model_steps) == 1 - fm_data = ert_config.forward_model_data_to_json( - "", - 0, - 0, - )["jobList"][0] + fm_data = data["jobList"][0] assert fm_data["argList"] == expected_args @@ -694,7 +748,16 @@ def test_that_private_over_global_args_gives_logging_message(caplog): fout.write("FORWARD_MODEL job_name(=B)") ert_config = ErtConfig.from_file("config_file.ert") - fm_data = ert_config.forward_model_data_to_json("", 0, 0)["jobList"][0] + data = forward_model_data_to_json( + substitutions=ert_config.substitutions, + forward_model_steps=ert_config.forward_model_steps, + env_vars=ert_config.env_vars, + user_config_file=ert_config.user_config_file, + run_id="", + ) + + fm_data = data["jobList"][0] + assert len(ert_config.forward_model_steps) == 1 assert fm_data["argList"] == ["B"] assert "Private arg '':'B' chosen over global 'A'" in caplog.text @@ -724,8 +787,15 @@ def test_that_private_over_global_args_does_not_give_logging_message_for_argpass fout.write("FORWARD_MODEL job_name(=)") ert_config = ErtConfig.from_file("config_file.ert") + data = forward_model_data_to_json( + substitutions=ert_config.substitutions, + forward_model_steps=ert_config.forward_model_steps, + env_vars=ert_config.env_vars, + user_config_file=ert_config.user_config_file, + run_id="", + ) - fm_data = ert_config.forward_model_data_to_json("", 0, 0)["jobList"][0] + fm_data = data["jobList"][0] assert len(ert_config.forward_model_steps) == 1 assert fm_data["argList"] == ["A"] assert "Private arg '':'' chosen over global 'A'" not in caplog.text @@ -768,17 +838,17 @@ def test_that_environment_variables_are_set_in_forward_model( fout.write(forward_model) ert_config = ErtConfig.from_file("config_file.ert") + data = forward_model_data_to_json( + substitutions=ert_config.substitutions, + forward_model_steps=ert_config.forward_model_steps, + env_vars=ert_config.env_vars, + user_config_file=ert_config.user_config_file, + run_id="", + ) forward_model_list = ert_config.forward_model_steps assert len(forward_model_list) == 1 - assert ( - ert_config.forward_model_data_to_json( - "", - 0, - 0, - )["jobList"][0]["argList"] - == expected_args - ) + assert data["jobList"][0]["argList"] == expected_args def test_that_executables_in_path_are_not_made_realpath(tmp_path): @@ -799,8 +869,12 @@ def test_that_executables_in_path_are_not_made_realpath(tmp_path): ) ert_config = ErtConfig.from_file(str(config_file)) - - assert ( - ert_config.forward_model_data_to_json("", 0, 0)["jobList"][0]["executable"] - == "echo" + data = forward_model_data_to_json( + substitutions=ert_config.substitutions, + forward_model_steps=ert_config.forward_model_steps, + env_vars=ert_config.env_vars, + user_config_file=ert_config.user_config_file, + run_id="", ) + + assert data["jobList"][0]["executable"] == "echo" diff --git a/tests/ert/unit_tests/config/test_gen_kw_config.py b/tests/ert/unit_tests/config/test_gen_kw_config.py index e87735a70ba..cced0533720 100644 --- a/tests/ert/unit_tests/config/test_gen_kw_config.py +++ b/tests/ert/unit_tests/config/test_gen_kw_config.py @@ -229,10 +229,15 @@ def test_gen_kw_is_log_or_not( ) sample_prior(prior_ensemble, [0]) create_run_path( - run_args(ert_config, prior_ensemble), - prior_ensemble, - ert_config, - run_paths(ert_config), + run_args=run_args(ert_config, prior_ensemble), + ensemble=prior_ensemble, + runpaths=run_paths(ert_config), + user_config_file=ert_config.user_config_file, + forward_model_steps=ert_config.forward_model_steps, + env_vars=ert_config.env_vars, + substitutions=ert_config.substitutions, + templates=ert_config.ert_templates, + model_config=ert_config.model_config, ) assert re.match( parameters_regex, diff --git a/tests/ert/unit_tests/forward_model_runner/test_forward_model_runner.py b/tests/ert/unit_tests/forward_model_runner/test_forward_model_runner.py index 5f0974791f3..dc4e9103b40 100644 --- a/tests/ert/unit_tests/forward_model_runner/test_forward_model_runner.py +++ b/tests/ert/unit_tests/forward_model_runner/test_forward_model_runner.py @@ -9,7 +9,10 @@ from _ert.forward_model_runner.reporting.message import Checksum, Exited, Start from _ert.forward_model_runner.runner import ForwardModelRunner from ert.config import ErtConfig, ForwardModelStep -from ert.config.ert_config import _forward_model_step_from_config_file +from ert.config.ert_config import ( + _forward_model_step_from_config_file, + forward_model_data_to_json, +) from ert.substitutions import Substitutions # Test data generated by ForwardModel @@ -230,9 +233,14 @@ def test_exec_env(): ) with open("jobs.json", mode="w", encoding="utf-8") as fptr: + ert_config = ErtConfig(forward_model_steps=[forward_model]) json.dump( - ErtConfig(forward_model_steps=[forward_model]).forward_model_data_to_json( - "run_id" + forward_model_data_to_json( + substitutions=ert_config.substitutions, + forward_model_steps=ert_config.forward_model_steps, + env_vars=ert_config.env_vars, + user_config_file=ert_config.user_config_file, + run_id="run_id", ), fptr, ) @@ -266,8 +274,15 @@ def test_env_var_available_inside_step_context(): step = _forward_model_step_from_config_file(name=None, config_file="RUN_ENV") with open("jobs.json", mode="w", encoding="utf-8") as fptr: + ert_config = ErtConfig(forward_model_steps=[step]) json.dump( - ErtConfig(forward_model_steps=[step]).forward_model_data_to_json("run_id"), + forward_model_data_to_json( + substitutions=ert_config.substitutions, + forward_model_steps=ert_config.forward_model_steps, + env_vars=ert_config.env_vars, + user_config_file=ert_config.user_config_file, + run_id="run_id", + ), fptr, ) @@ -306,12 +321,17 @@ def test_default_env_variables_available_inside_fm_step_context(): step = _forward_model_step_from_config_file(name=None, config_file="RUN_ENV") with open("jobs.json", mode="w", encoding="utf-8") as fptr: + ert_config = ErtConfig( + forward_model_steps=[step], + substitutions=Substitutions({"": "./"}), + ) json.dump( - ErtConfig( - forward_model_steps=[step], - substitutions=Substitutions({"": "./"}), - ).forward_model_data_to_json( - "run_id", + forward_model_data_to_json( + substitutions=ert_config.substitutions, + forward_model_steps=ert_config.forward_model_steps, + env_vars=ert_config.env_vars, + user_config_file=ert_config.user_config_file, + run_id="run_id", ), fptr, ) diff --git a/tests/ert/unit_tests/simulator/test_batch_sim.py b/tests/ert/unit_tests/simulator/test_batch_sim.py index 8f0bf94ebd9..5e961e07db4 100644 --- a/tests/ert/unit_tests/simulator/test_batch_sim.py +++ b/tests/ert/unit_tests/simulator/test_batch_sim.py @@ -81,42 +81,40 @@ def __init__(self, ert_config, storage, controls, results, callback=None): parameters=ens_config.parameter_configuration, responses=ens_config.response_configuration, ) - super().__init__(ert_config, experiment, set(controls), results, callback) + super().__init__( + perferred_num_cpu=ert_config.preferred_num_cpu, + runpath_file=ert_config.runpath_file, + user_config_file=ert_config.user_config_file, + env_vars=ert_config.env_vars, + forward_model_steps=ert_config.forward_model_steps, + parameter_configurations=ert_config.ensemble_config.parameter_configs, + substitutions=ert_config.substitutions, + queue_config=ert_config.queue_config, + model_config=ert_config.model_config, + analysis_config=ert_config.analysis_config, + hooked_workflows=ert_config.hooked_workflows, + templates=ert_config.ert_templates, + experiment=experiment, + controls=set(controls), + results=results, + callback=callback, + ) except Exception as e: - if isinstance(e, (TypeError, ValueError)): - raise e - else: - super().__init__(ert_config, None, set(controls), results, callback) - - -BatchSimulator = PatchedBatchSimulator - - -def test_that_simulator_raises_error_when_missing_ertconfig(storage): - with pytest.raises(ValueError, match="The first argument must be valid ErtConfig"): - _ = BatchSimulator( - "ARG", - storage, - { - "WELL_ORDER": ["W1", "W2", "W3"], - "WELL_ON_OFF": ["W1", "W2", "W3"], - }, - ["ORDER", "ON_OFF"], - ) + raise e def test_that_batch_simulator_gives_good_message_on_duplicate_keys( minimum_case, storage ): with pytest.raises(ValueError, match="Duplicate keys"): - _ = BatchSimulator( + _ = PatchedBatchSimulator( minimum_case, storage, {"WELL_ORDER": ["W3", "W2", "W3"]}, ["ORDER"] ) @pytest.fixture def batch_simulator(batch_sim_example, storage): - return BatchSimulator( + return PatchedBatchSimulator( batch_sim_example, storage, {"WELL_ORDER": ["W1", "W2", "W3"], "WELL_ON_OFF": ["W1", "W2", "W3"]}, @@ -278,7 +276,7 @@ def test_that_batch_simulation_handles_invalid_suffixes_at_init( batch_sim_example, suffix, error, storage ): with pytest.raises(error): - _ = BatchSimulator( + _ = PatchedBatchSimulator( batch_sim_example, storage, { @@ -326,7 +324,7 @@ def test_that_batch_simulation_handles_invalid_suffixes_at_init( def test_that_batch_simulator_handles_invalid_suffixes_at_start( batch_sim_example, inp, match, storage ): - rsim = BatchSimulator( + rsim = PatchedBatchSimulator( batch_sim_example, storage, { @@ -347,7 +345,7 @@ def test_that_batch_simulator_handles_invalid_suffixes_at_start( def test_batch_simulation_suffixes(batch_sim_example, storage): ert_config = batch_sim_example monitor = MockMonitor() - rsim = BatchSimulator( + rsim = PatchedBatchSimulator( ert_config, storage, { @@ -430,7 +428,7 @@ def test_stop_sim(copy_case, storage): ert_config = ErtConfig.from_file("sleepy_time.ert") - rsim = BatchSimulator( + rsim = PatchedBatchSimulator( ert_config, storage, {"WELL_ORDER": ["W1", "W2", "W3"], "WELL_ON_OFF": ["W1", "W2", "W3"]}, @@ -506,7 +504,7 @@ def test_batch_ctx_status_failing_jobs(setup_case, storage): "WELL_ON_OFF": ("W1", "W2", "W3"), } results = ("ORDER", "ON_OFF") - rsim = BatchSimulator(ert_config, storage, external_parameters, results) + rsim = PatchedBatchSimulator(ert_config, storage, external_parameters, results) ensembles = [ ( diff --git a/tests/ert/unit_tests/simulator/test_simulation_context.py b/tests/ert/unit_tests/simulator/test_simulation_context.py index c2dd6e1d855..15ee6c2d498 100644 --- a/tests/ert/unit_tests/simulator/test_simulation_context.py +++ b/tests/ert/unit_tests/simulator/test_simulation_context.py @@ -35,8 +35,44 @@ def test_simulation_context( ) case_data = [(geo_id, {}) for geo_id in range(size)] - even_ctx = BatchContext([], ert_config, even_half, even_mask, 0, case_data) - odd_ctx = BatchContext([], ert_config, odd_half, odd_mask, 0, case_data) + + even_ctx = BatchContext( + result_keys=[], + preferred_num_cpu=ert_config.preferred_num_cpu, + queue_config=ert_config.queue_config, + model_config=ert_config.model_config, + analysis_config=ert_config.analysis_config, + hooked_workflows=ert_config.hooked_workflows, + substitutions=ert_config.substitutions, + templates=ert_config.ert_templates, + user_config_file=ert_config.user_config_file, + env_vars=ert_config.env_vars, + forward_model_steps=ert_config.forward_model_steps, + runpath_file=ert_config.runpath_file, + ensemble=even_half, + mask=even_mask, + itr=0, + case_data=case_data, + ) + + odd_ctx = BatchContext( + result_keys=[], + preferred_num_cpu=ert_config.preferred_num_cpu, + queue_config=ert_config.queue_config, + model_config=ert_config.model_config, + analysis_config=ert_config.analysis_config, + hooked_workflows=ert_config.hooked_workflows, + substitutions=ert_config.substitutions, + templates=ert_config.ert_templates, + user_config_file=ert_config.user_config_file, + env_vars=ert_config.env_vars, + forward_model_steps=ert_config.forward_model_steps, + runpath_file=ert_config.runpath_file, + ensemble=odd_half, + mask=odd_mask, + itr=0, + case_data=case_data, + ) for iens in range(size): if iens % 2 == 0: diff --git a/tests/ert/unit_tests/storage/create_runpath.py b/tests/ert/unit_tests/storage/create_runpath.py index ca936faf058..1dc2b372098 100644 --- a/tests/ert/unit_tests/storage/create_runpath.py +++ b/tests/ert/unit_tests/storage/create_runpath.py @@ -45,10 +45,15 @@ def create_runpath( random_seed=random_seed, ) create_run_path( - run_args, - ensemble, - ert_config, - runpaths, + run_args=run_args, + ensemble=ensemble, + user_config_file=ert_config.user_config_file, + env_vars=ert_config.env_vars, + forward_model_steps=ert_config.forward_model_steps, + substitutions=ert_config.substitutions, + templates=ert_config.ert_templates, + model_config=ert_config.model_config, + runpaths=runpaths, ) return ert_config.ensemble_config, ensemble diff --git a/tests/ert/unit_tests/test_enkf_main.py b/tests/ert/unit_tests/test_enkf_main.py index f65d1b7de07..3336d6a3666 100644 --- a/tests/ert/unit_tests/test_enkf_main.py +++ b/tests/ert/unit_tests/test_enkf_main.py @@ -75,13 +75,18 @@ def test_assert_symlink_deleted(snake_oil_field_example, storage, run_paths): [True] * prior_ensemble.ensemble_size, prior_ensemble, ) - config = snake_oil_field_example + sample_prior(prior_ensemble, range(prior_ensemble.ensemble_size)) create_run_path( - run_args, - prior_ensemble, - config, - runpaths, + run_args=run_args, + ensemble=prior_ensemble, + user_config_file=ert_config.user_config_file, + forward_model_steps=ert_config.forward_model_steps, + env_vars=ert_config.env_vars, + substitutions=ert_config.substitutions, + templates=ert_config.ert_templates, + model_config=ert_config.model_config, + runpaths=runpaths, ) # replace field file with symlink @@ -94,10 +99,15 @@ def test_assert_symlink_deleted(snake_oil_field_example, storage, run_paths): # recreate directory structure create_run_path( - run_args, - prior_ensemble, - config, - runpaths, + run_args=run_args, + ensemble=prior_ensemble, + user_config_file=ert_config.user_config_file, + forward_model_steps=ert_config.forward_model_steps, + env_vars=ert_config.env_vars, + substitutions=ert_config.substitutions, + templates=ert_config.ert_templates, + model_config=ert_config.model_config, + runpaths=runpaths, ) # ensure field symlink is replaced by file diff --git a/tests/ert/unit_tests/test_enkf_runpath.py b/tests/ert/unit_tests/test_enkf_runpath.py index ba462b35221..8a0d21e2959 100755 --- a/tests/ert/unit_tests/test_enkf_runpath.py +++ b/tests/ert/unit_tests/test_enkf_runpath.py @@ -20,10 +20,15 @@ def test_with_gen_kw(storage, run_paths, run_args): sample_prior(prior_ensemble, [0]) create_run_path( - run_args(ert_config, prior_ensemble, 1), - prior_ensemble, - ert_config, - run_paths(ert_config), + run_args=run_args(ert_config, prior_ensemble, 1), + ensemble=prior_ensemble, + user_config_file=ert_config.user_config_file, + forward_model_steps=ert_config.forward_model_steps, + env_vars=ert_config.env_vars, + substitutions=ert_config.substitutions, + templates=ert_config.ert_templates, + model_config=ert_config.model_config, + runpaths=run_paths(ert_config), ) assert os.path.exists( "storage/snake_oil/runpath/realization-0/iter-0/parameters.txt" @@ -43,10 +48,15 @@ def test_without_gen_kw(prior_ensemble, run_args, run_paths): ert_config = ErtConfig.from_file("snake_oil.ert") sample_prior(prior_ensemble, [0]) create_run_path( - run_args(ert_config, prior_ensemble, 1), - prior_ensemble, - ert_config, - run_paths(ert_config), + run_args=run_args(ert_config, prior_ensemble, 1), + ensemble=prior_ensemble, + user_config_file=ert_config.user_config_file, + forward_model_steps=ert_config.forward_model_steps, + env_vars=ert_config.env_vars, + substitutions=ert_config.substitutions, + templates=ert_config.ert_templates, + model_config=ert_config.model_config, + runpaths=run_paths(ert_config), ) assert os.path.exists("storage/snake_oil/runpath/realization-0/iter-0") assert not os.path.exists( @@ -68,17 +78,27 @@ def test_jobs_file_is_backed_up(storage, run_args, run_paths): run_arg = run_args(ert_config, prior_ensemble, 1) sample_prior(prior_ensemble, [0]) create_run_path( - run_arg, - prior_ensemble, - ert_config, - run_paths(ert_config), + run_args=run_arg, + ensemble=prior_ensemble, + user_config_file=ert_config.user_config_file, + forward_model_steps=ert_config.forward_model_steps, + env_vars=ert_config.env_vars, + substitutions=ert_config.substitutions, + templates=ert_config.ert_templates, + model_config=ert_config.model_config, + runpaths=run_paths(ert_config), ) assert os.path.exists("storage/snake_oil/runpath/realization-0/iter-0/jobs.json") create_run_path( - run_arg, - prior_ensemble, - ert_config, - run_paths(ert_config), + run_args=run_arg, + ensemble=prior_ensemble, + user_config_file=ert_config.user_config_file, + forward_model_steps=ert_config.forward_model_steps, + env_vars=ert_config.env_vars, + substitutions=ert_config.substitutions, + templates=ert_config.ert_templates, + model_config=ert_config.model_config, + runpaths=run_paths(ert_config), ) iter0_output_files = os.listdir("storage/snake_oil/runpath/realization-0/iter-0/") jobs_files = [f for f in iter0_output_files if f.startswith("jobs.json")] diff --git a/tests/ert/unit_tests/test_load_forward_model.py b/tests/ert/unit_tests/test_load_forward_model.py index ada18db7e77..ffd775daa96 100644 --- a/tests/ert/unit_tests/test_load_forward_model.py +++ b/tests/ert/unit_tests/test_load_forward_model.py @@ -28,10 +28,15 @@ def func(config_text): ensemble_size=ert_config.model_config.num_realizations, ) create_run_path( - run_args(ert_config, prior_ensemble), - prior_ensemble, - ert_config, - run_paths(ert_config), + run_args=run_args(ert_config, prior_ensemble), + ensemble=prior_ensemble, + user_config_file=ert_config.user_config_file, + env_vars=ert_config.env_vars, + forward_model_steps=ert_config.forward_model_steps, + substitutions=ert_config.substitutions, + templates=ert_config.ert_templates, + model_config=ert_config.model_config, + runpaths=run_paths(ert_config), ) return ert_config, prior_ensemble @@ -134,10 +139,15 @@ def test_load_forward_model_summary( ) create_run_path( - run_args(ert_config, prior_ensemble), - prior_ensemble, - ert_config, - run_paths(ert_config), + run_args=run_args(ert_config, prior_ensemble), + ensemble=prior_ensemble, + user_config_file=ert_config.user_config_file, + env_vars=ert_config.env_vars, + forward_model_steps=ert_config.forward_model_steps, + substitutions=ert_config.substitutions, + templates=ert_config.ert_templates, + model_config=ert_config.model_config, + runpaths=run_paths(ert_config), ) facade = LibresFacade(ert_config) with caplog.at_level(logging.ERROR): @@ -235,10 +245,15 @@ def test_loading_gen_data_without_restart(storage, run_paths, run_args): ) create_run_path( - run_args(ert_config, prior_ensemble), - prior_ensemble, - ert_config, - run_paths(ert_config), + run_args=run_args(ert_config, prior_ensemble), + ensemble=prior_ensemble, + user_config_file=ert_config.user_config_file, + env_vars=ert_config.env_vars, + forward_model_steps=ert_config.forward_model_steps, + substitutions=ert_config.substitutions, + templates=ert_config.ert_templates, + model_config=ert_config.model_config, + runpaths=run_paths(ert_config), ) run_path = Path("simulations/realization-0/iter-0/") with open(run_path / "response.out", "w", encoding="utf-8") as fout: diff --git a/tests/ert/unit_tests/test_run_path_creation.py b/tests/ert/unit_tests/test_run_path_creation.py index d571f366e70..a54ee55fc7e 100644 --- a/tests/ert/unit_tests/test_run_path_creation.py +++ b/tests/ert/unit_tests/test_run_path_creation.py @@ -46,10 +46,15 @@ def test_that_run_template_replace_symlink_does_not_write_to_source( ) os.symlink("start.txt", run_path / "result.txt") create_run_path( - run_arg, - prior_ensemble, - ert_config, - run_paths(ert_config), + run_args=run_arg, + ensemble=prior_ensemble, + user_config_file=ert_config.user_config_file, + env_vars=ert_config.env_vars, + forward_model_steps=ert_config.forward_model_steps, + substitutions=ert_config.substitutions, + templates=ert_config.ert_templates, + model_config=ert_config.model_config, + runpaths=run_paths(ert_config), ) assert (run_path / "result.txt").read_text( encoding="utf-8" @@ -83,10 +88,15 @@ def test_run_template_replace_in_file_with_custom_define( ert_config = ErtConfig.from_file("config.ert") run_arg = run_args(ert_config, prior_ensemble) create_run_path( - run_arg, - prior_ensemble, - ert_config, - run_paths(ert_config), + run_args=run_arg, + ensemble=prior_ensemble, + user_config_file=ert_config.user_config_file, + env_vars=ert_config.env_vars, + forward_model_steps=ert_config.forward_model_steps, + substitutions=ert_config.substitutions, + templates=ert_config.ert_templates, + model_config=ert_config.model_config, + runpaths=run_paths(ert_config), ) assert ( Path(run_arg[0].runpath) / "result.txt" @@ -125,10 +135,15 @@ def test_run_template_replace_in_file( ert_config = ErtConfig.from_file("config.ert") run_arg = run_args(ert_config, prior_ensemble) create_run_path( - run_arg, - prior_ensemble, - ert_config, - run_paths(ert_config), + run_args=run_arg, + ensemble=prior_ensemble, + user_config_file=ert_config.user_config_file, + env_vars=ert_config.env_vars, + forward_model_steps=ert_config.forward_model_steps, + substitutions=ert_config.substitutions, + templates=ert_config.ert_templates, + model_config=ert_config.model_config, + runpaths=run_paths(ert_config), ) assert (Path(run_arg[0].runpath) / "result.txt").read_text( encoding="utf-8" @@ -163,10 +178,15 @@ def test_run_template_replace_in_ecl( ert_config = ErtConfig.from_file("config.ert") run_arg = run_args(ert_config, prior_ensemble) create_run_path( - run_arg, - prior_ensemble, - ert_config, - run_paths(ert_config), + run_args=run_arg, + ensemble=prior_ensemble, + user_config_file=ert_config.user_config_file, + env_vars=ert_config.env_vars, + forward_model_steps=ert_config.forward_model_steps, + substitutions=ert_config.substitutions, + templates=ert_config.ert_templates, + model_config=ert_config.model_config, + runpaths=run_paths(ert_config), ) assert ( Path(run_arg[0].runpath) / expected_file @@ -210,10 +230,15 @@ def test_run_template_replace_in_ecl_data_file( ert_config = ErtConfig.from_file("config.ert") run_arg = run_args(ert_config, prior_ensemble) create_run_path( - run_arg, - prior_ensemble, - ert_config, - run_paths(ert_config), + run_args=run_arg, + ensemble=prior_ensemble, + user_config_file=ert_config.user_config_file, + env_vars=ert_config.env_vars, + forward_model_steps=ert_config.forward_model_steps, + substitutions=ert_config.substitutions, + templates=ert_config.ert_templates, + model_config=ert_config.model_config, + runpaths=run_paths(ert_config), ) assert (Path(run_arg[0].runpath) / "ECL_CASE0.DATA").read_text( encoding="utf-8" @@ -245,10 +270,15 @@ def test_that_error_is_raised_when_data_file_is_badly_encoded( match="Unsupported non UTF-8 character found in file: .*MY_DATA_FILE.DATA", ): create_run_path( - run_args(ert_config, prior_ensemble), - prior_ensemble, - ert_config, - run_paths(ert_config), + run_args=run_args(ert_config, prior_ensemble), + ensemble=prior_ensemble, + user_config_file=ert_config.user_config_file, + env_vars=ert_config.env_vars, + forward_model_steps=ert_config.forward_model_steps, + substitutions=ert_config.substitutions, + templates=ert_config.ert_templates, + model_config=ert_config.model_config, + runpaths=run_paths(ert_config), ) @@ -274,10 +304,15 @@ def test_run_template_replace_in_file_name(prior_ensemble, run_args, run_paths): ert_config = ErtConfig.from_file("config.ert") run_arg = run_args(ert_config, prior_ensemble) create_run_path( - run_arg, - prior_ensemble, - ert_config, - run_paths(ert_config), + run_args=run_arg, + ensemble=prior_ensemble, + user_config_file=ert_config.user_config_file, + env_vars=ert_config.env_vars, + forward_model_steps=ert_config.forward_model_steps, + substitutions=ert_config.substitutions, + templates=ert_config.ert_templates, + model_config=ert_config.model_config, + runpaths=run_paths(ert_config), ) assert ( Path(run_arg[0].runpath) / "result.txt" @@ -385,10 +420,15 @@ def test_that_deprecated_runpath_substitution_remain_valid( run_arg = run_args(ert_config, prior_ensemble, 2) create_run_path( - run_arg, - prior_ensemble, - ert_config, - run_paths(ert_config), + run_args=run_arg, + ensemble=prior_ensemble, + user_config_file=ert_config.user_config_file, + env_vars=ert_config.env_vars, + forward_model_steps=ert_config.forward_model_steps, + substitutions=ert_config.substitutions, + templates=ert_config.ert_templates, + model_config=ert_config.model_config, + runpaths=run_paths(ert_config), ) for realization in run_arg: @@ -433,10 +473,15 @@ def test_write_snakeoil_runpath_file(snake_oil_case, storage, itr): prior_ensemble, ) create_run_path( - run_args, - prior_ensemble, - ert_config, - run_paths, + run_args=run_args, + ensemble=prior_ensemble, + user_config_file=ert_config.user_config_file, + env_vars=ert_config.env_vars, + forward_model_steps=ert_config.forward_model_steps, + substitutions=ert_config.substitutions, + templates=ert_config.ert_templates, + model_config=ert_config.model_config, + runpaths=run_paths, ) for run_arg in run_args: @@ -482,10 +527,15 @@ def test_assert_export(prior_ensemble, run_args, run_paths): sample_prior(prior_ensemble, [0]) run_arg = run_args(ert_config, prior_ensemble) create_run_path( - run_arg, - prior_ensemble, - ert_config, - run_paths(ert_config), + run_args=run_arg, + ensemble=prior_ensemble, + user_config_file=ert_config.user_config_file, + env_vars=ert_config.env_vars, + forward_model_steps=ert_config.forward_model_steps, + substitutions=ert_config.substitutions, + templates=ert_config.ert_templates, + model_config=ert_config.model_config, + runpaths=run_paths(ert_config), ) assert runpath_list_file.exists() assert runpath_list_file.name == "test_runpath_list.txt" @@ -511,10 +561,17 @@ def _create_runpath(ert_config: ErtConfig, storage: Storage) -> None: substitutions=ert_config.substitutions, ) create_run_path( - create_run_arguments(run_paths, [True] * ensemble.ensemble_size, ensemble), - ensemble, - ert_config, - run_paths, + run_args=create_run_arguments( + run_paths, [True] * ensemble.ensemble_size, ensemble + ), + ensemble=ensemble, + user_config_file=ert_config.user_config_file, + env_vars=ert_config.env_vars, + forward_model_steps=ert_config.forward_model_steps, + substitutions=ert_config.substitutions, + templates=ert_config.ert_templates, + model_config=ert_config.model_config, + runpaths=run_paths, ) @@ -672,7 +729,17 @@ def test_crete_runpath_adds_manifest_to_runpath(snake_oil_case, storage, itr): prior_ensemble, ) - create_run_path(run_args, prior_ensemble, ert_config, run_paths) + create_run_path( + run_args=run_args, + ensemble=prior_ensemble, + user_config_file=ert_config.user_config_file, + env_vars=ert_config.env_vars, + forward_model_steps=ert_config.forward_model_steps, + substitutions=ert_config.substitutions, + templates=ert_config.ert_templates, + model_config=ert_config.model_config, + runpaths=run_paths, + ) exp_runpaths = [ runpath_fmt.replace("", str(itr)) diff --git a/tests/ert/unit_tests/test_summary_response.py b/tests/ert/unit_tests/test_summary_response.py index 8f57fa0d231..0439bb271c7 100644 --- a/tests/ert/unit_tests/test_summary_response.py +++ b/tests/ert/unit_tests/test_summary_response.py @@ -40,10 +40,15 @@ def test_load_summary_response_restart_not_zero( ) create_run_path( - run_args(ert_config, ensemble), - ensemble, - ert_config, - run_paths(ert_config), + run_args=run_args(ert_config, ensemble), + ensemble=ensemble, + user_config_file=ert_config.user_config_file, + forward_model_steps=ert_config.forward_model_steps, + env_vars=ert_config.env_vars, + substitutions=ert_config.substitutions, + templates=ert_config.ert_templates, + model_config=ert_config.model_config, + runpaths=run_paths(ert_config), ) shutil.copy(test_path / "PRED_RUN.SMSPEC", sim_path / "PRED_RUN.SMSPEC") shutil.copy(test_path / "PRED_RUN.UNSMRY", sim_path / "PRED_RUN.UNSMRY")