From ee02327ccaa8a3803777bc20ec77f3e50e792106 Mon Sep 17 00:00:00 2001 From: DanSava Date: Fri, 6 Sep 2024 15:07:33 +0300 Subject: [PATCH] Generate Ert manifest file using ensemble Experiment content --- src/ert/config/ert_config.py | 30 ----------- src/ert/enkf_main.py | 43 ++++++++++++++-- tests/unit_tests/test_run_path_creation.py | 60 ++++++++++++++++++++++ 3 files changed, 100 insertions(+), 33 deletions(-) diff --git a/src/ert/config/ert_config.py b/src/ert/config/ert_config.py index bbf46a00b06..a294ba5ce9f 100644 --- a/src/ert/config/ert_config.py +++ b/src/ert/config/ert_config.py @@ -26,7 +26,6 @@ from pydantic import ValidationError as PydanticValidationError from typing_extensions import Self -from ert.config.gen_data_config import GenDataConfig from ert.plugins import ErtPluginManager from ert.substitution_list import SubstitutionList @@ -64,7 +63,6 @@ parse, ) from .queue_config import QueueConfig -from .summary_config import SummaryConfig from .workflow import Workflow from .workflow_job import ErtScriptLoadFailure, WorkflowJob @@ -528,34 +526,6 @@ def _create_list_of_forward_model_steps_to_run( def forward_model_step_name_list(self) -> List[str]: return [j.name for j in self.forward_model_steps] - def manifest_to_json(self, iens: int = 0, iter: int = 0) -> Dict[str, Any]: - manifest = {} - # Add expected parameter files to manifest - if iter == 0: - for ( - name, - parameter_config, - ) in self.ensemble_config.parameter_configs.items(): - if parameter_config.forward_init and parameter_config.forward_init_file: - file_path = parameter_config.forward_init_file.replace( - "%d", str(iens) - ) - manifest[name] = file_path - # Add expected response files to manifest - for name, respons_config in self.ensemble_config.response_configs.items(): - input_file = str(respons_config.input_file) - if isinstance(respons_config, SummaryConfig): - input_file = input_file.replace("", str(iens)) - manifest[f"{name}_UNSMRY"] = f"{input_file}.UNSMRY" - manifest[f"{name}_SMSPEC"] = f"{input_file}.SMSPEC" - if isinstance(respons_config, GenDataConfig): - if respons_config.report_steps: - for step in respons_config.report_steps: - manifest[f"{name}_{step}"] = input_file.replace("%d", str(step)) - elif "%d" not in input_file: - manifest[name] = input_file - return manifest - def forward_model_data_to_json( self, run_id: Optional[str] = None, diff --git a/src/ert/enkf_main.py b/src/ert/enkf_main.py index ce1e2a5255a..536af0bfafc 100644 --- a/src/ert/enkf_main.py +++ b/src/ert/enkf_main.py @@ -6,12 +6,20 @@ import time from datetime import datetime from pathlib import Path -from typing import TYPE_CHECKING, Dict, Iterable, List, Mapping, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Mapping, Optional, Union import orjson from numpy.random import SeedSequence -from .config import ParameterConfig +from .config import ( + ExtParamConfig, + Field, + GenDataConfig, + GenKwConfig, + ParameterConfig, + SummaryConfig, + SurfaceConfig, +) from .run_arg import RunArg from .runpaths import Runpaths @@ -106,6 +114,35 @@ def _generate_parameter_files( _value_export_json(run_path, export_base_name, exports) +def _manifest_to_json(ensemble: Ensemble, iens: int = 0) -> Dict[str, Any]: + manifest = {} + # Add expected parameter files to manifest + for param_config in ensemble.experiment.parameter_configuration.values(): + assert isinstance( + param_config, + (ExtParamConfig, GenKwConfig, Field, SurfaceConfig), + ) + if param_config.forward_init and param_config.forward_init_file is not None: + file_path = param_config.forward_init_file.replace("%d", str(iens)) + manifest[param_config.name] = file_path + elif param_config.output_file is not None and not param_config.forward_init: + manifest[param_config.name] = str(param_config.output_file) + # Add expected response files to manifest + for name, respons_config in ensemble.experiment.response_configuration.items(): + if isinstance(respons_config, SummaryConfig): + input_file = respons_config.input_file.replace("", str(iens)) + manifest[f"{name}_UNSMRY"] = f"{input_file}.UNSMRY" + manifest[f"{name}_SMSPEC"] = f"{input_file}.SMSPEC" + if isinstance(respons_config, GenDataConfig): + input_file = respons_config.input_file + if respons_config.report_steps: + for step in respons_config.report_steps: + manifest[f"{name}_{step}"] = input_file.replace("%d", str(step)) + elif "%d" not in input_file: + manifest[name] = input_file + return manifest + + def _seed_sequence(seed: Optional[int]) -> int: # Set up RNG if seed is None: @@ -213,8 +250,8 @@ def create_run_path( orjson.dumps(forward_model_output, option=orjson.OPT_NON_STR_KEYS) ) # Write MANIFEST file to runpath use to avoid NFS sync issues + data = _manifest_to_json(ensemble, run_arg.iens) with open(run_path / "manifest.json", mode="wb") as fptr: - data = ert_config.manifest_to_json(run_arg.iens, run_arg.itr) fptr.write(orjson.dumps(data, option=orjson.OPT_NON_STR_KEYS)) runpaths.write_runpath_list( diff --git a/tests/unit_tests/test_run_path_creation.py b/tests/unit_tests/test_run_path_creation.py index 75125acd793..69ee1ec6fd7 100644 --- a/tests/unit_tests/test_run_path_creation.py +++ b/tests/unit_tests/test_run_path_creation.py @@ -636,3 +636,63 @@ def test_assert_ertcase_replaced_in_runpath(placeholder, prior_ensemble, storage assert Path(runpath_file).exists() jobs_json = Path(runpath_file) / "jobs.json" assert jobs_json.exists() + + +@pytest.mark.parametrize("itr", [0, 1, 2, 17]) +def test_crete_runpath_adds_manifest_to_runpath(snake_oil_case, storage, itr): + ert_config = snake_oil_case + experiment_id = storage.create_experiment( + parameters=ert_config.ensemble_config.parameter_configuration, + responses=ert_config.ensemble_config.response_configuration, + ) + prior_ensemble = storage.create_ensemble( + experiment_id, name="prior", ensemble_size=25, iteration=itr + ) + + num_realizations = 25 + runpath_fmt = ( + "simulations//realization-/iter-/" + "magic-real-/magic-iter-" + ) + global_substitutions = ert_config.substitution_list + for i in range(num_realizations): + global_substitutions[f""] = str(10 * i) + + run_paths = Runpaths( + jobname_format="SNAKE_OIL_%d", + runpath_format=runpath_fmt, + filename="a_file_name", + substitution_list=global_substitutions, + ) + + sample_prior(prior_ensemble, range(num_realizations)) + run_args = create_run_arguments( + run_paths, + [True, True], + prior_ensemble, + ) + + create_run_path(run_args, prior_ensemble, ert_config, run_paths) + + exp_runpaths = [ + runpath_fmt.replace("", str(itr)) + .replace("", str(run_arg.iens)) + .replace("", str(10 * run_arg.iens)) + for run_arg in run_args + if run_arg.active + ] + exp_runpaths = list(map(os.path.realpath, exp_runpaths)) + expected_manifest_values = [ + "snake_oil_params.txt", + "snake_oil_opr_diff_199.txt", + "snake_oil_wpr_diff_199.txt", + "snake_oil_gpr_diff_199.txt", + "SNAKE_OIL_FIELD.UNSMRY", + "SNAKE_OIL_FIELD.SMSPEC", + ] + for run_path in exp_runpaths: + manifest_path = Path(run_path) / "manifest.json" + assert manifest_path.exists() + with open(manifest_path, encoding="utf-8") as f: + manifest = orjson.loads(f.read()) + assert list(manifest.values()) == expected_manifest_values