From 4f12c2649ecde80a7f818e68b91f7c637baceeaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5vard=20Berland?= Date: Fri, 25 Oct 2024 12:24:32 +0200 Subject: [PATCH 1/2] Add new hook for specifying config per forward model step --- src/ert/plugins/__init__.py | 1 + .../plugins/hook_specifications/__init__.py | 2 + .../forward_model_steps.py | 8 + src/ert/plugins/plugin_manager.py | 42 ++++++ tests/ert/unit_tests/plugins/dummy_plugins.py | 5 + .../unit_tests/plugins/test_plugin_manager.py | 141 ++++++++++++++++++ 6 files changed, 199 insertions(+) diff --git a/src/ert/plugins/__init__.py b/src/ert/plugins/__init__.py index 7ea1f61235b..584cb1dc8d6 100644 --- a/src/ert/plugins/__init__.py +++ b/src/ert/plugins/__init__.py @@ -27,6 +27,7 @@ def inner(*args: P.args, **kwargs: P.kwargs) -> Any: "installable_workflow_jobs", "help_links", "installable_forward_model_steps", + "forward_model_configuration", "ecl100_config_path", "ecl300_config_path", "flow_config_path", diff --git a/src/ert/plugins/hook_specifications/__init__.py b/src/ert/plugins/hook_specifications/__init__.py index 60f946f9dc5..b3740bcf23a 100644 --- a/src/ert/plugins/hook_specifications/__init__.py +++ b/src/ert/plugins/hook_specifications/__init__.py @@ -4,6 +4,7 @@ flow_config_path, ) from .forward_model_steps import ( + forward_model_configuration, installable_forward_model_steps, ) from .help_resources import help_links @@ -25,6 +26,7 @@ "ecl100_config_path", "ecl300_config_path", "flow_config_path", + "forward_model_configuration", "help_links", "installable_forward_model_steps", "installable_jobs", diff --git a/src/ert/plugins/hook_specifications/forward_model_steps.py b/src/ert/plugins/hook_specifications/forward_model_steps.py index f710803cfc9..ea82e698205 100644 --- a/src/ert/plugins/hook_specifications/forward_model_steps.py +++ b/src/ert/plugins/hook_specifications/forward_model_steps.py @@ -18,3 +18,11 @@ def installable_forward_model_steps() -> ( :return: List of forward model step plugins in the form of subclasses of the ForwardModelStepPlugin class """ + + +@no_type_check +@hook_specification +def forward_model_configuration() -> PluginResponse[List[Type[ForwardModelStepPlugin]]]: + """ + :return: List of configurations to be merged to be provided to forward model steps. + """ diff --git a/src/ert/plugins/plugin_manager.py b/src/ert/plugins/plugin_manager.py index 1744f63d646..c4fb297de96 100644 --- a/src/ert/plugins/plugin_manager.py +++ b/src/ert/plugins/plugin_manager.py @@ -1,5 +1,6 @@ from __future__ import annotations +import collections import logging import os import shutil @@ -140,6 +141,47 @@ def get_flow_config_path(self) -> Optional[str]: hook=self.hook.flow_config_path, config_name="flow" ) + def get_forward_model_configuration(self) -> Dict[str, Dict[str, Any]]: + response: List[PluginResponse[Dict[str, str]]] = ( + self.hook.forward_model_configuration() + ) + if response == []: + return {} + + fm_configs: Dict[str, Dict[str, Any]] = collections.defaultdict(dict) + for res in response: + if not isinstance(res.data, dict): + raise TypeError( + f"{res.plugin_metadata.plugin_name} did not return a dict" + ) + + for fmstep_name, fmstep_config in res.data.items(): + if not isinstance(fmstep_name, str) or not isinstance( + fmstep_config, dict + ): + raise TypeError( + f"{res.plugin_metadata.plugin_name} did not " + "provide dict[str, dict[str, Any]]" + ) + for key, value in fmstep_config.items(): + if not isinstance(key, str): + raise TypeError( + f"{res.plugin_metadata.plugin_name} did not " + f"provide dict[str, dict[str, Any]], got {key} " + "which was not a string." + ) + if key.lower() in [ + existing.lower() for existing in fm_configs[fmstep_name] + ]: + raise RuntimeError( + "Duplicate configuration or fm_step " + f"{fmstep_name} for key {key} when parsing plugin " + f"{res.plugin_metadata.plugin_name}, it is already " + "registered by another plugin." + ) + fm_configs[fmstep_name][key] = value + return fm_configs + def _site_config_lines(self) -> List[str]: try: plugin_responses = self.hook.site_config_lines() diff --git a/tests/ert/unit_tests/plugins/dummy_plugins.py b/tests/ert/unit_tests/plugins/dummy_plugins.py index fa1c7536654..10ca9695815 100644 --- a/tests/ert/unit_tests/plugins/dummy_plugins.py +++ b/tests/ert/unit_tests/plugins/dummy_plugins.py @@ -11,6 +11,11 @@ def help_links(): return {"test": "test", "test2": "test"} +@plugin(name="dummy") +def forward_model_configuration(): + return {"FLOW": {"mpipath": "/foo"}} + + @plugin(name="dummy") def ecl100_config_path(): return "/dummy/path/ecl100_config.yml" diff --git a/tests/ert/unit_tests/plugins/test_plugin_manager.py b/tests/ert/unit_tests/plugins/test_plugin_manager.py index 18a9bdcdbd2..49d8c8fc037 100644 --- a/tests/ert/unit_tests/plugins/test_plugin_manager.py +++ b/tests/ert/unit_tests/plugins/test_plugin_manager.py @@ -3,9 +3,11 @@ import tempfile from unittest.mock import Mock +import pytest from opentelemetry.sdk.trace import TracerProvider import ert.plugins.hook_implementations +from ert import plugin from ert.plugins import ErtPluginManager from tests.ert.unit_tests.plugins import dummy_plugins from tests.ert.unit_tests.plugins.dummy_plugins import ( @@ -19,6 +21,7 @@ def test_no_plugins(): assert pm.get_flow_config_path() is None assert pm.get_ecl100_config_path() is None assert pm.get_ecl300_config_path() is None + assert pm.get_forward_model_configuration() == {} assert len(pm.forward_model_steps) > 0 assert len(pm._get_config_workflow_jobs()) > 0 @@ -41,6 +44,7 @@ def test_with_plugins(): assert pm.get_flow_config_path() == "/dummy/path/flow_config.yml" assert pm.get_ecl100_config_path() == "/dummy/path/ecl100_config.yml" assert pm.get_ecl300_config_path() == "/dummy/path/ecl300_config.yml" + assert pm.get_forward_model_configuration() == {"FLOW": {"mpipath": "/foo"}} assert pm.get_installable_jobs()["job1"] == "/dummy/path/job1" assert pm.get_installable_jobs()["job2"] == "/dummy/path/job2" @@ -58,6 +62,143 @@ def test_with_plugins(): ] +def test_fm_config_with_empty_config(): + class SomePlugin: + @plugin(name="foo") + def forward_model_configuration(): + return {} + + assert ( + ErtPluginManager(plugins=[SomePlugin]).get_forward_model_configuration() == {} + ) + + +def test_fm_config_with_empty_config_for_step(): + class SomePlugin: + @plugin(name="foo") + def forward_model_configuration(): + return {"foo": {}} + + assert ( + ErtPluginManager(plugins=[SomePlugin]).get_forward_model_configuration() == {} + ) + + +def test_fm_config_merges_data_for_step(): + class SomePlugin: + @plugin(name="foo") + def forward_model_configuration(): + return {"foo": {"com": 3}} + + class OtherPlugin: + @plugin(name="bar") + def forward_model_configuration(): + return {"foo": {"bar": 2}} + + assert ErtPluginManager( + plugins=[SomePlugin, OtherPlugin] + ).get_forward_model_configuration() == {"foo": {"com": 3, "bar": 2}} + + +def test_fm_config_multiple_steps(): + class SomePlugin: + @plugin(name="foo") + def forward_model_configuration(): + return {"foo100": {"com": 3}} + + class OtherPlugin: + @plugin(name="bar") + def forward_model_configuration(): + return {"foo200": {"bar": 2}} + + assert ErtPluginManager( + plugins=[SomePlugin, OtherPlugin] + ).get_forward_model_configuration() == {"foo100": {"com": 3}, "foo200": {"bar": 2}} + + +def test_fm_config_conflicting_config(): + class SomePlugin: + @plugin(name="foo") + def forward_model_configuration(): + return {"foo100": {"com": "from_someplugin"}} + + class OtherPlugin: + @plugin(name="foo") + def forward_model_configuration(): + return {"foo100": {"com": "from_otherplugin"}} + + with pytest.raises(RuntimeError, match="Duplicate configuration"): + ErtPluginManager( + plugins=[SomePlugin, OtherPlugin] + ).get_forward_model_configuration() + + +def test_fm_config_with_repeated_keys_different_fm_step(): + class SomePlugin: + @plugin(name="foo") + def forward_model_configuration(): + return {"foo1": {"bar": "1"}} + + class OtherPlugin: + @plugin(name="foo2") + def forward_model_configuration(): + return {"foo2": {"bar": "2"}} + + assert ErtPluginManager( + plugins=[SomePlugin, OtherPlugin] + ).get_forward_model_configuration() == {"foo1": {"bar": "1"}, "foo2": {"bar": "2"}} + + +def test_fm_config_with_repeated_keys_with_different_case(): + class SomePlugin: + @plugin(name="foo") + def forward_model_configuration(): + return {"foo": {"bar": "lower", "BAR": "higher"}} + + with pytest.raises(RuntimeError, match="Duplicate configuration"): + ErtPluginManager(plugins=[SomePlugin]).get_forward_model_configuration() + + +def test_fm_config_with_wrong_type(): + class SomePlugin: + @plugin(name="foo") + def forward_model_configuration(): + return 1 + + with pytest.raises(TypeError, match="foo did not return a dict"): + ErtPluginManager(plugins=[SomePlugin]).get_forward_model_configuration() + + +def test_fm_config_with_wrong_steptype(): + class SomePlugin: + @plugin(name="foo") + def forward_model_configuration(): + return {1: {"bar": "1"}} + + with pytest.raises(TypeError, match="foo did not provide dict"): + ErtPluginManager(plugins=[SomePlugin]).get_forward_model_configuration() + + +def test_fm_config_with_wrong_subtype(): + class SomePlugin: + @plugin(name="foo") + def forward_model_configuration(): + return {"foo100": 1} + + with pytest.raises(TypeError, match="foo did not provide dict"): + ErtPluginManager(plugins=[SomePlugin]).get_forward_model_configuration() + + +def test_fm_config_with_wrong_keytype(): + class SomePlugin: + @plugin(name="foo") + def forward_model_configuration(): + return {"foo100": {1: "bar"}} + + with pytest.raises(TypeError, match="foo did not provide dict"): + ErtPluginManager(plugins=[SomePlugin]).get_forward_model_configuration() + + def test_job_documentation(): pm = ErtPluginManager(plugins=[dummy_plugins]) expected = { From e3ec1e7baf741d322933bbc0a8d369cacecfafb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5vard=20Berland?= Date: Tue, 29 Oct 2024 09:53:28 +0100 Subject: [PATCH 2/2] Inject plugin step config into jobs-json for steps General configuration using key-values via the plugin system for individual steps will be merged with environment property of each ForwardModelStep that is dumped as json in every runpath. --- src/ert/config/ert_config.py | 34 ++- src/ert/enkf_main.py | 2 + src/ert/run_models/base_run_model.py | 1 + src/ert/simulator/batch_simulator.py | 3 + src/ert/simulator/batch_simulator_context.py | 2 + src/everest/simulator/simulator.py | 1 + .../ert/unit_tests/config/test_ert_config.py | 228 ++++++++++++++++++ .../unit_tests/config/test_gen_kw_config.py | 1 + .../unit_tests/simulator/test_batch_sim.py | 1 + .../simulator/test_simulation_context.py | 2 + .../ert/unit_tests/storage/create_runpath.py | 1 + tests/ert/unit_tests/test_enkf_main.py | 2 + tests/ert/unit_tests/test_enkf_runpath.py | 4 + .../ert/unit_tests/test_load_forward_model.py | 4 + .../ert/unit_tests/test_run_path_creation.py | 12 + tests/ert/unit_tests/test_summary_response.py | 1 + 16 files changed, 298 insertions(+), 1 deletion(-) diff --git a/src/ert/config/ert_config.py b/src/ert/config/ert_config.py index 61d5032439d..cb73a1c51c4 100644 --- a/src/ert/config/ert_config.py +++ b/src/ert/config/ert_config.py @@ -95,10 +95,13 @@ def create_forward_model_json( itr: int = 0, user_config_file: Optional[str] = "", env_vars: Optional[Dict[str, str]] = None, + env_pr_fm_step: Optional[Dict[str, Dict[str, Any]]] = None, skip_pre_experiment_validation: bool = False, ) -> Dict[str, Any]: if env_vars is None: env_vars = {} + if env_pr_fm_step is None: + env_pr_fm_step = {} class Substituter: def __init__(self, fm_step): @@ -191,7 +194,9 @@ def handle_default(fm_step: ForwardModelStep, arg: str) -> str: handle_default(fm_step, substituter.substitute(arg)) for arg in fm_step.arglist ], - "environment": substituter.filter_env_dict(fm_step.environment), + "environment": substituter.filter_env_dict( + dict(env_pr_fm_step.get(fm_step.name, {}), **fm_step.environment) + ), "exec_env": substituter.filter_env_dict(fm_step.exec_env), "max_running_minutes": fm_step.max_running_minutes, } @@ -226,6 +231,7 @@ def forward_model_data_to_json( substitutions: Substitutions, forward_model_steps: List[ForwardModelStep], env_vars: Dict[str, str], + env_pr_fm_step: Optional[Dict[str, Dict[str, Any]]] = None, user_config_file: Optional[str] = "", run_id: Optional[str] = None, iens: int = 0, @@ -234,11 +240,14 @@ def forward_model_data_to_json( ): if context_env is None: context_env = {} + if env_pr_fm_step is None: + env_pr_fm_step = {} return create_forward_model_json( context=substitutions, forward_model_steps=forward_model_steps, user_config_file=user_config_file, env_vars={**env_vars, **context_env}, + env_pr_fm_step=env_pr_fm_step, run_id=run_id, iens=iens, itr=itr, @@ -250,6 +259,7 @@ class ErtConfig: DEFAULT_ENSPATH: ClassVar[str] = "storage" DEFAULT_RUNPATH_FILE: ClassVar[str] = ".ert_runpath_list" PREINSTALLED_FORWARD_MODEL_STEPS: ClassVar[Dict[str, ForwardModelStep]] = {} + ENV_PR_FM_STEP: ClassVar[Dict[str, Dict[str, Any]]] = {} substitutions: Substitutions = field(default_factory=Substitutions) ensemble_config: EnsembleConfig = field(default_factory=EnsembleConfig) @@ -317,6 +327,7 @@ def __post_init__(self) -> None: @staticmethod def with_plugins( forward_model_step_classes: Optional[List[Type[ForwardModelStepPlugin]]] = None, + env_pr_fm_step: Optional[Dict[str, Dict[str, Any]]] = None, ) -> Type["ErtConfig"]: if forward_model_step_classes is None: forward_model_step_classes = ErtPluginManager().forward_model_steps @@ -326,10 +337,16 @@ def with_plugins( fm_step = fm_step_subclass() preinstalled_fm_steps[fm_step.name] = fm_step + if env_pr_fm_step is None: + env_pr_fm_step = _uppercase_subkeys_and_stringify_subvalues( + ErtPluginManager().get_forward_model_configuration() + ) + class ErtConfigWithPlugins(ErtConfig): PREINSTALLED_FORWARD_MODEL_STEPS: ClassVar[ Dict[str, ForwardModelStepPlugin] ] = preinstalled_fm_steps + ENV_PR_FM_STEP: ClassVar[Dict[str, Dict[str, Any]]] = env_pr_fm_step assert issubclass(ErtConfigWithPlugins, ErtConfig) return ErtConfigWithPlugins @@ -996,6 +1013,10 @@ def _installed_forward_model_steps_from_dict( def preferred_num_cpu(self) -> int: return int(self.substitutions.get(f"<{ConfigKeys.NUM_CPU}>", 1)) + @property + def env_pr_fm_step(self) -> Dict[str, Dict[str, Any]]: + return self.ENV_PR_FM_STEP + @staticmethod def _create_observations( obs_config_content: Optional[ @@ -1107,6 +1128,17 @@ def _substitutions_from_dict(config_dict) -> Substitutions: return Substitutions(subst_list) +def _uppercase_subkeys_and_stringify_subvalues( + nested_dict: Dict[str, Dict[str, Any]], +) -> Dict[str, Dict[str, str]]: + fixed_dict: dict[str, dict[str, str]] = {} + for key, value in nested_dict.items(): + fixed_dict[key] = { + subkey.upper(): str(subvalue) for subkey, subvalue in value.items() + } + return fixed_dict + + @no_type_check def _forward_model_step_from_config_file( config_file: str, name: Optional[str] = None diff --git a/src/ert/enkf_main.py b/src/ert/enkf_main.py index 070b5fcdc5e..a4c4530d020 100644 --- a/src/ert/enkf_main.py +++ b/src/ert/enkf_main.py @@ -203,6 +203,7 @@ def create_run_path( ensemble: Ensemble, user_config_file: str, env_vars: Dict[str, str], + env_pr_fm_step: Dict[str, Dict[str, Any]], forward_model_steps: List[ForwardModelStep], substitutions: Substitutions, templates: List[Tuple[str, str]], @@ -259,6 +260,7 @@ def create_run_path( forward_model_steps=forward_model_steps, user_config_file=user_config_file, env_vars=env_vars, + env_pr_fm_step=env_pr_fm_step, run_id=run_arg.run_id, iens=run_arg.iens, itr=ensemble.iteration, diff --git a/src/ert/run_models/base_run_model.py b/src/ert/run_models/base_run_model.py index e332f8ee9dc..2f9b9ccf35e 100644 --- a/src/ert/run_models/base_run_model.py +++ b/src/ert/run_models/base_run_model.py @@ -679,6 +679,7 @@ def _evaluate_and_postprocess( ensemble=ensemble, user_config_file=self.ert_config.user_config_file, env_vars=self.ert_config.env_vars, + env_pr_fm_step=self.ert_config.env_pr_fm_step, forward_model_steps=self.ert_config.forward_model_steps, substitutions=self.ert_config.substitutions, templates=self.ert_config.ert_templates, diff --git a/src/ert/simulator/batch_simulator.py b/src/ert/simulator/batch_simulator.py index 00b24710c4a..12d52a573f5 100644 --- a/src/ert/simulator/batch_simulator.py +++ b/src/ert/simulator/batch_simulator.py @@ -37,6 +37,7 @@ def __init__( runpath_file: str, user_config_file: str, env_vars: Dict[str, str], + env_pr_fm_step: Dict[str, Dict[str, Any]], forward_model_steps: List[ForwardModelStep], parameter_configurations: Dict[str, ParameterConfig], queue_config: QueueConfig, @@ -117,6 +118,7 @@ def callback(*args, **kwargs): self.preferred_num_cpu = perferred_num_cpu self.user_config_file = user_config_file self.env_vars = env_vars + self.env_pr_fm_step = env_pr_fm_step self.forward_model_steps = forward_model_steps self.runpath_file = runpath_file self.queue_config = queue_config @@ -265,6 +267,7 @@ def start( preferred_num_cpu=self.preferred_num_cpu, user_config_file=self.user_config_file, env_vars=self.env_vars, + env_pr_fm_step=self.env_pr_fm_step, forward_model_steps=self.forward_model_steps, runpath_file=self.runpath_file, queue_config=self.queue_config, diff --git a/src/ert/simulator/batch_simulator_context.py b/src/ert/simulator/batch_simulator_context.py index 41126c60561..8f77643a711 100644 --- a/src/ert/simulator/batch_simulator_context.py +++ b/src/ert/simulator/batch_simulator_context.py @@ -134,6 +134,7 @@ class BatchContext: templates: List[Tuple[str, str]] user_config_file: str env_vars: Dict[str, str] + env_pr_fm_step: Dict[str, Dict[str, Any]] forward_model_steps: List[ForwardModelStep] runpath_file: str ensemble: Ensemble @@ -176,6 +177,7 @@ def __post_init__(self) -> None: ensemble=self.ensemble, user_config_file=self.user_config_file, env_vars=self.env_vars, + env_pr_fm_step=self.env_pr_fm_step, forward_model_steps=self.forward_model_steps, substitutions=self.substitutions, templates=self.templates, diff --git a/src/everest/simulator/simulator.py b/src/everest/simulator/simulator.py index 3e60f533460..75c7315fd69 100644 --- a/src/everest/simulator/simulator.py +++ b/src/everest/simulator/simulator.py @@ -36,6 +36,7 @@ def __init__( perferred_num_cpu=ert_config.preferred_num_cpu, user_config_file=ert_config.user_config_file, env_vars=ert_config.env_vars, + env_pr_fm_step=ert_config.env_pr_fm_step, forward_model_steps=ert_config.forward_model_steps, runpath_file=str(ert_config.runpath_file), parameter_configurations=ert_config.ensemble_config.parameter_configs, diff --git a/tests/ert/unit_tests/config/test_ert_config.py b/tests/ert/unit_tests/config/test_ert_config.py index e86941abcf7..e56d5974c56 100644 --- a/tests/ert/unit_tests/config/test_ert_config.py +++ b/tests/ert/unit_tests/config/test_ert_config.py @@ -8,6 +8,7 @@ from datetime import date from pathlib import Path from textwrap import dedent +from unittest.mock import MagicMock import pytest from hypothesis import assume, given, settings @@ -29,6 +30,7 @@ ContextString, ) from ert.config.parsing.queue_system import QueueSystem +from ert.plugins import ErtPluginManager from .config_dict_generator import config_generators @@ -830,6 +832,232 @@ def test_that_include_statements_with_multiple_values_raises_error(): ) +@pytest.mark.usefixtures("use_tmpdir") +def test_fm_step_config_via_plugin_ends_up_json_data(monkeypatch): + monkeypatch.setattr( + ErtPluginManager, + "get_forward_model_configuration", + MagicMock(return_value={"SOME_STEP": {"FOO": "bar"}}), + ) + Path("SOME_STEP").write_text("EXECUTABLE /bin/ls", encoding="utf-8") + Path("config.ert").write_text( + dedent( + """ + NUM_REALIZATIONS 1 + INSTALL_JOB SOME_STEP SOME_STEP + FORWARD_MODEL SOME_STEP() + """ + ), + encoding="utf-8", + ) + ert_config = ErtConfig.with_plugins().from_file("config.ert") + step_json = forward_model_data_to_json( + substitutions=ert_config.substitutions, + forward_model_steps=ert_config.forward_model_steps, + env_vars=ert_config.env_vars, + env_pr_fm_step=ert_config.env_pr_fm_step, + ) + assert step_json["jobList"][0]["environment"]["FOO"] == "bar" + + +def test_fm_step_config_via_plugin_does_not_leak_to_other_step(monkeypatch): + monkeypatch.setattr( + ErtPluginManager, + "get_forward_model_configuration", + MagicMock(return_value={"SOME_STEP": {"FOO": "bar"}}), + ) + Path("SOME_OTHER_STEP").write_text("EXECUTABLE /bin/ls", encoding="utf-8") + Path("config.ert").write_text( + dedent( + """ + NUM_REALIZATIONS 1 + INSTALL_JOB SOME_OTHER_STEP SOME_OTHER_STEP + FORWARD_MODEL SOME_OTHER_STEP() + """ + ), + encoding="utf-8", + ) + ert_config = ErtConfig.with_plugins().from_file("config.ert") + step_json = forward_model_data_to_json( + substitutions=ert_config.substitutions, + forward_model_steps=ert_config.forward_model_steps, + env_vars=ert_config.env_vars, + env_pr_fm_step=ert_config.env_pr_fm_step, + ) + assert "FOO" not in step_json["jobList"][0]["environment"] + + +def test_fm_step_config_via_plugin_has_key_names_uppercased(monkeypatch): + monkeypatch.setattr( + ErtPluginManager, + "get_forward_model_configuration", + MagicMock(return_value={"SOME_STEP": {"foo": "bar"}}), + ) + Path("SOME_STEP").write_text("EXECUTABLE /bin/ls", encoding="utf-8") + Path("config.ert").write_text( + dedent( + """ + NUM_REALIZATIONS 1 + INSTALL_JOB SOME_STEP SOME_STEP + FORWARD_MODEL SOME_STEP() + """ + ), + encoding="utf-8", + ) + ert_config = ErtConfig.with_plugins().from_file("config.ert") + step_json = forward_model_data_to_json( + substitutions=ert_config.substitutions, + forward_model_steps=ert_config.forward_model_steps, + env_vars=ert_config.env_vars, + env_pr_fm_step=ert_config.env_pr_fm_step, + ) + assert step_json["jobList"][0]["environment"]["FOO"] == "bar" + + +def test_fm_step_config_via_plugin_stringifies_python_objects(monkeypatch): + monkeypatch.setattr( + ErtPluginManager, + "get_forward_model_configuration", + MagicMock(return_value={"SOME_STEP": {"FOO": {"a_dict_as_value": 1}}}), + ) + Path("SOME_STEP").write_text("EXECUTABLE /bin/ls", encoding="utf-8") + Path("config.ert").write_text( + dedent( + """ + NUM_REALIZATIONS 1 + INSTALL_JOB SOME_STEP SOME_STEP + FORWARD_MODEL SOME_STEP() + """ + ), + encoding="utf-8", + ) + ert_config = ErtConfig.with_plugins().from_file("config.ert") + step_json = forward_model_data_to_json( + substitutions=ert_config.substitutions, + forward_model_steps=ert_config.forward_model_steps, + env_vars=ert_config.env_vars, + env_pr_fm_step=ert_config.env_pr_fm_step, + ) + assert step_json["jobList"][0]["environment"]["FOO"] == "{'a_dict_as_value': 1}" + + +def test_fm_step_config_via_plugin_ignores_conflict_with_setenv(monkeypatch): + monkeypatch.setattr( + ErtPluginManager, + "get_forward_model_configuration", + MagicMock( + return_value={"SOME_STEP": {"FOO": "bar_from_plugin", "_ERT_RUNPATH": "0"}} + ), + ) + Path("SOME_STEP").write_text("EXECUTABLE /bin/ls", encoding="utf-8") + Path("config.ert").write_text( + dedent( + """ + NUM_REALIZATIONS 1 + SETENV FOO bar_from_setenv + INSTALL_JOB SOME_STEP SOME_STEP + FORWARD_MODEL SOME_STEP() + """ + ), + encoding="utf-8", + ) + ert_config = ErtConfig.with_plugins().from_file("config.ert") + step_json = forward_model_data_to_json( + substitutions=ert_config.substitutions, + forward_model_steps=ert_config.forward_model_steps, + env_vars=ert_config.env_vars, + env_pr_fm_step=ert_config.env_pr_fm_step, + ) + assert step_json["global_environment"]["FOO"] == "bar_from_setenv" + assert step_json["jobList"][0]["environment"]["FOO"] == "bar_from_plugin" + # It is up to forward_model_runner to define behaviour here + + +def test_fm_step_config_via_plugin_does_not_override_default_env(monkeypatch): + monkeypatch.setattr( + ErtPluginManager, + "get_forward_model_configuration", + MagicMock(return_value={"SOME_STEP": {"_ERT_RUNPATH": "0"}}), + ) + Path("SOME_STEP").write_text("EXECUTABLE /bin/ls", encoding="utf-8") + Path("config.ert").write_text( + dedent( + """ + NUM_REALIZATIONS 1 + INSTALL_JOB SOME_STEP SOME_STEP + FORWARD_MODEL SOME_STEP() + """ + ), + encoding="utf-8", + ) + ert_config = ErtConfig.with_plugins().from_file("config.ert") + step_json = forward_model_data_to_json( + substitutions=ert_config.substitutions, + forward_model_steps=ert_config.forward_model_steps, + env_vars=ert_config.env_vars, + env_pr_fm_step=ert_config.env_pr_fm_step, + ) + assert ( + step_json["jobList"][0]["environment"]["_ERT_RUNPATH"] + == "simulations/realization-0/iter-0" + ) + + +def test_fm_step_config_via_plugin_is_substituted_for_defines(monkeypatch): + monkeypatch.setattr( + ErtPluginManager, + "get_forward_model_configuration", + MagicMock(return_value={"SOME_STEP": {"FOO": ""}}), + ) + Path("SOME_STEP").write_text("EXECUTABLE /bin/ls", encoding="utf-8") + Path("config.ert").write_text( + dedent( + """ + DEFINE define_works + NUM_REALIZATIONS 1 + INSTALL_JOB SOME_STEP SOME_STEP + FORWARD_MODEL SOME_STEP() + """ + ), + encoding="utf-8", + ) + ert_config = ErtConfig.with_plugins().from_file("config.ert") + step_json = forward_model_data_to_json( + substitutions=ert_config.substitutions, + forward_model_steps=ert_config.forward_model_steps, + env_vars=ert_config.env_vars, + env_pr_fm_step=ert_config.env_pr_fm_step, + ) + assert step_json["jobList"][0]["environment"]["FOO"] == "define_works" + + +def test_fm_step_config_via_plugin_is_dropped_if_not_define_exists(monkeypatch): + monkeypatch.setattr( + ErtPluginManager, + "get_forward_model_configuration", + MagicMock(return_value={"SOME_STEP": {"FOO": ""}}), + ) + Path("SOME_STEP").write_text("EXECUTABLE /bin/ls", encoding="utf-8") + Path("config.ert").write_text( + dedent( + """ + NUM_REALIZATIONS 1 + INSTALL_JOB SOME_STEP SOME_STEP + FORWARD_MODEL SOME_STEP() + """ + ), + encoding="utf-8", + ) + ert_config = ErtConfig.with_plugins().from_file("config.ert") + step_json = forward_model_data_to_json( + substitutions=ert_config.substitutions, + forward_model_steps=ert_config.forward_model_steps, + env_vars=ert_config.env_vars, + env_pr_fm_step=ert_config.env_pr_fm_step, + ) + assert "FOO" not in step_json["jobList"][0]["environment"] + + @pytest.mark.usefixtures("use_tmpdir") def test_that_workflows_with_errors_are_not_loaded(): """ diff --git a/tests/ert/unit_tests/config/test_gen_kw_config.py b/tests/ert/unit_tests/config/test_gen_kw_config.py index cced0533720..53a78ad0176 100644 --- a/tests/ert/unit_tests/config/test_gen_kw_config.py +++ b/tests/ert/unit_tests/config/test_gen_kw_config.py @@ -235,6 +235,7 @@ def test_gen_kw_is_log_or_not( user_config_file=ert_config.user_config_file, forward_model_steps=ert_config.forward_model_steps, env_vars=ert_config.env_vars, + env_pr_fm_step=ert_config.env_pr_fm_step, substitutions=ert_config.substitutions, templates=ert_config.ert_templates, model_config=ert_config.model_config, diff --git a/tests/ert/unit_tests/simulator/test_batch_sim.py b/tests/ert/unit_tests/simulator/test_batch_sim.py index 4ff43593752..db4eb0051d9 100644 --- a/tests/ert/unit_tests/simulator/test_batch_sim.py +++ b/tests/ert/unit_tests/simulator/test_batch_sim.py @@ -82,6 +82,7 @@ def __init__(self, ert_config, storage, controls, results, callback=None): runpath_file=ert_config.runpath_file, user_config_file=ert_config.user_config_file, env_vars=ert_config.env_vars, + env_pr_fm_step=ert_config.env_pr_fm_step, forward_model_steps=ert_config.forward_model_steps, parameter_configurations=ert_config.ensemble_config.parameter_configs, substitutions=ert_config.substitutions, diff --git a/tests/ert/unit_tests/simulator/test_simulation_context.py b/tests/ert/unit_tests/simulator/test_simulation_context.py index 15ee6c2d498..409382a30ba 100644 --- a/tests/ert/unit_tests/simulator/test_simulation_context.py +++ b/tests/ert/unit_tests/simulator/test_simulation_context.py @@ -47,6 +47,7 @@ def test_simulation_context( templates=ert_config.ert_templates, user_config_file=ert_config.user_config_file, env_vars=ert_config.env_vars, + env_pr_fm_step=ert_config.env_pr_fm_step, forward_model_steps=ert_config.forward_model_steps, runpath_file=ert_config.runpath_file, ensemble=even_half, @@ -66,6 +67,7 @@ def test_simulation_context( templates=ert_config.ert_templates, user_config_file=ert_config.user_config_file, env_vars=ert_config.env_vars, + env_pr_fm_step=ert_config.env_pr_fm_step, forward_model_steps=ert_config.forward_model_steps, runpath_file=ert_config.runpath_file, ensemble=odd_half, diff --git a/tests/ert/unit_tests/storage/create_runpath.py b/tests/ert/unit_tests/storage/create_runpath.py index dbf45a7a5f5..f4d2c6275dd 100644 --- a/tests/ert/unit_tests/storage/create_runpath.py +++ b/tests/ert/unit_tests/storage/create_runpath.py @@ -49,6 +49,7 @@ def create_runpath( ensemble=ensemble, user_config_file=ert_config.user_config_file, env_vars=ert_config.env_vars, + env_pr_fm_step=ert_config.env_pr_fm_step, forward_model_steps=ert_config.forward_model_steps, substitutions=ert_config.substitutions, templates=ert_config.ert_templates, diff --git a/tests/ert/unit_tests/test_enkf_main.py b/tests/ert/unit_tests/test_enkf_main.py index 3336d6a3666..5295eb25b06 100644 --- a/tests/ert/unit_tests/test_enkf_main.py +++ b/tests/ert/unit_tests/test_enkf_main.py @@ -83,6 +83,7 @@ def test_assert_symlink_deleted(snake_oil_field_example, storage, run_paths): user_config_file=ert_config.user_config_file, forward_model_steps=ert_config.forward_model_steps, env_vars=ert_config.env_vars, + env_pr_fm_step=ert_config.env_pr_fm_step, substitutions=ert_config.substitutions, templates=ert_config.ert_templates, model_config=ert_config.model_config, @@ -104,6 +105,7 @@ def test_assert_symlink_deleted(snake_oil_field_example, storage, run_paths): user_config_file=ert_config.user_config_file, forward_model_steps=ert_config.forward_model_steps, env_vars=ert_config.env_vars, + env_pr_fm_step=ert_config.env_pr_fm_step, substitutions=ert_config.substitutions, templates=ert_config.ert_templates, model_config=ert_config.model_config, diff --git a/tests/ert/unit_tests/test_enkf_runpath.py b/tests/ert/unit_tests/test_enkf_runpath.py index 8a0d21e2959..841e9251259 100755 --- a/tests/ert/unit_tests/test_enkf_runpath.py +++ b/tests/ert/unit_tests/test_enkf_runpath.py @@ -25,6 +25,7 @@ def test_with_gen_kw(storage, run_paths, run_args): user_config_file=ert_config.user_config_file, forward_model_steps=ert_config.forward_model_steps, env_vars=ert_config.env_vars, + env_pr_fm_step=ert_config.env_pr_fm_step, substitutions=ert_config.substitutions, templates=ert_config.ert_templates, model_config=ert_config.model_config, @@ -53,6 +54,7 @@ def test_without_gen_kw(prior_ensemble, run_args, run_paths): user_config_file=ert_config.user_config_file, forward_model_steps=ert_config.forward_model_steps, env_vars=ert_config.env_vars, + env_pr_fm_step=ert_config.env_pr_fm_step, substitutions=ert_config.substitutions, templates=ert_config.ert_templates, model_config=ert_config.model_config, @@ -83,6 +85,7 @@ def test_jobs_file_is_backed_up(storage, run_args, run_paths): user_config_file=ert_config.user_config_file, forward_model_steps=ert_config.forward_model_steps, env_vars=ert_config.env_vars, + env_pr_fm_step=ert_config.env_pr_fm_step, substitutions=ert_config.substitutions, templates=ert_config.ert_templates, model_config=ert_config.model_config, @@ -95,6 +98,7 @@ def test_jobs_file_is_backed_up(storage, run_args, run_paths): user_config_file=ert_config.user_config_file, forward_model_steps=ert_config.forward_model_steps, env_vars=ert_config.env_vars, + env_pr_fm_step=ert_config.env_pr_fm_step, substitutions=ert_config.substitutions, templates=ert_config.ert_templates, model_config=ert_config.model_config, diff --git a/tests/ert/unit_tests/test_load_forward_model.py b/tests/ert/unit_tests/test_load_forward_model.py index ba6a5eaa599..b143122a853 100644 --- a/tests/ert/unit_tests/test_load_forward_model.py +++ b/tests/ert/unit_tests/test_load_forward_model.py @@ -32,6 +32,7 @@ def func(config_text): ensemble=prior_ensemble, user_config_file=ert_config.user_config_file, env_vars=ert_config.env_vars, + env_pr_fm_step=ert_config.env_pr_fm_step, forward_model_steps=ert_config.forward_model_steps, substitutions=ert_config.substitutions, templates=ert_config.ert_templates, @@ -143,6 +144,7 @@ def test_load_forward_model_summary( ensemble=prior_ensemble, user_config_file=ert_config.user_config_file, env_vars=ert_config.env_vars, + env_pr_fm_step=ert_config.env_pr_fm_step, forward_model_steps=ert_config.forward_model_steps, substitutions=ert_config.substitutions, templates=ert_config.ert_templates, @@ -249,6 +251,7 @@ def test_loading_gen_data_without_restart(storage, run_paths, run_args): ensemble=prior_ensemble, user_config_file=ert_config.user_config_file, env_vars=ert_config.env_vars, + env_pr_fm_step=ert_config.env_pr_fm_step, forward_model_steps=ert_config.forward_model_steps, substitutions=ert_config.substitutions, templates=ert_config.ert_templates, @@ -315,6 +318,7 @@ def test_loading_from_any_available_iter(storage, run_paths, run_args, itr): ensemble=prior_ensemble, user_config_file=ert_config.user_config_file, env_vars=ert_config.env_vars, + env_pr_fm_step=ert_config.env_pr_fm_step, forward_model_steps=ert_config.forward_model_steps, substitutions=ert_config.substitutions, templates=ert_config.ert_templates, diff --git a/tests/ert/unit_tests/test_run_path_creation.py b/tests/ert/unit_tests/test_run_path_creation.py index 2d46aa8e21f..5d1e6e03f3a 100644 --- a/tests/ert/unit_tests/test_run_path_creation.py +++ b/tests/ert/unit_tests/test_run_path_creation.py @@ -50,6 +50,7 @@ def test_that_run_template_replace_symlink_does_not_write_to_source( ensemble=prior_ensemble, user_config_file=ert_config.user_config_file, env_vars=ert_config.env_vars, + env_pr_fm_step=ert_config.env_pr_fm_step, forward_model_steps=ert_config.forward_model_steps, substitutions=ert_config.substitutions, templates=ert_config.ert_templates, @@ -92,6 +93,7 @@ def test_run_template_replace_in_file_with_custom_define( ensemble=prior_ensemble, user_config_file=ert_config.user_config_file, env_vars=ert_config.env_vars, + env_pr_fm_step=ert_config.env_pr_fm_step, forward_model_steps=ert_config.forward_model_steps, substitutions=ert_config.substitutions, templates=ert_config.ert_templates, @@ -139,6 +141,7 @@ def test_run_template_replace_in_file( ensemble=prior_ensemble, user_config_file=ert_config.user_config_file, env_vars=ert_config.env_vars, + env_pr_fm_step=ert_config.env_pr_fm_step, forward_model_steps=ert_config.forward_model_steps, substitutions=ert_config.substitutions, templates=ert_config.ert_templates, @@ -182,6 +185,7 @@ def test_run_template_replace_in_ecl( ensemble=prior_ensemble, user_config_file=ert_config.user_config_file, env_vars=ert_config.env_vars, + env_pr_fm_step=ert_config.env_pr_fm_step, forward_model_steps=ert_config.forward_model_steps, substitutions=ert_config.substitutions, templates=ert_config.ert_templates, @@ -234,6 +238,7 @@ def test_run_template_replace_in_ecl_data_file( ensemble=prior_ensemble, user_config_file=ert_config.user_config_file, env_vars=ert_config.env_vars, + env_pr_fm_step=ert_config.env_pr_fm_step, forward_model_steps=ert_config.forward_model_steps, substitutions=ert_config.substitutions, templates=ert_config.ert_templates, @@ -274,6 +279,7 @@ def test_that_error_is_raised_when_data_file_is_badly_encoded( ensemble=prior_ensemble, user_config_file=ert_config.user_config_file, env_vars=ert_config.env_vars, + env_pr_fm_step=ert_config.env_pr_fm_step, forward_model_steps=ert_config.forward_model_steps, substitutions=ert_config.substitutions, templates=ert_config.ert_templates, @@ -308,6 +314,7 @@ def test_run_template_replace_in_file_name(prior_ensemble, run_args, run_paths): ensemble=prior_ensemble, user_config_file=ert_config.user_config_file, env_vars=ert_config.env_vars, + env_pr_fm_step=ert_config.env_pr_fm_step, forward_model_steps=ert_config.forward_model_steps, substitutions=ert_config.substitutions, templates=ert_config.ert_templates, @@ -424,6 +431,7 @@ def test_that_deprecated_runpath_substitution_remain_valid( ensemble=prior_ensemble, user_config_file=ert_config.user_config_file, env_vars=ert_config.env_vars, + env_pr_fm_step=ert_config.env_pr_fm_step, forward_model_steps=ert_config.forward_model_steps, substitutions=ert_config.substitutions, templates=ert_config.ert_templates, @@ -477,6 +485,7 @@ def test_write_snakeoil_runpath_file(snake_oil_case, storage, itr): ensemble=prior_ensemble, user_config_file=ert_config.user_config_file, env_vars=ert_config.env_vars, + env_pr_fm_step=ert_config.env_pr_fm_step, forward_model_steps=ert_config.forward_model_steps, substitutions=ert_config.substitutions, templates=ert_config.ert_templates, @@ -533,6 +542,7 @@ def test_assert_export(prior_ensemble, run_args, run_paths): ensemble=prior_ensemble, user_config_file=ert_config.user_config_file, env_vars=ert_config.env_vars, + env_pr_fm_step=ert_config.env_pr_fm_step, forward_model_steps=ert_config.forward_model_steps, substitutions=ert_config.substitutions, templates=ert_config.ert_templates, @@ -569,6 +579,7 @@ def _create_runpath(ert_config: ErtConfig, storage: Storage) -> None: ensemble=ensemble, user_config_file=ert_config.user_config_file, env_vars=ert_config.env_vars, + env_pr_fm_step=ert_config.env_pr_fm_step, forward_model_steps=ert_config.forward_model_steps, substitutions=ert_config.substitutions, templates=ert_config.ert_templates, @@ -736,6 +747,7 @@ def test_crete_runpath_adds_manifest_to_runpath(snake_oil_case, storage, itr): ensemble=prior_ensemble, user_config_file=ert_config.user_config_file, env_vars=ert_config.env_vars, + env_pr_fm_step=ert_config.env_pr_fm_step, forward_model_steps=ert_config.forward_model_steps, substitutions=ert_config.substitutions, templates=ert_config.ert_templates, diff --git a/tests/ert/unit_tests/test_summary_response.py b/tests/ert/unit_tests/test_summary_response.py index 606c6713811..aee0e58a753 100644 --- a/tests/ert/unit_tests/test_summary_response.py +++ b/tests/ert/unit_tests/test_summary_response.py @@ -45,6 +45,7 @@ def test_load_summary_response_restart_not_zero( user_config_file=ert_config.user_config_file, forward_model_steps=ert_config.forward_model_steps, env_vars=ert_config.env_vars, + env_pr_fm_step=ert_config.env_pr_fm_step, substitutions=ert_config.substitutions, templates=ert_config.ert_templates, model_config=ert_config.model_config,