From d5ea8878a8fc4bdf5cee58f5f30bbc7c75e9c8e3 Mon Sep 17 00:00:00 2001 From: larsevj <60844986+larsevj@users.noreply.github.com> Date: Mon, 18 Sep 2023 14:43:30 +0200 Subject: [PATCH] Print overwritten Queue Options values to log files --- src/ert/config/parsing/lark_parser.py | 22 ++++++++------ src/ert/config/queue_config.py | 31 ++++++++++++++++++++ tests/unit_tests/config/test_queue_config.py | 31 ++++++++++++++++++++ 3 files changed, 75 insertions(+), 9 deletions(-) diff --git a/src/ert/config/parsing/lark_parser.py b/src/ert/config/parsing/lark_parser.py index f3abaeaed8a..9815443b28e 100644 --- a/src/ert/config/parsing/lark_parser.py +++ b/src/ert/config/parsing/lark_parser.py @@ -153,11 +153,13 @@ def _substitute_token( if key in current: warnings.warn( ConfigWarning.with_context( - f"Gave up replacing in {token}.\n" - f"After replacing the value is now: {current}.\n" - f"This still contains the replacement value: {key}, " - f"which would be replaced by {val}. " - "Probably this causes a loop.", + ( + f"Gave up replacing in {token}.\n" + f"After replacing the value is now: {current}.\n" + f"This still contains the replacement value: {key}, " + f"which would be replaced by {val}. " + "Probably this causes a loop." + ), token, ), stacklevel=1, @@ -260,8 +262,8 @@ def substitute_arg( return [substitute_arglist_tuple(x) for x in arg] raise ValueError( - f"Expected " - f"Union[FileContextToken, List[Tuple[FileContextToken]]], " + "Expected " + "Union[FileContextToken, List[Tuple[FileContextToken]]], " f"got {arg}" ) @@ -462,8 +464,10 @@ def _parse_file(file: str) -> Tree[Instruction]: raise ConfigValidationError( [ ErrorInfo( - message=f"Unsupported non UTF-8 character {unknown_char!r} " - f"found in file: {file!r}", + message=( + f"Unsupported non UTF-8 character {unknown_char!r} " + f"found in file: {file!r}" + ), filename=str(file), column=0, line=bad_line, diff --git a/src/ert/config/queue_config.py b/src/ert/config/queue_config.py index 1a3a3c3b8ef..7d07c6c7f42 100644 --- a/src/ert/config/queue_config.py +++ b/src/ert/config/queue_config.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging import re import shutil from collections import defaultdict @@ -100,6 +101,15 @@ def from_dict(cls, config_dict: ConfigDict) -> QueueConfig: ): _validate_torque_options(queue_options[QueueSystem.TORQUE]) + if ( + selected_queue_system != QueueSystem.LOCAL + and queue_options[selected_queue_system] + ): + _check_for_overwritten_queue_system_options( + selected_queue_system, + queue_options[selected_queue_system], + ) + return QueueConfig(job_script, max_submit, selected_queue_system, queue_options) def create_local_copy(self) -> QueueConfig: @@ -111,6 +121,27 @@ def create_local_copy(self) -> QueueConfig: ) +def _check_for_overwritten_queue_system_options( + selected_queue_system: QueueSystem, + queue_system_options: List[Union[Tuple[str, str], str]], +) -> None: + def generate_dict( + option_list: List[Union[Tuple[str, str], str]] + ) -> Dict[str, List[str]]: + temp_dict: Dict[str, List[str]] = defaultdict(list) + for option_string in option_list: + if isinstance(option_string, tuple) and (option_string[0] != "MAX_RUNNING"): + temp_dict.setdefault(option_string[0], []).append(option_string[1]) + return temp_dict + + for option_name, option_values in generate_dict(queue_system_options).items(): + if len(option_values) > 1: + logging.info( + f"Overwriting QUEUE_OPTION {selected_queue_system} {option_name}:" + f" \n Old value: {option_values[0]} \n New value: {option_values[-1]}" + ) + + def _validate_torque_options(torque_options: List[Tuple[str, str]]) -> None: for option_strings in torque_options: if isinstance(option_strings, tuple): diff --git a/tests/unit_tests/config/test_queue_config.py b/tests/unit_tests/config/test_queue_config.py index aebcb3e597e..f2bfd54dff1 100644 --- a/tests/unit_tests/config/test_queue_config.py +++ b/tests/unit_tests/config/test_queue_config.py @@ -1,3 +1,4 @@ +import logging import os import os.path import stat @@ -129,3 +130,33 @@ def test_torque_queue_config_invalid_memory_pr_job(memory_with_unit_str): f.write(f"QUEUE_OPTION TORQUE MEMORY_PER_JOB {memory_with_unit_str}") with pytest.raises(ConfigValidationError): ErtConfig.from_file(filename) + + +@pytest.mark.usefixtures("use_tmpdir") +@pytest.mark.parametrize( + "queue_system, queue_system_option", + [("LSF", "LSF_SERVER"), ("SLURM", "SQUEUE"), ("TORQUE", "QUEUE")], +) +def test_overwriting_QUEUE_OPTIONS_warning( + tmp_path, monkeypatch, queue_system, queue_system_option, caplog +): + filename = "config.ert" + with open(filename, "w", encoding="utf-8") as f: + f.write("NUM_REALIZATIONS 1\n") + f.write(f"QUEUE_SYSTEM {queue_system}\n") + f.write(f"QUEUE_OPTION {queue_system} {queue_system_option} test_1\n") + f.write(f"QUEUE_OPTION {queue_system} {queue_system_option} \n") + test_site_config = tmp_path / "test_site_config.ert" + test_site_config.write_text( + "JOB_SCRIPT job_dispatch.py\n" + f"QUEUE_SYSTEM {queue_system}\n" + f"QUEUE_OPTION {queue_system} {queue_system_option} test_0\n" + ) + monkeypatch.setenv("ERT_SITE_CONFIG", str(test_site_config)) + + with caplog.at_level(logging.INFO): + ErtConfig.from_file(filename) + assert ( + f"Overwriting QUEUE_OPTION {queue_system} {queue_system_option}: \n Old value:" + " test_0 \n New value: test_1" in caplog.text + )