Skip to content

Commit

Permalink
Print overwritten Queue Options values to log files
Browse files Browse the repository at this point in the history
  • Loading branch information
larsevj authored Sep 18, 2023
1 parent 1d153e7 commit d5ea887
Show file tree
Hide file tree
Showing 3 changed files with 75 additions and 9 deletions.
22 changes: 13 additions & 9 deletions src/ert/config/parsing/lark_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -153,11 +153,13 @@ def _substitute_token(
if key in current:
warnings.warn(
ConfigWarning.with_context(
f"Gave up replacing in {token}.\n"
f"After replacing the value is now: {current}.\n"
f"This still contains the replacement value: {key}, "
f"which would be replaced by {val}. "
"Probably this causes a loop.",
(
f"Gave up replacing in {token}.\n"
f"After replacing the value is now: {current}.\n"
f"This still contains the replacement value: {key}, "
f"which would be replaced by {val}. "
"Probably this causes a loop."
),
token,
),
stacklevel=1,
Expand Down Expand Up @@ -260,8 +262,8 @@ def substitute_arg(
return [substitute_arglist_tuple(x) for x in arg]

raise ValueError(
f"Expected "
f"Union[FileContextToken, List[Tuple[FileContextToken]]], "
"Expected "
"Union[FileContextToken, List[Tuple[FileContextToken]]], "
f"got {arg}"
)

Expand Down Expand Up @@ -462,8 +464,10 @@ def _parse_file(file: str) -> Tree[Instruction]:
raise ConfigValidationError(
[
ErrorInfo(
message=f"Unsupported non UTF-8 character {unknown_char!r} "
f"found in file: {file!r}",
message=(
f"Unsupported non UTF-8 character {unknown_char!r} "
f"found in file: {file!r}"
),
filename=str(file),
column=0,
line=bad_line,
Expand Down
31 changes: 31 additions & 0 deletions src/ert/config/queue_config.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from __future__ import annotations

import logging
import re
import shutil
from collections import defaultdict
Expand Down Expand Up @@ -100,6 +101,15 @@ def from_dict(cls, config_dict: ConfigDict) -> QueueConfig:
):
_validate_torque_options(queue_options[QueueSystem.TORQUE])

if (
selected_queue_system != QueueSystem.LOCAL
and queue_options[selected_queue_system]
):
_check_for_overwritten_queue_system_options(
selected_queue_system,
queue_options[selected_queue_system],
)

return QueueConfig(job_script, max_submit, selected_queue_system, queue_options)

def create_local_copy(self) -> QueueConfig:
Expand All @@ -111,6 +121,27 @@ def create_local_copy(self) -> QueueConfig:
)


def _check_for_overwritten_queue_system_options(
selected_queue_system: QueueSystem,
queue_system_options: List[Union[Tuple[str, str], str]],
) -> None:
def generate_dict(
option_list: List[Union[Tuple[str, str], str]]
) -> Dict[str, List[str]]:
temp_dict: Dict[str, List[str]] = defaultdict(list)
for option_string in option_list:
if isinstance(option_string, tuple) and (option_string[0] != "MAX_RUNNING"):
temp_dict.setdefault(option_string[0], []).append(option_string[1])
return temp_dict

for option_name, option_values in generate_dict(queue_system_options).items():
if len(option_values) > 1:
logging.info(
f"Overwriting QUEUE_OPTION {selected_queue_system} {option_name}:"
f" \n Old value: {option_values[0]} \n New value: {option_values[-1]}"
)


def _validate_torque_options(torque_options: List[Tuple[str, str]]) -> None:
for option_strings in torque_options:
if isinstance(option_strings, tuple):
Expand Down
31 changes: 31 additions & 0 deletions tests/unit_tests/config/test_queue_config.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import logging
import os
import os.path
import stat
Expand Down Expand Up @@ -129,3 +130,33 @@ def test_torque_queue_config_invalid_memory_pr_job(memory_with_unit_str):
f.write(f"QUEUE_OPTION TORQUE MEMORY_PER_JOB {memory_with_unit_str}")
with pytest.raises(ConfigValidationError):
ErtConfig.from_file(filename)


@pytest.mark.usefixtures("use_tmpdir")
@pytest.mark.parametrize(
"queue_system, queue_system_option",
[("LSF", "LSF_SERVER"), ("SLURM", "SQUEUE"), ("TORQUE", "QUEUE")],
)
def test_overwriting_QUEUE_OPTIONS_warning(
tmp_path, monkeypatch, queue_system, queue_system_option, caplog
):
filename = "config.ert"
with open(filename, "w", encoding="utf-8") as f:
f.write("NUM_REALIZATIONS 1\n")
f.write(f"QUEUE_SYSTEM {queue_system}\n")
f.write(f"QUEUE_OPTION {queue_system} {queue_system_option} test_1\n")
f.write(f"QUEUE_OPTION {queue_system} {queue_system_option} \n")
test_site_config = tmp_path / "test_site_config.ert"
test_site_config.write_text(
"JOB_SCRIPT job_dispatch.py\n"
f"QUEUE_SYSTEM {queue_system}\n"
f"QUEUE_OPTION {queue_system} {queue_system_option} test_0\n"
)
monkeypatch.setenv("ERT_SITE_CONFIG", str(test_site_config))

with caplog.at_level(logging.INFO):
ErtConfig.from_file(filename)
assert (
f"Overwriting QUEUE_OPTION {queue_system} {queue_system_option}: \n Old value:"
" test_0 \n New value: test_1" in caplog.text
)

0 comments on commit d5ea887

Please sign in to comment.