diff --git a/docs/everest/config_generated.rst b/docs/everest/config_generated.rst index 2b26169ae75..20ffb487c00 100644 --- a/docs/everest/config_generated.rst +++ b/docs/everest/config_generated.rst @@ -870,25 +870,6 @@ Type: *Optional[SimulatorConfig]* Simulation settings -**name (optional)** - Type: *Optional[str]* - - Specifies which queue to use - - -**cores (optional)** - Type: *Optional[PositiveInt]* - - Defines the number of simultaneously running forward models. - - When using queue system lsf, this corresponds to number of nodes used at one - time, whereas when using the local queue system, cores refers to the number of - cores you want to use on your system. - - This number is specified in Ert as MAX_RUNNING. - - - **cores_per_node (optional)** Type: *Optional[PositiveInt]* @@ -906,32 +887,6 @@ Simulation settings Whether the batch folder for a successful simulation needs to be deleted. -**exclude_host (optional)** - Type: *Optional[str]* - - Comma separated list of nodes that should be - excluded from the slurm run. - - -**include_host (optional)** - Type: *Optional[str]* - - Comma separated list of nodes that - should be included in the slurm run - - -**max_memory (optional)** - Type: *Optional[str]* - - Maximum memory usage for a slurm job. - - -**max_memory_cpu (optional)** - Type: *Optional[str]* - - Maximum memory usage per cpu for a slurm job. - - **max_runtime (optional)** Type: *Optional[NonNegativeInt]* @@ -941,18 +896,10 @@ Simulation settings -**options (optional)** - Type: *Optional[str]* - - Used to specify options to LSF. - Examples to set memory requirement is: - * rusage[mem=1000] - - **queue_system (optional)** - Type: *Optional[Literal['lsf', 'local', 'slurm', 'torque']]* + Type: *Optional[LocalQueueOptions, LsfQueueOptions, SlurmQueueOptions, TorqueQueueOptions]* - Defines which queue system the everest server runs on. + Defines which queue system the everest submits jobs to **resubmit_limit (optional)** @@ -968,54 +915,6 @@ Simulation settings If not specified, a default value of 1 will be used. -**sbatch (optional)** - Type: *Optional[str]* - - sbatch executable to be used by the slurm queue interface. - - -**scancel (optional)** - Type: *Optional[str]* - - scancel executable to be used by the slurm queue interface. - - -**scontrol (optional)** - Type: *Optional[str]* - - scontrol executable to be used by the slurm queue interface. - - -**sacct (optional)** - Type: *Optional[str]* - - sacct executable to be used by the slurm queue interface. - - -**squeue (optional)** - Type: *Optional[str]* - - squeue executable to be used by the slurm queue interface. - - -**server (optional)** - Type: *Optional[str]* - - Name of LSF server to use. This option is deprecated and no longer required - - -**slurm_timeout (optional)** - Type: *Optional[int]* - - Timeout for cached status used by the slurm queue interface - - -**squeue_timeout (optional)** - Type: *Optional[int]* - - Timeout for cached status used by the slurm queue interface. - - **enable_cache (optional)** Type: *bool* @@ -1031,72 +930,6 @@ Simulation settings optimizer. -**qsub_cmd (optional)** - Type: *Optional[str]* - - The submit command - - -**qstat_cmd (optional)** - Type: *Optional[str]* - - The query command - - -**qdel_cmd (optional)** - Type: *Optional[str]* - - The kill command - - -**qstat_options (optional)** - Type: *Optional[str]* - - Options to be supplied to the qstat command. This defaults to -x, which tells the qstat command to include exited processes. - - -**cluster_label (optional)** - Type: *Optional[str]* - - The name of the cluster you are running simulations in. - - -**memory_per_job (optional)** - Type: *Optional[str]* - - You can specify the amount of memory you will need for running your job. This will ensure that not too many jobs will run on a single shared memory node at once, possibly crashing the compute node if it runs out of memory. - You can get an indication of the memory requirement by watching the course of a local run using the htop utility. Whether you should set the peak memory usage as your requirement or a lower figure depends on how simultaneously each job will run. - The option to be supplied will be used as a string in the qsub argument. You must specify the unit, either gb or mb. - - - -**keep_qsub_output (optional)** - Type: *Optional[int]* - - Set to 1 to keep error messages from qsub. Usually only to be used if somethign is seriously wrong with the queue environment/setup. - - -**submit_sleep (optional)** - Type: *Optional[float]* - - To avoid stressing the TORQUE/PBS system you can instruct the driver to sleep for every submit request. The argument to the SUBMIT_SLEEP is the number of seconds to sleep for every submit, which can be a fraction like 0.5 - - -**queue_query_timeout (optional)** - Type: *Optional[int]* - - - The driver allows the backend TORQUE/PBS system to be flaky, i.e. it may intermittently not respond and give error messages when submitting jobs or asking for job statuses. The timeout (in seconds) determines how long ERT will wait before it will give up. Applies to job submission (qsub) and job status queries (qstat). Default is 126 seconds. - ERT will do exponential sleeps, starting at 2 seconds, and the provided timeout is a maximum. Let the timeout be sums of series like 2+4+8+16+32+64 in order to be explicit about the number of retries. Set to zero to disallow flakyness, setting it to 2 will allow for one re-attempt, and 6 will give two re-attempts. Example allowing six retries: - - - -**project_code (optional)** - Type: *Optional[str]* - - String identifier used to map hardware resource usage to a project or account. The project or account does not have to exist. - - install_jobs (optional) ----------------------- @@ -1258,32 +1091,10 @@ requirements of the forward models. -**exclude_host (optional)** - Type: *Optional[str]* - - Comma separated list of nodes that should be - excluded from the slurm run - - -**include_host (optional)** - Type: *Optional[str]* - - Comma separated list of nodes that - should be included in the slurm run - - -**options (optional)** - Type: *Optional[str]* - - Used to specify options to LSF. - Examples to set memory requirement is: - * rusage[mem=1000] - - **queue_system (optional)** - Type: *Optional[Literal['lsf', 'local', 'slurm']]* + Type: *Optional[LocalQueueOptions, LsfQueueOptions, SlurmQueueOptions, TorqueQueueOptions]* - Defines which queue system the everest server runs on. + Defines which queue system the everest submits jobs to diff --git a/src/ert/config/queue_config.py b/src/ert/config/queue_config.py index f4f5a1d0b58..0f9cb9bb255 100644 --- a/src/ert/config/queue_config.py +++ b/src/ert/config/queue_config.py @@ -5,16 +5,13 @@ import re import shutil from abc import abstractmethod -from dataclasses import field -from copy import copy - -from typing import Any, Literal, Mapping, Optional, no_type_check +from collections.abc import Mapping +from typing import Any, Literal, no_type_check import pydantic -from pydantic import Field, field_validator +from pydantic import Field, constr, field_validator from pydantic.dataclasses import dataclass from pydantic_core.core_schema import ValidationInfo -from typing_extensions import Annotated from .parsing import ( BaseModelWithContextSupport, @@ -29,7 +26,7 @@ logger = logging.getLogger(__name__) -NonEmptyString = Annotated[str, pydantic.StringConstraints(min_length=1)] +NonEmptyString = constr(min_length=1) def activate_script() -> str: @@ -121,7 +118,6 @@ class LsfQueueOptions(QueueOptions): lsf_queue: NonEmptyString | None = None lsf_resource: str | None = None - @property def driver_options(self) -> dict[str, Any]: driver_dict = self.model_dump(exclude={"name", "submit_sleep", "max_running"}) @@ -290,7 +286,7 @@ class QueueConfig: queue_options: ( LsfQueueOptions | TorqueQueueOptions | SlurmQueueOptions | LocalQueueOptions ) = pydantic.Field(default_factory=LocalQueueOptions, discriminator="name") - queue_options_test_run: LocalQueueOptions = field(default_factory=LocalQueueOptions) + queue_options_test_run: LocalQueueOptions = Field(default_factory=LocalQueueOptions) stop_long_running: bool = False @no_type_check diff --git a/src/ert/gui/simulation/experiment_panel.py b/src/ert/gui/simulation/experiment_panel.py index 5923a6d5aef..b92abaa28ba 100644 --- a/src/ert/gui/simulation/experiment_panel.py +++ b/src/ert/gui/simulation/experiment_panel.py @@ -3,7 +3,6 @@ import os import platform from collections import OrderedDict -from dataclasses import fields from datetime import datetime from pathlib import Path from queue import SimpleQueue @@ -379,10 +378,10 @@ def populate_clipboard_debug_info(self) -> None: if isinstance(self.get_current_experiment_type(), SingleTestRun): queue_opts = LocalQueueOptions(max_running=1) - for field in fields(queue_opts): - field_value = getattr(queue_opts, field.name) + for name in queue_opts.model_fields: + field_value = getattr(queue_opts, name) if field_value is not None: - kv[field.name.replace("_", " ").capitalize()] = str(field_value) + kv[name.replace("_", " ").capitalize()] = str(field_value) kv["**Status**"] = "" kv["Trace ID"] = get_trace_id() diff --git a/tests/ert/unit_tests/config/config_dict_generator.py b/tests/ert/unit_tests/config/config_dict_generator.py index e0485a1ffda..3f0b522eb33 100644 --- a/tests/ert/unit_tests/config/config_dict_generator.py +++ b/tests/ert/unit_tests/config/config_dict_generator.py @@ -4,9 +4,9 @@ import os.path import stat from collections import defaultdict -from dataclasses import dataclass, fields +from dataclasses import dataclass from pathlib import Path -from typing import Any, Literal +from typing import Any, Literal, get_args, get_origin from warnings import filterwarnings import hypothesis.strategies as st @@ -134,37 +134,52 @@ def memory_with_unit_lsf(draw): def valid_queue_options(queue_system: str): return [ - field.name.upper() - for field in fields( - queue_systems_and_options[QueueSystemWithGeneric(queue_system)] - ) - if field.name != "name" + name.upper() + for name in queue_systems_and_options[ + QueueSystemWithGeneric(queue_system) + ].model_fields + if name != "name" ] +def has_base_type( + field_type, base_type: type[int] | bool | type[str] | type[float] +) -> bool: + if field_type is base_type: + return True + origin = get_origin(field_type) + if origin: + args = get_args(field_type) + if any(arg is base_type for arg in args): + return True + return any(has_base_type(arg, base_type) for arg in args) + return False + + queue_options_by_type: dict[str, dict[str, list[str]]] = defaultdict(dict) for system, options in queue_systems_and_options.items(): queue_options_by_type["string"][system.name] = [ - field.name.upper() - for field in fields(options) - if ("String" in field.type or "str" in field.type) - and "memory" not in field.name + name.upper() + for name, field in options.model_fields.items() + if has_base_type(field.annotation, str) and "memory" not in name ] queue_options_by_type["bool"][system.name] = [ - field.name.upper() for field in fields(options) if field.type == "bool" + name.upper() + for name, field in options.model_fields.items() + if has_base_type(field.annotation, bool) ] queue_options_by_type["posint"][system.name] = [ - field.name.upper() - for field in fields(options) - if "PositiveInt" in field.type or "NonNegativeInt" in field.type + name.upper() + for name, field in options.model_fields.items() + if has_base_type(field.annotation, int) ] queue_options_by_type["posfloat"][system.name] = [ - field.name.upper() - for field in fields(options) - if "NonNegativeFloat" in field.type or "PositiveFloat" in field.type + name.upper() + for name, field in options.model_fields.items() + if has_base_type(field.annotation, float) ] queue_options_by_type["memory"][system.name] = [ - field.name.upper() for field in fields(options) if "memory" in field.name + name.upper() for name in options.model_fields if "memory" in name ] diff --git a/tests/ert/unit_tests/config/test_queue_config.py b/tests/ert/unit_tests/config/test_queue_config.py index bfa26641652..adbdbdeaec5 100644 --- a/tests/ert/unit_tests/config/test_queue_config.py +++ b/tests/ert/unit_tests/config/test_queue_config.py @@ -15,7 +15,6 @@ from ert.config.queue_config import ( LocalQueueOptions, LsfQueueOptions, - QueueOptions, SlurmQueueOptions, TorqueQueueOptions, ) @@ -520,5 +519,5 @@ def test_default_activate_script_generation(expected, monkeypatch, venv): monkeypatch.setenv("VIRTUAL_ENV", venv) else: monkeypatch.delenv("VIRTUAL_ENV", raising=False) - options = QueueOptions(name="local") + options = LocalQueueOptions() assert options.activate_script == expected diff --git a/tests/everest/test_config_validation.py b/tests/everest/test_config_validation.py index af23d2bad46..57481f36335 100644 --- a/tests/everest/test_config_validation.py +++ b/tests/everest/test_config_validation.py @@ -141,13 +141,11 @@ def test_that_max_runtime_errors_only_on_negative(): def test_that_invalid_queue_system_errors(): - with pytest.raises(ValueError) as e: + with pytest.raises( + ValueError, match="does not match .*'local',.*'lsf',.*'slurm', .*'torque'" + ): EverestConfig.with_defaults(simulator={"queue_system": {"name": "docal"}}) - assert has_error( - e.value, match="does not match .*'lsf', .*'local', .*'slurm', .*'torque'" - ) - @pytest.mark.parametrize( ["cores", "expected_error"], [(0, False), (-1, True), (1, False)] diff --git a/tests/everest/test_res_initialization.py b/tests/everest/test_res_initialization.py index f74e58515c3..9bbbec734c0 100644 --- a/tests/everest/test_res_initialization.py +++ b/tests/everest/test_res_initialization.py @@ -29,7 +29,7 @@ [ { "name": "torque", - "queue_name": "permanent_8", + "queue": "permanent_8", "qsub_cmd": "qsub", "qstat_cmd": "qstat", "qdel_cmd": "qdel", @@ -52,7 +52,7 @@ [ { "name": "slurm", - "queue_name": "default-queue", + "partition": "default-queue", "memory": "1000M", "exclude_host": "host1,host2,host3,host4", "include_host": "host5,host6,host7,host8", @@ -73,7 +73,7 @@ [ { "name": "lsf", - "queue_name": "mr", + "lsf_queue": "mr", "lsf_resource": "span = 1 && select[x86 and GNU/Linux]", }, { diff --git a/tests/everest/test_yaml_parser.py b/tests/everest/test_yaml_parser.py index 436a594c0b3..8849a08350d 100644 --- a/tests/everest/test_yaml_parser.py +++ b/tests/everest/test_yaml_parser.py @@ -49,6 +49,7 @@ def test_read_file(tmp_path, monkeypatch): ConfigKeys.MODEL, ConfigKeys.SIMULATOR, ConfigKeys.OPTIMIZATION, + ConfigKeys.EVERSERVER, ConfigKeys.DEFINITIONS, ConfigKeys.CONFIGPATH, ]