Make joint option for queue_name

equinor · Nov 27, 2024 · efcd654 · efcd654
1 parent ca1806e
commit efcd654
Show file tree

Hide file tree

Showing 5 changed files with 34 additions and 20 deletions.
diff --git a/src/ert/config/queue_config.py b/src/ert/config/queue_config.py
@@ -4,6 +4,7 @@
 import re
 import shutil
 from abc import abstractmethod
+from copy import copy
 from dataclasses import asdict, field, fields
 from typing import Any, Dict, List, Literal, Mapping, Optional, Union, no_type_check
 
@@ -32,6 +33,7 @@ class QueueOptions:
     max_running: pydantic.NonNegativeInt = 0
     submit_sleep: pydantic.NonNegativeFloat = 0.0
     project_code: Optional[str] = None
+    queue_name: Optional[NonEmptyString] = None
 
     @staticmethod
     def create_queue_options(
@@ -96,15 +98,13 @@ class LsfQueueOptions(QueueOptions):
     bkill_cmd: Optional[NonEmptyString] = None
     bsub_cmd: Optional[NonEmptyString] = None
     exclude_host: Optional[str] = None
-    lsf_queue: Optional[NonEmptyString] = None
     lsf_resource: Optional[str] = None
 
     @property
     def driver_options(self) -> Dict[str, Any]:
         driver_dict = asdict(self)
         driver_dict.pop("name")
         driver_dict["exclude_hosts"] = driver_dict.pop("exclude_host")
-        driver_dict["queue_name"] = driver_dict.pop("lsf_queue")
         driver_dict["resource_requirement"] = driver_dict.pop("lsf_resource")
         driver_dict.pop("submit_sleep")
         driver_dict.pop("max_running")
@@ -117,7 +117,6 @@ class TorqueQueueOptions(QueueOptions):
     qsub_cmd: Optional[NonEmptyString] = None
     qstat_cmd: Optional[NonEmptyString] = None
     qdel_cmd: Optional[NonEmptyString] = None
-    queue: Optional[NonEmptyString] = None
     memory_per_job: Optional[NonEmptyString] = None
     num_cpus_per_node: pydantic.PositiveInt = 1
     num_nodes: pydantic.PositiveInt = 1
@@ -132,7 +131,6 @@ class TorqueQueueOptions(QueueOptions):
     def driver_options(self) -> Dict[str, Any]:
         driver_dict = asdict(self)
         driver_dict.pop("name")
-        driver_dict["queue_name"] = driver_dict.pop("queue")
         driver_dict.pop("max_running")
         driver_dict.pop("submit_sleep")
         driver_dict.pop("qstat_options")
@@ -158,7 +156,6 @@ class SlurmQueueOptions(QueueOptions):
     include_host: str = ""
     memory: Optional[NonEmptyString] = None
     memory_per_cpu: Optional[NonEmptyString] = None
-    partition: Optional[NonEmptyString] = None  # aka queue_name
     squeue_timeout: pydantic.PositiveFloat = 2
     max_runtime: Optional[pydantic.NonNegativeFloat] = None
 
@@ -172,7 +169,6 @@ def driver_options(self) -> Dict[str, Any]:
         driver_dict["squeue_cmd"] = driver_dict.pop("squeue")
         driver_dict["exclude_hosts"] = driver_dict.pop("exclude_host")
         driver_dict["include_hosts"] = driver_dict.pop("include_host")
-        driver_dict["queue_name"] = driver_dict.pop("partition")
         driver_dict.pop("max_running")
         driver_dict.pop("submit_sleep")
         return driver_dict
@@ -289,6 +285,21 @@ def from_dict(cls, config_dict: ConfigDict) -> QueueConfig:
         stop_long_running = config_dict.get(ConfigKeys.STOP_LONG_RUNNING, False)
 
         _raw_queue_options = config_dict.get("QUEUE_OPTION", [])
+        for i, (q_system, *options) in enumerate(copy(_raw_queue_options)):
+            if (
+                (q_system == QueueSystem.LSF and options[0] == "LSF_QUEUE")  # noqa: PLR0916
+                or (q_system == QueueSystem.SLURM and options[0] == "PARTITION")
+                or (q_system == QueueSystem.TORQUE and options[0] == "QUEUE")
+            ):
+                ConfigWarning.deprecation_warn(
+                    f"Deprecated keyword: {options[0]} for QUEUE_OPTION {q_system}, use: "
+                    f"QUEUE_OPTION GENERIC QUEUE_NAME {options[1] if len(options) >= 2 else ''}",
+                    _raw_queue_options[i],
+                )
+                _raw_queue_options[i] = [
+                    QueueSystemWithGeneric.GENERIC,
+                    "QUEUE_NAME",
+                ] + options[1:]
         _grouped_queue_options = _group_queue_options_by_queue_system(
             _raw_queue_options
         )

diff --git a/src/everest/detached/__init__.py b/src/everest/detached/__init__.py
@@ -239,16 +239,19 @@ def start_monitor(
 _QUEUE_SYSTEMS: Mapping[Literal["LSF", "SLURM"], dict] = {
     "LSF": {
         "options": [(CK.LSF_OPTIONS, "LSF_RESOURCE")],
-        "name": "LSF_QUEUE",
+        "name": "QUEUE_NAME",
     },
     "SLURM": {
         "options": [
             (CK.SLURM_EXCLUDE_HOST_OPTION, "EXCLUDE_HOST"),
             (CK.SLURM_INCLUDE_HOST_OPTION, "INCLUDE_HOST"),
         ],
-        "name": "PARTITION",
+        "name": "QUEUE_NAME",
+    },
+    "TORQUE": {
+        "options": [CK.TORQUE_CLUSTER_LABEL, "CLUSTER_LABEL"],
+        "name": "QUEUE_NAME",
     },
-    "TORQUE": {"options": [CK.TORQUE_CLUSTER_LABEL, "CLUSTER_LABEL"], "name": "QUEUE"},
 }
 
 
@@ -284,14 +287,14 @@ def get_server_queue_options(
 
     if queue_system == QueueSystem.LSF:
         queue = LsfQueueOptions(
-            lsf_queue=ever_queue_config.name,
+            queue_name=ever_queue_config.name,
             lsf_resource=ever_queue_config.options,
         )
     elif queue_system == QueueSystem.SLURM:
         queue = SlurmQueueOptions(
             exclude_host=ever_queue_config.exclude_host,
             include_host=ever_queue_config.include_host,
-            partition=ever_queue_config.name,
+            queue_name=ever_queue_config.name,
         )
     elif queue_system == QueueSystem.TORQUE:
         queue = TorqueQueueOptions()

diff --git a/src/everest/queue_driver/queue_driver.py b/src/everest/queue_driver/queue_driver.py
@@ -7,13 +7,13 @@
 
 _LSF_OPTIONS = [
     (ConfigKeys.CORES, "MAX_RUNNING"),
-    (ConfigKeys.LSF_QUEUE_NAME, "LSF_QUEUE"),
+    (ConfigKeys.LSF_QUEUE_NAME, "QUEUE_NAME"),
     (ConfigKeys.LSF_OPTIONS, "LSF_RESOURCE"),
 ]
 
 _SLURM_OPTIONS = [
     (ConfigKeys.CORES, "MAX_RUNNING"),
-    (ConfigKeys.SLURM_QUEUE, "PARTITION"),
+    (ConfigKeys.SLURM_QUEUE, "QUEUE_NAME"),
     (ConfigKeys.SLURM_SBATCH, "SBATCH"),
     (ConfigKeys.SLURM_SCANCEL, "SCANCEL"),
     (ConfigKeys.SLURM_SCONTROL, "SCONTROL"),
@@ -31,7 +31,7 @@
     (ConfigKeys.TORQUE_QSUB_CMD, "QSUB_CMD"),
     (ConfigKeys.TORQUE_QSTAT_CMD, "QSTAT_CMD"),
     (ConfigKeys.TORQUE_QDEL_CMD, "QDEL_CMD"),
-    (ConfigKeys.TORQUE_QUEUE_NAME, "QUEUE"),
+    (ConfigKeys.TORQUE_QUEUE_NAME, "QUEUE_NAME"),
     (ConfigKeys.TORQUE_CLUSTER_LABEL, "CLUSTER_LABEL"),
     (ConfigKeys.CORES_PER_NODE, "NUM_CPUS_PER_NODE"),
     (ConfigKeys.TORQUE_MEMORY_PER_JOB, "MEMORY_PER_JOB"),

diff --git a/tests/ert/unit_tests/config/test_queue_config.py b/tests/ert/unit_tests/config/test_queue_config.py
@@ -216,7 +216,7 @@ def test_that_invalid_memory_pr_job_raises_validation_error(
 
 @pytest.mark.parametrize(
     "queue_system, queue_system_option",
-    [("LSF", "LSF_QUEUE"), ("SLURM", "SQUEUE"), ("TORQUE", "QUEUE")],
+    [("LSF", "LSF_QUEUE"), ("SLURM", "PARTITION"), ("TORQUE", "QUEUE")],
 )
 def test_that_overwriting_QUEUE_OPTIONS_warns(
     queue_system, queue_system_option, caplog
@@ -233,7 +233,7 @@ def test_that_overwriting_QUEUE_OPTIONS_warns(
             f"QUEUE_OPTION {queue_system} MAX_RUNNING 10\n",
         )
     assert (
-        f"Overwriting QUEUE_OPTION {queue_system} {queue_system_option}: \n Old value:"
+        "Overwriting QUEUE_OPTION GENERIC QUEUE_NAME: \n Old value:"
         " test_0 \n New value: test_1"
     ) in caplog.text and (
         f"Overwriting QUEUE_OPTION {queue_system} MAX_RUNNING: \n Old value:"
@@ -256,7 +256,7 @@ def test_initializing_empty_config_queue_options_resets_to_default_value(
     )
 
     if queue_system == "LSF":
-        assert config_object.queue_config.queue_options.lsf_queue is None
+        assert config_object.queue_config.queue_options.queue_name is None
     if queue_system == "SLURM":
         assert config_object.queue_config.queue_options.squeue == "squeue"
     assert config_object.queue_config.queue_options.max_running == 0

diff --git a/tests/everest/test_res_initialization.py b/tests/everest/test_res_initialization.py
@@ -94,7 +94,7 @@ def slurm_queue_system():
             ErtConfigKeys.QUEUE_OPTION: [
                 (
                     "SLURM",
-                    "PARTITION",
+                    "QUEUE_NAME",
                     "default-queue",
                 ),
                 (
@@ -263,7 +263,7 @@ def test_snake_everest_to_ert_torque(copy_test_data_to_tmp):
         ("TORQUE", "QSUB_CMD", "qsub"),
         ("TORQUE", "QSTAT_CMD", "qstat"),
         ("TORQUE", "QDEL_CMD", "qdel"),
-        ("TORQUE", "QUEUE", "permanent_8"),
+        ("TORQUE", "QUEUE_NAME", "permanent_8"),
         ("TORQUE", "MEMORY_PER_JOB", "100mb"),
         ("TORQUE", "KEEP_QSUB_OUTPUT", 1),
         ("TORQUE", "SUBMIT_SLEEP", 0.5),
@@ -365,7 +365,7 @@ def test_queue_configuration(copy_test_data_to_tmp):
 
     expected_options = [
         ("LSF", "MAX_RUNNING", 3),
-        ("LSF", "LSF_QUEUE", "mr"),
+        ("LSF", "QUEUE_NAME", "mr"),
         ("LSF", "LSF_RESOURCE", "span = 1 && select[x86 and GNU/Linux]"),
     ]