enhancement(slurmctld): Signal children processes

charms/slurmctld/charmcraft.yaml

@@ -36,6 +36,10 @@ provides:
     interface: cos_agent
     limit: 1
 
+peers:
+  slurmctld-peer:
+    interface: slurmctld-peer
+
 assumes:
   - juju
 
@@ -67,7 +71,7 @@ config:
   options:
     cluster-name:
       type: string
-      default: osd-cluster
+      default: "osd-cluster"
       description: |
         Name to be recorded in database for jobs from this cluster.
 
@@ -97,6 +101,7 @@ config:
         ConstrainDevices=yes
         ConstrainRAMSpace=yes
         ConstrainSwapSpace=yes
+        SignalChildrenProcesses=yes
       description: |
         User supplied configuration for `cgroup.conf`.
 

charms/slurmctld/src/charm.py

@@ -9,7 +9,8 @@
 import subprocess
 from typing import Any, Dict, List, Optional, Union
 
-from constants import CHARM_MAINTAINED_SLURM_CONF_PARAMETERS
+from constants import CHARM_MAINTAINED_SLURM_CONF_PARAMETERS, PEER_RELATION
+from exceptions import IngressAddressUnavailableError
 from interface_slurmd import (
     PartitionAvailableEvent,
     PartitionUnavailableEvent,
@@ -302,8 +303,8 @@ def _assemble_slurmctld_parameters() -> dict[str, Any]:
             }
 
         slurm_conf = SlurmConfig(
-            ClusterName=self.cluster_name,
-            SlurmctldAddr=self._slurmd_ingress_address,
+            ClusterName=self._cluster_name,
+            SlurmctldAddr=self._ingress_address,
             SlurmctldHost=[self._slurmctld.hostname],
             SlurmctldParameters=_assemble_slurmctld_parameters(),
             ProctrackType="proctrack/linuxproc" if is_container() else "proctrack/cgroup",
@@ -377,7 +378,7 @@ def _resume_nodes(self, nodelist: List[str]) -> None:
         self._slurmctld.scontrol(update_cmd)
 
     @property
-    def cluster_name(self) -> str:
+    def _cluster_name(self) -> str:
         """Return the cluster name."""
         cluster_name = "charmedhpc"
         if cluster_name_from_config := self.config.get("cluster-name"):
@@ -403,12 +404,13 @@ def hostname(self) -> str:
         return self._slurmctld.hostname
 
     @property
-    def _slurmd_ingress_address(self) -> str:
-        """Return the ingress_address from the slurmd relation if it exists."""
-        ingress_address = ""
-        if binding := self.model.get_binding("slurmd"):
-            ingress_address = f"{binding.network.ingress_address}"
-        return ingress_address
+    def _ingress_address(self) -> str:
+        """Return the ingress_address from the peer relation if it exists."""
+        if (peer_binding := self.model.get_binding(PEER_RELATION)) is not None:
+            ingress_address = f"{peer_binding.network.ingress_address}"
+            logger.debug(f"Slurmctld ingress_address: {ingress_address}")
+            return ingress_address
+        raise IngressAddressUnavailableError("Ingress address unavailable")
 
     @property
     def slurm_installed(self) -> bool:

charms/slurmctld/src/constants.py

@@ -3,6 +3,8 @@
 
 """This module provides constants for the slurmctld-operator charm."""
 
+PEER_RELATION = "slurmctld-peer"
+
 CHARM_MAINTAINED_SLURM_CONF_PARAMETERS = {
     "AuthAltParameters": {"jwt_key": "/var/lib/slurm/checkpoint/jwt_hs256.key"},
     "AuthAltTypes": ["auth/jwt"],

charms/slurmctld/src/exceptions.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2024 Omnivector Corp
+# See LICENSE file for licensing details.
+
+"""Custom exceptions for the slurmctld operator."""
+
+
+class IngressAddressUnavailableError(Exception):
+    """Exception raised when a slurm operation failed."""
+
+    @property
+    def message(self) -> str:
+        """Return message passed as argument to exception."""
+        return self.args[0]

charms/slurmctld/src/interface_slurmd.py

@@ -89,7 +89,6 @@ def _on_relation_created(self, event: RelationCreatedEvent) -> None:
             {
                 "munge_key": self._charm.get_munge_key(),
                 "slurmctld_host": self._charm.hostname,
-                "cluster_name": self._charm.cluster_name,
                 "nhc_params": health_check_params,
             }
         )

charms/slurmctld/tests/unit/test_charm.py

@@ -34,12 +34,12 @@ def setUp(self):
         self.harness.begin()
 
     def test_cluster_name(self) -> None:
-        """Test that the cluster_name property works."""
-        self.assertEqual(self.harness.charm.cluster_name, "osd-cluster")
+        """Test that the _cluster_name property works."""
+        self.assertEqual(self.harness.charm._cluster_name, "osd-cluster")
 
     def test_cluster_info(self) -> None:
         """Test the cluster_info property works."""
-        self.assertEqual(type(self.harness.charm.cluster_name), str)
+        self.assertEqual(type(self.harness.charm._cluster_name), str)
 
     def test_is_slurm_installed(self) -> None:
         """Test that the is_slurm_installed method works."""
@@ -160,6 +160,7 @@ def test_on_slurmdbd_unavailable(self) -> None:
     def test_get_user_supplied_parameters(self, *_) -> None:
         """Test that user supplied parameters are parsed correctly."""
         self.harness.add_relation("slurmd", "slurmd")
+        self.harness.add_relation("slurmctld-peer", self.harness.charm.app.name)
         self.harness.update_config(
             {"slurm-conf-parameters": "JobAcctGatherFrequency=task=30,network=40"}
         )

charms/slurmd/src/interface_slurmctld.py

@@ -24,30 +24,26 @@ class SlurmctldAvailableEvent(EventBase):
     def __init__(
         self,
         handle,
-        cluster_name,
         munge_key,
         nhc_params,
         slurmctld_host,
     ):
         super().__init__(handle)
 
-        self.cluster_name = cluster_name
         self.munge_key = munge_key
         self.nhc_params = nhc_params
         self.slurmctld_host = slurmctld_host
 
     def snapshot(self):
         """Snapshot the event data."""
         return {
-            "cluster_name": self.cluster_name,
             "munge_key": self.munge_key,
             "nhc_params": self.nhc_params,
             "slurmctld_host": self.slurmctld_host,
         }
 
     def restore(self, snapshot):
         """Restore the snapshot of the event data."""
-        self.cluster_name = snapshot.get("cluster_name")
         self.munge_key = snapshot.get("munge_key")
         self.nhc_params = snapshot.get("nhc_params")
         self.slurmctld_host = snapshot.get("slurmctld_host")