feat: Add logrotate to rotate slurm logs

Signed-off-by: Jason C. Nucciarone <[email protected]>

overlays/templates/logrotate.conf.tmpl

@@ -0,0 +1,29 @@
+# Slurm logs (slurmctld, slurmd and slurmdbd).
+$SNAP_COMMON/var/log/slurm/*.log {
+    # Rotate daemon logs every week; keep 4 weeks worth on backlog.
+    weekly
+    rotate 4
+    size=5M
+    create 640 slurm root
+    missingok
+    nocopytruncate
+    nomail
+    notifempty
+    noolddir
+    sharedscripts
+
+    # Compress logs - they can get quite large depending on demand.
+    compress
+    delaycompress
+    compresscmd $SNAP/bin/bzip2
+    compressext .bz2
+
+    # Re-read log level and reopen log files using `SIGUSR2` signal.
+    # https://slurm.schedmd.com/slurmctld.html#SECTION_SIGNALS
+    postrotate
+        $SNAP/usr/bin/pkill -x --signal SIGUSR2 slurmctld
+        $SNAP/usr/bin/pkill -x --signal SIGUSR2 slurmd
+        $SNAP/usr/bin/pkill -x --signal SIGUSR2 slurmdbd
+        exit 0
+    endscript
+}

slurmhelpers/hooks.py

@@ -15,6 +15,7 @@
 """Hooks for the Slurm snap."""
 
 import logging
+import os
 from pathlib import Path
 
 from snaphelpers import Snap
@@ -36,6 +37,7 @@ def _setup_dirs(snap: Snap) -> None:
     run = Path(snap.paths.common) / "run"
     for directory in [
         # etc - configuration files
+        etc / "logrotate",
         etc / "munge",
         etc / "slurm",
         etc / "slurm" / "plugstack.conf.d",
@@ -67,6 +69,17 @@ def _setup_dirs(snap: Snap) -> None:
     (run / "munge").chmod(0o755)
 
 
+def _setup_logrotate(snap: Snap) -> None:
+    """Configure `logrotate` for the Slurm.
+
+    Args:
+        snap: The Snap instance.
+    """
+    tmpl = (snap.paths.snap / "templates" / "logrotate.conf.tmpl").read_text()
+    config = os.path.expandvars(tmpl)
+    (snap.paths.common / "etc" / "logrotate" / "logrotate.conf").write_text(config)
+
+
 def install(snap: Snap) -> None:
     """Install hook for the Slurm snap.
 
@@ -83,6 +96,7 @@ def install(snap: Snap) -> None:
 
     logging.info("Executing snap `install` hook.")
     _setup_dirs(snap)
+    _setup_logrotate(snap)
 
     logging.info("Setting default global configuration for snap.")
     munge.max_thread_count = 1

snap/snapcraft.yaml

@@ -37,6 +37,11 @@ environment:
   PATH: $SNAP/usr/sbin:$SNAP/usr/bin:$SNAP/sbin:$SNAP/bin:$SNAP/usr/local/bin:$SNAP/usr/local/sbin:$PATH
 
 apps:
+  logrotate:
+    command: usr/sbin/logrotate $SNAP_COMMON/etc/logrotate/logrotate.conf
+    daemon: oneshot
+    timer: 00:00  # Run `logrotate` once everyday at midnight.
+
   munged:
     command: sbin/munged.wrapper
     daemon: simple
@@ -162,6 +167,7 @@ parts:
     plugin: dump
     organize:
       "overlays/sbin/*": sbin/
+      "overlays/templates/*": templates/
 
   hooks:
     source: .
@@ -180,6 +186,14 @@ parts:
       craftctl default
       snap-helpers write-hooks
 
+  logrotate:
+    plugin: nil
+    build-attributes: [enable-patchelf]
+    stage-packages:
+      - logrotate
+      - procps  # `pkill`
+      - bzip2
+
   munge:
     plugin: autotools
     source: https://github.com/dun/munge.git

tests/unit/test_hooks.py

@@ -20,8 +20,60 @@
   - `configure`
 """
 
+import os
+import pathlib
+
 from slurmhelpers import hooks
 
+mock_logrotate_config = """
+$SNAP_COMMON/var/log/slurm/*.log {
+    weekly
+    rotate 4
+    size=5M
+    create 640 slurm root
+    missingok
+    nocopytruncate
+    nomail
+    notifempty
+    noolddir
+    sharedscripts
+    compress
+    delaycompress
+    compresscmd $SNAP/bin/bzip2
+    compressext .bz2
+    postrotate
+        $SNAP/usr/bin/pkill -x --signal SIGUSR2 slurmctld
+        $SNAP/usr/bin/pkill -x --signal SIGUSR2 slurmd
+        $SNAP/usr/bin/pkill -x --signal SIGUSR2 slurmdbd
+        exit 0
+    endscript
+}
+""".strip()
+target_logrotate_config = """
+/var/snap/slurm/common/var/log/slurm/*.log {
+    weekly
+    rotate 4
+    size=5M
+    create 640 slurm root
+    missingok
+    nocopytruncate
+    nomail
+    notifempty
+    noolddir
+    sharedscripts
+    compress
+    delaycompress
+    compresscmd /snap/slurm/x1/bin/bzip2
+    compressext .bz2
+    postrotate
+        /snap/slurm/x1/usr/bin/pkill -x --signal SIGUSR2 slurmctld
+        /snap/slurm/x1/usr/bin/pkill -x --signal SIGUSR2 slurmd
+        /snap/slurm/x1/usr/bin/pkill -x --signal SIGUSR2 slurmdbd
+        exit 0
+    endscript
+}
+""".strip()
+
 
 class TestHooks:
     """Test the hooks and relevant branches from slurmhelpers.hooks."""
@@ -31,10 +83,23 @@ def test_install_hook(self, mocker, snap) -> None:
         mocker.patch("pathlib.Path.chmod")
         mocker.patch("pathlib.Path.mkdir")
         mocker.patch("pathlib.Path.touch")
+        mocker.patch("pathlib.Path.read_text")
+        mocker.patch("pathlib.Path.write_text")
         mocker.patch("pathlib.Path.write_bytes")
         mocker.patch("dotenv.set_key")
         hooks.install(snap)
 
+    def test_setup_logrotate(self, mocker, snap) -> None:
+        """Test `_setup_logrotate` helper method."""
+        mocker.patch("pathlib.Path.read_text", return_value=mock_logrotate_config)
+        mocker.patch("pathlib.Path.write_text")
+        os.environ["SNAP"] = "/snap/slurm/x1"
+        os.environ["SNAP_COMMON"] = "/var/snap/slurm/common"
+
+        # Assert that logrotate template was rendered correctly.
+        hooks._setup_logrotate(snap)
+        pathlib.Path.write_text.assert_called_once_with(target_logrotate_config)
+
     def test_configure_hook(self, mocker, snap) -> None:
         """Test `configure` hook."""
         mocker.patch("slurmhelpers.models.Munge.update_config")