Add integration tests for slurmd operator (#8)

* initial integration test

* almost working build and deploy integration test

* integration test fix for bug with slurmd and slurmctld where relation has to join after slurmctld is ready

* cleaned up initial test

* working deploy integration test

* mpi integration test + mpi bug fix

* mpi_install working test

* latest working tests with standards update

* add license header to pipelines

* Update tests/integration/test_charm.py

Co-authored-by: Jason Nucciarone <[email protected]>

* updated integration tests and actions

* add missing spacing and license header

* enhance: add toml license header and remove mpi action test

* fix: rearrange test order and mark slurmd active test as xfail due to bug

* bugfix: Add missing line continuation for coverage

---------

Co-authored-by: Jason Nucciarone <[email protected]>

.github/workflows/ci.yaml

@@ -0,0 +1,75 @@
+# Copyright 2023 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: slurmd charm tests
+on:
+  workflow_call:
+  pull_request:
+
+jobs:
+  inclusive-naming-check:
+    name: Inclusive naming check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: woke
+        uses: get-woke/woke-action@v0
+        with:
+          # Cause the check to fail on any broke rules
+          fail-on-error: true
+
+  lint:
+    name: Lint
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install dependencies
+        run: python3 -m pip install tox
+      - name: Run linters
+        run: tox -e lint
+
+  unit-test:
+    name: Unit tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install dependencies
+        run: python3 -m pip install tox
+      - name: Run tests
+        run: tox -e unit
+
+  integration-test:
+    name: Integration tests (LXD)
+    runs-on: ubuntu-latest
+    needs:
+      - inclusive-naming-check
+      - lint
+      - unit-test
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install dependencies
+        run: python3 -m pip install tox
+      - name: Setup operator environment
+        uses: charmed-kubernetes/actions-operator@main
+        with:
+          provider: lxd
+          # Juju channel should eventually be updated to 3.0/stable
+          juju-channel: 2.9/stable
+      - name: Run tests
+        run: tox -e integration

pyproject.toml

@@ -1,5 +1,16 @@
 # Copyright 2023 Canonical Ltd.
-# See LICENSE file for licensing details.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 [tool.codespell]
 skip = "./build,./lib,./venv,./icon.svg"

src/charm.py

@@ -534,8 +534,8 @@ def mpi_install(self, event):
         """Install MPI (mpich)."""
         self._slurm_manager.mpi.install()
 
-        if self._slurm_manager.mpi.installed():
-            event.set_results({"installation": "Successfull."})
+        if self._slurm_manager.mpi.installed:
+            event.set_results({'installation': 'Successfull.'})
         else:
             event.fail(message="Error installing mpich. Check the logs.")
 

tests/integration/conftest.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+# Copyright 2023 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Configure integration test run."""
+
+import pathlib
+
+from pytest import fixture
+from pytest_operator.plugin import OpsTest
+
+from helpers import ETCD, NHC, VERSION
+
+
+@fixture(scope="module")
+async def slurmd_charm(ops_test: OpsTest):
+    charm = await ops_test.build_charm(".")
+    return charm
+
+def pytest_sessionfinish(session, exitstatus) -> None:
+    """Clean up repository after test session has completed."""
+    pathlib.Path(ETCD).unlink(missing_ok=True)
+    pathlib.Path(NHC).unlink(missing_ok=True)
+    pathlib.Path(VERSION).unlink(missing_ok=True)

tests/integration/helpers.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+# Copyright 2023 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Helpers for the slurmd integration tests."""
+
+import logging
+import pathlib
+import shlex
+import subprocess
+
+from typing import Dict
+from urllib import request
+
+logger = logging.getLogger(__name__)
+
+ETCD = "etcd-v3.5.0-linux-amd64.tar.gz"
+ETCD_URL = f"https://github.com/etcd-io/etcd/releases/download/v3.5.0/{ETCD}"
+NHC = "lbnl-nhc-1.4.3.tar.gz"
+NHC_URL = f"https://github.com/mej/nhc/releases/download/1.4.3/{NHC}"
+VERSION = "version"
+VERSION_NUM = subprocess.run(
+    shlex.split("git describe --always"), stdout=subprocess.PIPE, text=True
+).stdout.strip("\n")
+
+
+def get_slurmctld_res() -> Dict[str, pathlib.Path]:
+    """Get slurmctld resources needed for charm deployment."""
+    if not (version := pathlib.Path(VERSION)).exists():
+        logger.info(f"Setting resource {VERSION} to value {VERSION_NUM}...")
+        version.write_text(VERSION_NUM)
+    if not (etcd := pathlib.Path(ETCD)).exists():
+        logger.info(f"Getting resource {ETCD} from {ETCD_URL}...")
+        request.urlretrieve(ETCD_URL, etcd)
+
+    return {"etcd": etcd}
+
+def get_slurmd_res() -> Dict[str, pathlib.Path]:
+    """Get slurmd resources needed for charm deployment."""
+    if not (nhc := pathlib.Path(NHC)).exists():
+        logger.info(f"Getting resource {NHC} from {NHC_URL}...")
+        request.urlretrieve(NHC_URL, nhc)
+
+    return {"nhc": nhc}

tests/integration/test_charm.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+# Copyright 2023 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Test slurmd charm against other SLURM charms in the latest/edge channel."""
+
+import asyncio
+import pytest
+
+from helpers import (
+    get_slurmd_res,
+    get_slurmctld_res,
+)
+
+from pathlib import Path
+from pytest_operator.plugin import OpsTest
+from tenacity import retry
+from tenacity.stop import stop_after_attempt
+from tenacity.wait import wait_exponential as wexp
+
+SERIES = ["focal"]
+SLURMD = "slurmd"
+SLURMDBD = "slurmdbd"
+SLURMCTLD = "slurmctld"
+
+
+@pytest.mark.abort_on_fail
+@pytest.mark.parametrize("series", SERIES)
+@pytest.mark.skip_if_deployed
+async def test_build_and_deploy(ops_test: OpsTest, series: str, slurmd_charm):
+    """Test that the slurmd charm can stabilize against slurmctld, slurmdbd and percona."""
+    res_slurmd = get_slurmd_res()
+    res_slurmctld = get_slurmctld_res()
+
+    # Fetch edge from charmhub for slurmctld, slurmdbd and percona and deploy
+    await asyncio.gather(
+        ops_test.model.deploy(
+            SLURMCTLD,
+            application_name=SLURMCTLD,
+            channel="edge",
+            num_units=1,
+            resources=res_slurmctld,
+            series=series,
+        ),
+        ops_test.model.deploy(
+            SLURMDBD,
+            application_name=SLURMDBD,
+            channel="edge",
+            num_units=1,
+            series=series,
+        ),
+        ops_test.model.deploy(
+            "percona-cluster",
+            application_name="mysql",
+            channel="edge",
+            num_units=1,
+            series="bionic",
+        ),
+    )
+
+    # Attach ETCD resource to the slurmctld controller
+    await ops_test.juju("attach-resource", SLURMCTLD, f"etcd={res_slurmctld['etcd']}")
+
+    # Add slurmdbd integration to slurmctld
+    await ops_test.model.relate(SLURMCTLD, SLURMDBD)
+
+    # Add mysql integration to slurmdbd
+    await ops_test.model.relate(SLURMDBD, "mysql")
+
+    # IMPORTANT: It's possible for slurmd to be stuck waiting for slurmctld despite slurmctld and slurmdbd
+    # available. Relation between slurmd and slurmctld has to be added after slurmctld is ready
+    # otherwise risk running into race-condition type behavior.
+    await ops_test.model.wait_for_idle(apps=[SLURMCTLD], status="blocked", timeout=1000)
+
+    # Build and Deploy Slurmd
+    await ops_test.model.deploy(
+            str(await slurmd_charm),
+            application_name=SLURMD,
+            num_units=1,
+            resources=res_slurmd,
+            series=series,
+    )
+
+    # Attach NHC resource to the slurmd controller
+    await ops_test.juju("attach-resource", SLURMD, f"nhc={res_slurmd['nhc']}")
+
+    # Add slurmctld integration to slurmd
+    await ops_test.model.relate(SLURMD, SLURMCTLD)
+
+    # Reduce the update status frequency to accelerate the triggering of deferred events.
+    async with ops_test.fast_forward():
+        await ops_test.model.wait_for_idle(apps=[SLURMD], status="active", timeout=1000)
+        assert ops_test.model.applications[SLURMD].units[0].workload_status == "active"
+
+
+@pytest.mark.abort_on_fail
+async def test_munge_is_active(ops_test: OpsTest):
+    """Test that munge is active."""
+    unit = ops_test.model.applications[SLURMD].units[0]
+    cmd_res = (await unit.ssh(command="systemctl is-active munge")).strip("\n")
+    assert cmd_res == "active"
+
+
+# IMPORTANT: Currently there is a bug where slurmd can reach active status despite the
+# systemd service failing.
+@pytest.mark.xfail
+@pytest.mark.abort_on_fail
+async def test_slurmd_is_active(ops_test: OpsTest):
+    """Test that slurmd is active."""
+    unit = ops_test.model.applications[SLURMD].units[0]
+    cmd_res = (await unit.ssh(command="systemctl is-active slurmd")).strip("\n")
+    assert cmd_res == "active"

tests/unit/test_charm.py

@@ -1,3 +1,20 @@
+#!/usr/bin/env python3
+# Copyright 2023 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Test default charm events such as upgrade charm, install, etc."""
+
 import unittest
 from unittest.mock import PropertyMock, patch