tests: Add support for using local charms in integration tests

* Added `--use-local` test option to direct pytest to use local charms.
    Invoked with `tox run -e integration -- --use-local`
* pytest fixtures for charms return a generic string if a local
    copy of the charm does not exist. Returns a Path if a charm
    is packed.
* Made `get_slurmd_res()` a coroutine so it could be used with asyncio.gather(...)
* Updated pylibjuju to generic version rather than pin to 3.1.0.1.
* Removed references to ETCD.
* Fixed faulty integration test that was calling out to slurmctld and not slurmrestd.

Signed-off-by: Jason C. Nucciarone <[email protected]>

tests/integration/conftest.py

@@ -13,36 +13,110 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Configure integration test run."""
+"""Configure slurmrestd operator integration tests."""
 
-import pathlib
+import logging
+import os
+from pathlib import Path
+from typing import Union
 
 import pytest
-from _pytest.config.argparsing import Parser
-from helpers import ETCD, NHC
+from helpers import NHC
 from pytest_operator.plugin import OpsTest
 
+logger = logging.getLogger(__name__)
+SLURMCTLD_DIR = Path(os.getenv("SLURMCTLD_DIR", "../slurmctld-operator"))
+SLURMD_DIR = Path(os.getenv("SLURMD_DIR", "../slurmd-operator"))
+SLURMDBD_DIR = Path(os.getenv("SLURMDBD_DIR", "../slurmdbd-operator"))
 
-def pytest_addoption(parser: Parser) -> None:
+
+def pytest_addoption(parser) -> None:
+    parser.addoption(
+        "--charm-base",
+        action="store",
+        default="[email protected]",
+        help="Charm base version to use for integration tests",
+    )
     parser.addoption(
-        "--charm-base", action="store", default="[email protected]", help="Charm base to test."
+        "--use-local",
+        action="store_true",
+        default=False,
+        help="Use SLURM operators located on localhost rather than pull from Charmhub",
     )
 
 
 @pytest.fixture(scope="module")
 def charm_base(request) -> str:
     """Get slurmdbd charm base to use."""
-    return request.config.getoption("--charm-base")
+    return request.config.option.charm_base
+
+
+@pytest.fixture(scope="module")
+async def slurmrestd_charm(ops_test: OpsTest) -> Path:
+    """Pack slurmrestd charm to use for integration testing."""
+    return await ops_test.build_charm(".")
+
+
+@pytest.fixture(scope="module")
+async def slurmctld_charm(request, ops_test: OpsTest) -> Union[str, Path]:
+    """Pack slurmctld charm to use for integration tests when --use-local is specified.
+
+    Returns:
+        `str` "slurmctld" if --use-local not specified or if SLURMD_DIR does not exist.
+    """
+    if request.config.option.use_local:
+        logger.info("Using local slurmctld operator rather than pulling from Charmhub")
+        if SLURMCTLD_DIR.exists():
+            return await ops_test.build_charm(SLURMCTLD_DIR)
+        else:
+            logger.warning(
+                f"{SLURMCTLD_DIR} not found. "
+                f"Defaulting to latest/edge slurmctld operator from Charmhub"
+            )
+
+    return "slurmctld"
 
 
 @pytest.fixture(scope="module")
-async def slurmrestd_charm(ops_test: OpsTest):
-    """Slurmrestd charm used for integration testing."""
-    charm = await ops_test.build_charm(".")
-    return charm
+async def slurmd_charm(request, ops_test: OpsTest) -> Union[str, Path]:
+    """Pack slurmd charm to use for integration tests when --use-local is specified.
+
+    Returns:
+        `str` "slurmd" if --use-local not specified or if SLURMD_DIR does not exist.
+    """
+    if request.config.option.use_local:
+        logger.info("Using local slurmd operator rather than pulling from Charmhub")
+        if SLURMD_DIR.exists():
+            return await ops_test.build_charm(SLURMD_DIR)
+        else:
+            logger.warning(
+                f"{SLURMD_DIR} not found. "
+                f"Defaulting to latest/edge slurmd operator from Charmhub"
+            )
+
+    return "slurmd"
+
+
+@pytest.fixture(scope="module")
+async def slurmdbd_charm(request, ops_test: OpsTest) -> Union[str, Path]:
+    """Pack slurmdbd charm to use for integration tests when --use-local is specified.
+
+    Returns:
+        `str` "slurmdbd" if --use-local not specified or if SLURMDBD_DIR does not exist.
+    """
+    if request.config.option.use_local:
+        logger.info("Using local slurmdbd operator rather than pulling from Charmhub")
+        if SLURMDBD_DIR.exists():
+            return await ops_test.build_charm(SLURMDBD_DIR)
+        else:
+            logger.warning(
+                f"{SLURMDBD_DIR} not found. "
+                f"Defaulting to latest/edge slurmdbd operator from Charmhub"
+            )
+
+    return "slurmdbd"
 
 
 def pytest_sessionfinish(session, exitstatus) -> None:
     """Clean up repository after test session has completed."""
-    pathlib.Path(ETCD).unlink(missing_ok=True)
-    pathlib.Path(NHC).unlink(missing_ok=True)
+    Path(NHC).unlink(missing_ok=True)

tests/integration/helpers.py

@@ -12,33 +12,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Helpers for the slurmd integration tests."""
+"""Helpers for the slurmrestd integration tests."""
 
 import logging
-import pathlib
+from pathlib import Path
 from typing import Dict
 from urllib import request
 
 logger = logging.getLogger(__name__)
 
-ETCD = "etcd-v3.5.0-linux-amd64.tar.gz"
-ETCD_URL = f"https://github.com/etcd-io/etcd/releases/download/v3.5.0/{ETCD}"
 NHC = "lbnl-nhc-1.4.3.tar.gz"
 NHC_URL = f"https://github.com/mej/nhc/releases/download/1.4.3/{NHC}"
 
 
-def get_slurmctld_res() -> Dict[str, pathlib.Path]:
-    """Get slurmctld resources needed for charm deployment."""
-    if not (etcd := pathlib.Path(ETCD)).exists():
-        logger.info(f"Getting resource {ETCD} from {ETCD_URL}")
-        request.urlretrieve(ETCD_URL, etcd)
-
-    return {"etcd": etcd}
-
-
-def get_slurmd_res() -> Dict[str, pathlib.Path]:
+async def get_slurmd_res() -> Dict[str, Path]:
     """Get slurmd resources needed for charm deployment."""
-    if not (nhc := pathlib.Path(NHC)).exists():
+    if not (nhc := Path(NHC)).exists():
         logger.info(f"Getting resource {NHC} from {NHC_URL}")
         request.urlretrieve(NHC_URL, nhc)
 

tests/integration/test_charm.py

@@ -13,16 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Test slurmrestd charm against other SLURM charms in the latest/edge channel."""
+"""Test slurmrestd charm against other SLURM operators."""
 
 import asyncio
 import logging
-import pathlib
-from typing import Any, Coroutine
 
 import pytest
 import tenacity
-from helpers import get_slurmctld_res, get_slurmd_res
+from helpers import get_slurmd_res
 from pytest_operator.plugin import OpsTest
 
 logger = logging.getLogger(__name__)
@@ -33,45 +31,55 @@
 SLURMRESTD = "slurmrestd"
 DATABASE = "mysql"
 ROUTER = "mysql-router"
+UNIT_NAME = f"{SLURMRESTD}/0"
 
 
 @pytest.mark.abort_on_fail
 @pytest.mark.skip_if_deployed
 @pytest.mark.order(1)
 async def test_build_and_deploy(
-    ops_test: OpsTest, slurmrestd_charm: Coroutine[Any, Any, pathlib.Path], charm_base: str
+    ops_test: OpsTest,
+    charm_base: str,
+    slurmrestd_charm,
+    slurmctld_charm,
+    slurmd_charm,
+    slurmdbd_charm,
 ) -> None:
-    """Deploy minimal working slurmrestd charm."""
-    res_slurmd = get_slurmd_res()
-    res_slurmctld = get_slurmctld_res()
+    """Test that the slurmrestd charm can stabilize against slurmctld, slurmd, and slurmdbd."""
+    logger.info(
+        f"Deploying {SLURMRESTD} against {SLURMCTLD}, {SLURMD}, {SLURMDBD}, and {DATABASE}"
+    )
+    # Pack charms and download NHC resource for slurmd operator.
+    slurmrestd, slurmctld, slurmd_res, slurmd, slurmdbd = await asyncio.gather(
+        slurmrestd_charm, slurmctld_charm, get_slurmd_res(), slurmd_charm, slurmdbd_charm
+    )
+    # Deploy the test Charmed SLURM cloud.
     await asyncio.gather(
-        # Fetch from charmhub slurmctld
         ops_test.model.deploy(
-            SLURMCTLD,
-            application_name=SLURMCTLD,
-            channel="edge",
+            slurmrestd,
+            application_name=SLURMRESTD,
             num_units=1,
-            resources=res_slurmctld,
             base=charm_base,
         ),
         ops_test.model.deploy(
-            SLURMD,
-            application_name=SLURMD,
-            channel="edge",
+            str(slurmctld),
+            application_name=SLURMCTLD,
+            channel="edge" if isinstance(slurmctld, str) else None,
             num_units=1,
-            resources=res_slurmd,
             base=charm_base,
         ),
         ops_test.model.deploy(
-            SLURMDBD,
-            application_name=SLURMDBD,
-            channel="edge",
+            str(slurmd),
+            application_name=SLURMD,
+            channel="edge" if isinstance(slurmd, str) else None,
             num_units=1,
+            resources=slurmd_res,
             base=charm_base,
         ),
         ops_test.model.deploy(
-            str(await slurmrestd_charm),
-            application_name=SLURMRESTD,
+            str(slurmdbd),
+            application_name=SLURMDBD,
+            channel="edge" if isinstance(slurmdbd, str) else None,
             num_units=1,
             base=charm_base,
         ),
@@ -91,8 +99,7 @@ async def test_build_and_deploy(
         ),
     )
     # Attach resources to charms.
-    await ops_test.juju("attach-resource", SLURMCTLD, f"etcd={res_slurmctld['etcd']}")
-    await ops_test.juju("attach-resource", SLURMD, f"nhc={res_slurmd['nhc']}")
+    await ops_test.juju("attach-resource", SLURMD, f"nhc={slurmd_res['nhc']}")
     # Set relations for charmed applications.
     await ops_test.model.integrate(f"{SLURMCTLD}:{SLURMDBD}", f"{SLURMDBD}:{SLURMDBD}")
     await ops_test.model.integrate(f"{SLURMDBD}-{ROUTER}:backend-database", f"{DATABASE}:database")
@@ -102,7 +109,7 @@ async def test_build_and_deploy(
     # Reduce the update status frequency to accelerate the triggering of deferred events.
     async with ops_test.fast_forward():
         await ops_test.model.wait_for_idle(apps=[SLURMRESTD], status="active", timeout=1000)
-        assert ops_test.model.applications[SLURMRESTD].units[0].workload_status == "active"
+        assert ops_test.model.units.get(UNIT_NAME).workload_status == "active"
 
 
 @pytest.mark.abort_on_fail
@@ -115,15 +122,12 @@ async def test_build_and_deploy(
 async def test_munge_is_active(ops_test: OpsTest) -> None:
     """Test that munge is active."""
     logger.info("Checking that munge is active inside Juju unit")
-    slurmctld_unit = ops_test.model.applications[SLURMCTLD].units[0]
-    res = (await slurmctld_unit.ssh("systemctl is-active munge")).strip("\n")
+    slurmrestd_unit = ops_test.model.units.get(UNIT_NAME)
+    res = (await slurmrestd_unit.ssh("systemctl is-active munge")).strip("\n")
     assert res == "active"
 
 
-# IMPORTANT: Currently there is a bug where slurmrestd can reach active status despite the
-# systemd service failing. Error is "unable to get address" and "Temporary failure in
-# name resolution".
-@pytest.mark.xfail
+@pytest.mark.abort_on_fail
 @pytest.mark.order(3)
 @tenacity.retry(
     wait=tenacity.wait.wait_exponential(multiplier=2, min=1, max=30),
@@ -133,6 +137,6 @@ async def test_munge_is_active(ops_test: OpsTest) -> None:
 async def test_slurmrestd_is_active(ops_test: OpsTest) -> None:
     """Test that slurmrestd is active."""
     logger.info("Checking that slurmrestd is active inside Juju unit")
-    unit = ops_test.model.applications[SLURMRESTD].units[0]
-    cmd_res = (await unit.ssh("systemctl is-active slurmrestd")).strip("\n")
+    slurmrestd_unit = ops_test.model.units.get(UNIT_NAME)
+    cmd_res = (await slurmrestd_unit.ssh("systemctl is-active slurmrestd")).strip("\n")
     assert cmd_res == "active"

tox.ini

@@ -20,6 +20,9 @@ passenv =
     PYTHONPATH
     CHARM_BUILD_DIR
     MODEL_SETTINGS
+    SLURMCTLD_DIR
+    SLURMD_DIR
+    SLURMDBD_DIR
 
 [testenv:fmt]
 description = Apply coding style standards to code
@@ -56,7 +59,7 @@ commands =
 [testenv:integration]
 description = Run integration tests
 deps =
-    juju==3.1.0.1
+    juju
     pytest==7.2.0
     pytest-operator==0.26.0
     pytest-order==1.1.0