diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index ca0f13f..ef2a4fc 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -13,36 +13,110 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Configure integration test run.""" +"""Configure slurmrestd operator integration tests.""" -import pathlib +import logging +import os +from pathlib import Path +from typing import Union import pytest -from _pytest.config.argparsing import Parser -from helpers import ETCD, NHC +from helpers import NHC from pytest_operator.plugin import OpsTest +logger = logging.getLogger(__name__) +SLURMCTLD_DIR = Path(os.getenv("SLURMCTLD_DIR", "../slurmctld-operator")) +SLURMD_DIR = Path(os.getenv("SLURMD_DIR", "../slurmd-operator")) +SLURMDBD_DIR = Path(os.getenv("SLURMDBD_DIR", "../slurmdbd-operator")) -def pytest_addoption(parser: Parser) -> None: + +def pytest_addoption(parser) -> None: + parser.addoption( + "--charm-base", + action="store", + default="ubuntu@22.04", + help="Charm base version to use for integration tests", + ) parser.addoption( - "--charm-base", action="store", default="ubuntu@22.04", help="Charm base to test." + "--use-local", + action="store_true", + default=False, + help="Use SLURM operators located on localhost rather than pull from Charmhub", ) @pytest.fixture(scope="module") def charm_base(request) -> str: """Get slurmdbd charm base to use.""" - return request.config.getoption("--charm-base") + return request.config.option.charm_base + + +@pytest.fixture(scope="module") +async def slurmrestd_charm(ops_test: OpsTest) -> Path: + """Pack slurmrestd charm to use for integration testing.""" + return await ops_test.build_charm(".") + + +@pytest.fixture(scope="module") +async def slurmctld_charm(request, ops_test: OpsTest) -> Union[str, Path]: + """Pack slurmctld charm to use for integration tests when --use-local is specified. + + Returns: + `str` "slurmctld" if --use-local not specified or if SLURMD_DIR does not exist. + """ + if request.config.option.use_local: + logger.info("Using local slurmctld operator rather than pulling from Charmhub") + if SLURMCTLD_DIR.exists(): + return await ops_test.build_charm(SLURMCTLD_DIR) + else: + logger.warning( + f"{SLURMCTLD_DIR} not found. " + f"Defaulting to latest/edge slurmctld operator from Charmhub" + ) + + return "slurmctld" @pytest.fixture(scope="module") -async def slurmrestd_charm(ops_test: OpsTest): - """Slurmrestd charm used for integration testing.""" - charm = await ops_test.build_charm(".") - return charm +async def slurmd_charm(request, ops_test: OpsTest) -> Union[str, Path]: + """Pack slurmd charm to use for integration tests when --use-local is specified. + + Returns: + `str` "slurmd" if --use-local not specified or if SLURMD_DIR does not exist. + """ + if request.config.option.use_local: + logger.info("Using local slurmd operator rather than pulling from Charmhub") + if SLURMD_DIR.exists(): + return await ops_test.build_charm(SLURMD_DIR) + else: + logger.warning( + f"{SLURMD_DIR} not found. " + f"Defaulting to latest/edge slurmd operator from Charmhub" + ) + + return "slurmd" + + +@pytest.fixture(scope="module") +async def slurmdbd_charm(request, ops_test: OpsTest) -> Union[str, Path]: + """Pack slurmdbd charm to use for integration tests when --use-local is specified. + + Returns: + `str` "slurmdbd" if --use-local not specified or if SLURMDBD_DIR does not exist. + """ + if request.config.option.use_local: + logger.info("Using local slurmdbd operator rather than pulling from Charmhub") + if SLURMDBD_DIR.exists(): + return await ops_test.build_charm(SLURMDBD_DIR) + else: + logger.warning( + f"{SLURMDBD_DIR} not found. " + f"Defaulting to latest/edge slurmdbd operator from Charmhub" + ) + + return "slurmdbd" def pytest_sessionfinish(session, exitstatus) -> None: """Clean up repository after test session has completed.""" - pathlib.Path(ETCD).unlink(missing_ok=True) - pathlib.Path(NHC).unlink(missing_ok=True) + Path(NHC).unlink(missing_ok=True) diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index e7d309b..9ca94b0 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -12,33 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Helpers for the slurmd integration tests.""" +"""Helpers for the slurmrestd integration tests.""" import logging -import pathlib +from pathlib import Path from typing import Dict from urllib import request logger = logging.getLogger(__name__) -ETCD = "etcd-v3.5.0-linux-amd64.tar.gz" -ETCD_URL = f"https://github.com/etcd-io/etcd/releases/download/v3.5.0/{ETCD}" NHC = "lbnl-nhc-1.4.3.tar.gz" NHC_URL = f"https://github.com/mej/nhc/releases/download/1.4.3/{NHC}" -def get_slurmctld_res() -> Dict[str, pathlib.Path]: - """Get slurmctld resources needed for charm deployment.""" - if not (etcd := pathlib.Path(ETCD)).exists(): - logger.info(f"Getting resource {ETCD} from {ETCD_URL}") - request.urlretrieve(ETCD_URL, etcd) - - return {"etcd": etcd} - - -def get_slurmd_res() -> Dict[str, pathlib.Path]: +async def get_slurmd_res() -> Dict[str, Path]: """Get slurmd resources needed for charm deployment.""" - if not (nhc := pathlib.Path(NHC)).exists(): + if not (nhc := Path(NHC)).exists(): logger.info(f"Getting resource {NHC} from {NHC_URL}") request.urlretrieve(NHC_URL, nhc) diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 71b536b..422830a 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -13,16 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Test slurmrestd charm against other SLURM charms in the latest/edge channel.""" +"""Test slurmrestd charm against other SLURM operators.""" import asyncio import logging -import pathlib -from typing import Any, Coroutine import pytest import tenacity -from helpers import get_slurmctld_res, get_slurmd_res +from helpers import get_slurmd_res from pytest_operator.plugin import OpsTest logger = logging.getLogger(__name__) @@ -33,45 +31,55 @@ SLURMRESTD = "slurmrestd" DATABASE = "mysql" ROUTER = "mysql-router" +UNIT_NAME = f"{SLURMRESTD}/0" @pytest.mark.abort_on_fail @pytest.mark.skip_if_deployed @pytest.mark.order(1) async def test_build_and_deploy( - ops_test: OpsTest, slurmrestd_charm: Coroutine[Any, Any, pathlib.Path], charm_base: str + ops_test: OpsTest, + charm_base: str, + slurmrestd_charm, + slurmctld_charm, + slurmd_charm, + slurmdbd_charm, ) -> None: - """Deploy minimal working slurmrestd charm.""" - res_slurmd = get_slurmd_res() - res_slurmctld = get_slurmctld_res() + """Test that the slurmrestd charm can stabilize against slurmctld, slurmd, and slurmdbd.""" + logger.info( + f"Deploying {SLURMRESTD} against {SLURMCTLD}, {SLURMD}, {SLURMDBD}, and {DATABASE}" + ) + # Pack charms and download NHC resource for slurmd operator. + slurmrestd, slurmctld, slurmd_res, slurmd, slurmdbd = await asyncio.gather( + slurmrestd_charm, slurmctld_charm, get_slurmd_res(), slurmd_charm, slurmdbd_charm + ) + # Deploy the test Charmed SLURM cloud. await asyncio.gather( - # Fetch from charmhub slurmctld ops_test.model.deploy( - SLURMCTLD, - application_name=SLURMCTLD, - channel="edge", + slurmrestd, + application_name=SLURMRESTD, num_units=1, - resources=res_slurmctld, base=charm_base, ), ops_test.model.deploy( - SLURMD, - application_name=SLURMD, - channel="edge", + str(slurmctld), + application_name=SLURMCTLD, + channel="edge" if isinstance(slurmctld, str) else None, num_units=1, - resources=res_slurmd, base=charm_base, ), ops_test.model.deploy( - SLURMDBD, - application_name=SLURMDBD, - channel="edge", + str(slurmd), + application_name=SLURMD, + channel="edge" if isinstance(slurmd, str) else None, num_units=1, + resources=slurmd_res, base=charm_base, ), ops_test.model.deploy( - str(await slurmrestd_charm), - application_name=SLURMRESTD, + str(slurmdbd), + application_name=SLURMDBD, + channel="edge" if isinstance(slurmdbd, str) else None, num_units=1, base=charm_base, ), @@ -91,8 +99,7 @@ async def test_build_and_deploy( ), ) # Attach resources to charms. - await ops_test.juju("attach-resource", SLURMCTLD, f"etcd={res_slurmctld['etcd']}") - await ops_test.juju("attach-resource", SLURMD, f"nhc={res_slurmd['nhc']}") + await ops_test.juju("attach-resource", SLURMD, f"nhc={slurmd_res['nhc']}") # Set relations for charmed applications. await ops_test.model.integrate(f"{SLURMCTLD}:{SLURMDBD}", f"{SLURMDBD}:{SLURMDBD}") await ops_test.model.integrate(f"{SLURMDBD}-{ROUTER}:backend-database", f"{DATABASE}:database") @@ -102,7 +109,7 @@ async def test_build_and_deploy( # Reduce the update status frequency to accelerate the triggering of deferred events. async with ops_test.fast_forward(): await ops_test.model.wait_for_idle(apps=[SLURMRESTD], status="active", timeout=1000) - assert ops_test.model.applications[SLURMRESTD].units[0].workload_status == "active" + assert ops_test.model.units.get(UNIT_NAME).workload_status == "active" @pytest.mark.abort_on_fail @@ -115,15 +122,12 @@ async def test_build_and_deploy( async def test_munge_is_active(ops_test: OpsTest) -> None: """Test that munge is active.""" logger.info("Checking that munge is active inside Juju unit") - slurmctld_unit = ops_test.model.applications[SLURMCTLD].units[0] - res = (await slurmctld_unit.ssh("systemctl is-active munge")).strip("\n") + slurmrestd_unit = ops_test.model.units.get(UNIT_NAME) + res = (await slurmrestd_unit.ssh("systemctl is-active munge")).strip("\n") assert res == "active" -# IMPORTANT: Currently there is a bug where slurmrestd can reach active status despite the -# systemd service failing. Error is "unable to get address" and "Temporary failure in -# name resolution". -@pytest.mark.xfail +@pytest.mark.abort_on_fail @pytest.mark.order(3) @tenacity.retry( wait=tenacity.wait.wait_exponential(multiplier=2, min=1, max=30), @@ -133,6 +137,6 @@ async def test_munge_is_active(ops_test: OpsTest) -> None: async def test_slurmrestd_is_active(ops_test: OpsTest) -> None: """Test that slurmrestd is active.""" logger.info("Checking that slurmrestd is active inside Juju unit") - unit = ops_test.model.applications[SLURMRESTD].units[0] - cmd_res = (await unit.ssh("systemctl is-active slurmrestd")).strip("\n") + slurmrestd_unit = ops_test.model.units.get(UNIT_NAME) + cmd_res = (await slurmrestd_unit.ssh("systemctl is-active slurmrestd")).strip("\n") assert cmd_res == "active" diff --git a/tox.ini b/tox.ini index 7bee544..072934d 100644 --- a/tox.ini +++ b/tox.ini @@ -20,6 +20,9 @@ passenv = PYTHONPATH CHARM_BUILD_DIR MODEL_SETTINGS + SLURMCTLD_DIR + SLURMD_DIR + SLURMDBD_DIR [testenv:fmt] description = Apply coding style standards to code @@ -56,7 +59,7 @@ commands = [testenv:integration] description = Run integration tests deps = - juju==3.1.0.1 + juju pytest==7.2.0 pytest-operator==0.26.0 pytest-order==1.1.0