From a070cc7640a1779ac08e9b5bf12eda0f42a65f48 Mon Sep 17 00:00:00 2001 From: dvdgomez <108831013+dvdgomez@users.noreply.github.com> Date: Fri, 27 Jan 2023 07:52:43 -0800 Subject: [PATCH] Add integration tests for slurmd operator (#8) * initial integration test * almost working build and deploy integration test * integration test fix for bug with slurmd and slurmctld where relation has to join after slurmctld is ready * cleaned up initial test * working deploy integration test * mpi integration test + mpi bug fix * mpi_install working test * latest working tests with standards update * add license header to pipelines * Update tests/integration/test_charm.py Co-authored-by: Jason Nucciarone <40342202+NucciTheBoss@users.noreply.github.com> * updated integration tests and actions * add missing spacing and license header * enhance: add toml license header and remove mpi action test * fix: rearrange test order and mark slurmd active test as xfail due to bug * bugfix: Add missing line continuation for coverage --------- Co-authored-by: Jason Nucciarone <40342202+NucciTheBoss@users.noreply.github.com> --- .github/workflows/build-and-test.yaml | 40 --------- .github/workflows/ci.yaml | 75 ++++++++++++++++ .github/workflows/pull-request.yaml | 9 -- pyproject.toml | 13 ++- src/charm.py | 4 +- tests/integration/conftest.py | 35 ++++++++ tests/integration/helpers.py | 55 ++++++++++++ tests/integration/test_charm.py | 123 ++++++++++++++++++++++++++ tests/unit/test_charm.py | 17 ++++ tox.ini | 44 +++++---- 10 files changed, 348 insertions(+), 67 deletions(-) delete mode 100644 .github/workflows/build-and-test.yaml create mode 100644 .github/workflows/ci.yaml delete mode 100644 .github/workflows/pull-request.yaml create mode 100644 tests/integration/conftest.py create mode 100644 tests/integration/helpers.py create mode 100644 tests/integration/test_charm.py diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml deleted file mode 100644 index 8b61d44..0000000 --- a/.github/workflows/build-and-test.yaml +++ /dev/null @@ -1,40 +0,0 @@ -name: Build/Test - -on: - workflow_call: - -jobs: - inclusive-naming-check: - name: Inclusive naming check - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: woke - uses: get-woke/woke-action@v0 - with: - # Cause the check to fail on any broke rules - fail-on-error: true - - lint: - name: Lint - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Install dependencies - run: python3 -m pip install tox - - name: Run linters - run: tox -e lint - - unit-test: - name: Unit tests - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Install dependencies - run: python3 -m pip install tox - - name: Run tests - run: tox -e unit diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..40b939c --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,75 @@ +# Copyright 2023 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: slurmd charm tests +on: + workflow_call: + pull_request: + +jobs: + inclusive-naming-check: + name: Inclusive naming check + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: woke + uses: get-woke/woke-action@v0 + with: + # Cause the check to fail on any broke rules + fail-on-error: true + + lint: + name: Lint + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Install dependencies + run: python3 -m pip install tox + - name: Run linters + run: tox -e lint + + unit-test: + name: Unit tests + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Install dependencies + run: python3 -m pip install tox + - name: Run tests + run: tox -e unit + + integration-test: + name: Integration tests (LXD) + runs-on: ubuntu-latest + needs: + - inclusive-naming-check + - lint + - unit-test + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Install dependencies + run: python3 -m pip install tox + - name: Setup operator environment + uses: charmed-kubernetes/actions-operator@main + with: + provider: lxd + # Juju channel should eventually be updated to 3.0/stable + juju-channel: 2.9/stable + - name: Run tests + run: tox -e integration diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml deleted file mode 100644 index 29d7270..0000000 --- a/.github/workflows/pull-request.yaml +++ /dev/null @@ -1,9 +0,0 @@ -name: Tests -on: - pull_request: - paths-ignore: - - ".gitignore" - -jobs: - test: - uses: ./.github/workflows/build-and-test.yaml diff --git a/pyproject.toml b/pyproject.toml index 6f182fa..ad3a70d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,16 @@ # Copyright 2023 Canonical Ltd. -# See LICENSE file for licensing details. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. [tool.codespell] skip = "./build,./lib,./venv,./icon.svg" diff --git a/src/charm.py b/src/charm.py index 1c180b8..f509265 100755 --- a/src/charm.py +++ b/src/charm.py @@ -534,8 +534,8 @@ def mpi_install(self, event): """Install MPI (mpich).""" self._slurm_manager.mpi.install() - if self._slurm_manager.mpi.installed(): - event.set_results({"installation": "Successfull."}) + if self._slurm_manager.mpi.installed: + event.set_results({'installation': 'Successfull.'}) else: event.fail(message="Error installing mpich. Check the logs.") diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py new file mode 100644 index 0000000..f2d36e5 --- /dev/null +++ b/tests/integration/conftest.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +# Copyright 2023 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Configure integration test run.""" + +import pathlib + +from pytest import fixture +from pytest_operator.plugin import OpsTest + +from helpers import ETCD, NHC, VERSION + + +@fixture(scope="module") +async def slurmd_charm(ops_test: OpsTest): + charm = await ops_test.build_charm(".") + return charm + +def pytest_sessionfinish(session, exitstatus) -> None: + """Clean up repository after test session has completed.""" + pathlib.Path(ETCD).unlink(missing_ok=True) + pathlib.Path(NHC).unlink(missing_ok=True) + pathlib.Path(VERSION).unlink(missing_ok=True) diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py new file mode 100644 index 0000000..e4eef04 --- /dev/null +++ b/tests/integration/helpers.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +# Copyright 2023 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helpers for the slurmd integration tests.""" + +import logging +import pathlib +import shlex +import subprocess + +from typing import Dict +from urllib import request + +logger = logging.getLogger(__name__) + +ETCD = "etcd-v3.5.0-linux-amd64.tar.gz" +ETCD_URL = f"https://github.com/etcd-io/etcd/releases/download/v3.5.0/{ETCD}" +NHC = "lbnl-nhc-1.4.3.tar.gz" +NHC_URL = f"https://github.com/mej/nhc/releases/download/1.4.3/{NHC}" +VERSION = "version" +VERSION_NUM = subprocess.run( + shlex.split("git describe --always"), stdout=subprocess.PIPE, text=True +).stdout.strip("\n") + + +def get_slurmctld_res() -> Dict[str, pathlib.Path]: + """Get slurmctld resources needed for charm deployment.""" + if not (version := pathlib.Path(VERSION)).exists(): + logger.info(f"Setting resource {VERSION} to value {VERSION_NUM}...") + version.write_text(VERSION_NUM) + if not (etcd := pathlib.Path(ETCD)).exists(): + logger.info(f"Getting resource {ETCD} from {ETCD_URL}...") + request.urlretrieve(ETCD_URL, etcd) + + return {"etcd": etcd} + +def get_slurmd_res() -> Dict[str, pathlib.Path]: + """Get slurmd resources needed for charm deployment.""" + if not (nhc := pathlib.Path(NHC)).exists(): + logger.info(f"Getting resource {NHC} from {NHC_URL}...") + request.urlretrieve(NHC_URL, nhc) + + return {"nhc": nhc} diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py new file mode 100644 index 0000000..b57696f --- /dev/null +++ b/tests/integration/test_charm.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +# Copyright 2023 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Test slurmd charm against other SLURM charms in the latest/edge channel.""" + +import asyncio +import pytest + +from helpers import ( + get_slurmd_res, + get_slurmctld_res, +) + +from pathlib import Path +from pytest_operator.plugin import OpsTest +from tenacity import retry +from tenacity.stop import stop_after_attempt +from tenacity.wait import wait_exponential as wexp + +SERIES = ["focal"] +SLURMD = "slurmd" +SLURMDBD = "slurmdbd" +SLURMCTLD = "slurmctld" + + +@pytest.mark.abort_on_fail +@pytest.mark.parametrize("series", SERIES) +@pytest.mark.skip_if_deployed +async def test_build_and_deploy(ops_test: OpsTest, series: str, slurmd_charm): + """Test that the slurmd charm can stabilize against slurmctld, slurmdbd and percona.""" + res_slurmd = get_slurmd_res() + res_slurmctld = get_slurmctld_res() + + # Fetch edge from charmhub for slurmctld, slurmdbd and percona and deploy + await asyncio.gather( + ops_test.model.deploy( + SLURMCTLD, + application_name=SLURMCTLD, + channel="edge", + num_units=1, + resources=res_slurmctld, + series=series, + ), + ops_test.model.deploy( + SLURMDBD, + application_name=SLURMDBD, + channel="edge", + num_units=1, + series=series, + ), + ops_test.model.deploy( + "percona-cluster", + application_name="mysql", + channel="edge", + num_units=1, + series="bionic", + ), + ) + + # Attach ETCD resource to the slurmctld controller + await ops_test.juju("attach-resource", SLURMCTLD, f"etcd={res_slurmctld['etcd']}") + + # Add slurmdbd integration to slurmctld + await ops_test.model.relate(SLURMCTLD, SLURMDBD) + + # Add mysql integration to slurmdbd + await ops_test.model.relate(SLURMDBD, "mysql") + + # IMPORTANT: It's possible for slurmd to be stuck waiting for slurmctld despite slurmctld and slurmdbd + # available. Relation between slurmd and slurmctld has to be added after slurmctld is ready + # otherwise risk running into race-condition type behavior. + await ops_test.model.wait_for_idle(apps=[SLURMCTLD], status="blocked", timeout=1000) + + # Build and Deploy Slurmd + await ops_test.model.deploy( + str(await slurmd_charm), + application_name=SLURMD, + num_units=1, + resources=res_slurmd, + series=series, + ) + + # Attach NHC resource to the slurmd controller + await ops_test.juju("attach-resource", SLURMD, f"nhc={res_slurmd['nhc']}") + + # Add slurmctld integration to slurmd + await ops_test.model.relate(SLURMD, SLURMCTLD) + + # Reduce the update status frequency to accelerate the triggering of deferred events. + async with ops_test.fast_forward(): + await ops_test.model.wait_for_idle(apps=[SLURMD], status="active", timeout=1000) + assert ops_test.model.applications[SLURMD].units[0].workload_status == "active" + + +@pytest.mark.abort_on_fail +async def test_munge_is_active(ops_test: OpsTest): + """Test that munge is active.""" + unit = ops_test.model.applications[SLURMD].units[0] + cmd_res = (await unit.ssh(command="systemctl is-active munge")).strip("\n") + assert cmd_res == "active" + + +# IMPORTANT: Currently there is a bug where slurmd can reach active status despite the +# systemd service failing. +@pytest.mark.xfail +@pytest.mark.abort_on_fail +async def test_slurmd_is_active(ops_test: OpsTest): + """Test that slurmd is active.""" + unit = ops_test.model.applications[SLURMD].units[0] + cmd_res = (await unit.ssh(command="systemctl is-active slurmd")).strip("\n") + assert cmd_res == "active" diff --git a/tests/unit/test_charm.py b/tests/unit/test_charm.py index 35e14e7..46190aa 100644 --- a/tests/unit/test_charm.py +++ b/tests/unit/test_charm.py @@ -1,3 +1,20 @@ +#!/usr/bin/env python3 +# Copyright 2023 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Test default charm events such as upgrade charm, install, etc.""" + import unittest from unittest.mock import PropertyMock, patch diff --git a/tox.ini b/tox.ini index 6115605..ad5604d 100644 --- a/tox.ini +++ b/tox.ini @@ -13,13 +13,13 @@ all_path = {[vars]src_path} {[vars]tst_path} [testenv] setenv = - PYTHONPATH = {toxinidir}:{toxinidir}/lib:{[vars]src_path} - PYTHONBREAKPOINT=ipdb.set_trace - PY_COLORS=1 + PYTHONPATH = {toxinidir}:{toxinidir}/lib:{[vars]src_path} + PYTHONBREAKPOINT=ipdb.set_trace + PY_COLORS=1 passenv = - PYTHONPATH - CHARM_BUILD_DIR - MODEL_SETTINGS + PYTHONPATH + CHARM_BUILD_DIR + MODEL_SETTINGS [testenv:fmt] description = Apply coding style standards to code @@ -28,7 +28,7 @@ deps = ruff commands = black {[vars]all_path} - ruff --fix {[vars]all_path} + ruff --fix {[vars]all_path} [testenv:lint] description = Check code against coding style standards @@ -44,14 +44,28 @@ commands = [testenv:unit] description = Run unit tests deps = - pytest==7.2.0 - coverage[toml]==6.5.0 + pytest + coverage[toml] -r{toxinidir}/requirements.txt commands = - coverage run --source={[vars]src_path} \ - -m pytest \ - --tb native \ - -v \ - -s \ - {posargs} + coverage run \ + --source={[vars]src_path} \ + -m pytest -v --tb native -s {posargs} {[vars]tst_path}unit coverage report + +[testenv:integration] +description = Run integration tests +deps = + juju==3.0.4 + pytest==7.2.0 + pytest-operator==0.22.0 + tenacity==8.1.0 +commands = + pytest -v \ + -s \ + --tb native \ + --ignore={[vars]tst_path}unit \ + --log-cli-level=INFO + --model controller \ + --keep-models \ + {posargs}