From 66ce409ac047feffc14bfed122c9f562e1ac59b9 Mon Sep 17 00:00:00 2001 From: Pietro Pasotti Date: Mon, 17 Jul 2023 15:29:27 +0200 Subject: [PATCH 01/10] new syntax nearly done --- interface_tester/__init__.py | 6 +- interface_tester/collector.py | 36 +- interface_tester/errors.py | 3 + interface_tester/interface_test.py | 393 ++++++++++++------ interface_tester/plugin.py | 234 +++-------- interface_tester/runner.py | 146 ------- .../tests/interface/conftest.py | 37 ++ .../interfaces/tracing/v42/charms.yaml | 5 + .../v42/interface_tests/test_provider.py | 30 ++ .../v42/interface_tests/test_requirer.py | 30 ++ .../interfaces/tracing/v42/schema.py | 28 ++ tests/unit/test_collect_interface_tests.py | 121 +----- tests/unit/test_e2e.py | 267 ++++++++++++ tests/unit/utils.py | 3 + 14 files changed, 765 insertions(+), 574 deletions(-) delete mode 100644 interface_tester/runner.py create mode 100644 tests/resources/charm-like-path/tests/interface/conftest.py create mode 100644 tests/resources/cri-like-path/interfaces/tracing/v42/charms.yaml create mode 100644 tests/resources/cri-like-path/interfaces/tracing/v42/interface_tests/test_provider.py create mode 100644 tests/resources/cri-like-path/interfaces/tracing/v42/interface_tests/test_requirer.py create mode 100644 tests/resources/cri-like-path/interfaces/tracing/v42/schema.py create mode 100644 tests/unit/test_e2e.py create mode 100644 tests/unit/utils.py diff --git a/interface_tester/__init__.py b/interface_tester/__init__.py index 6e78801..71bae33 100644 --- a/interface_tester/__init__.py +++ b/interface_tester/__init__.py @@ -2,9 +2,9 @@ # See LICENSE file for licensing details. import pytest -from .interface_test import interface_test_case # noqa: F401 -from .plugin import InterfaceTester -from .schema_base import DataBagSchema # noqa: F401 +from interface_tester.interface_test import Tester +from interface_tester.plugin import InterfaceTester +from interface_tester.schema_base import DataBagSchema # noqa: F401 @pytest.fixture(scope="function") diff --git a/interface_tester/collector.py b/interface_tester/collector.py index 70fa626..aaccf4c 100644 --- a/interface_tester/collector.py +++ b/interface_tester/collector.py @@ -11,20 +11,19 @@ """ import dataclasses import importlib +import inspect import json import logging import sys import types from pathlib import Path -from typing import TYPE_CHECKING, Dict, List, Literal, Optional, Type, TypedDict +from typing import Dict, List, Literal, Optional, Type, TypedDict, Callable import pydantic import yaml -from .interface_test import DataBagSchema, Role, get_registered_test_cases - -if TYPE_CHECKING: - from .interface_test import _InterfaceTestCase +from interface_tester.interface_test import Role +from interface_tester.schema_base import DataBagSchema logger = logging.getLogger("interface_tests_checker") @@ -71,7 +70,7 @@ class _CharmsDotYamlSpec(TypedDict): class _RoleTestSpec(TypedDict): """The tests, schema, and charms for a single role of a given relation interface version.""" - tests: List["_InterfaceTestCase"] + tests: List[Callable[[None], None]] schema: Optional[Type[DataBagSchema]] charms: List[_CharmTestConfig] @@ -198,6 +197,14 @@ def _gather_charms_for_version(version_dir: Path) -> Optional[_CharmsDotYamlSpec return spec +def _scrape_module_for_tests(module) -> List[Callable[[None], None]]: + tests = [] + for name, obj in inspect.getmembers(module): + if inspect.isfunction(obj): + tests.append(obj) + return tests + + def _gather_test_cases_for_version(version_dir: Path, interface_name: str, version: int): """Collect interface test cases from a directory containing an interface version spec.""" @@ -210,21 +217,20 @@ def _gather_test_cases_for_version(version_dir: Path, interface_name: str, versi # so we can import without tricks sys.path.append(str(interface_tests_dir)) - for possible_test_file in interface_tests_dir.glob("*.py"): - # strip .py - module_name = str(possible_test_file.with_suffix("").name) + for role in Role: + module_name = "test_requirer" if role is Role.requirer else "test_provider" try: - importlib.import_module(module_name) + module = importlib.import_module(module_name) except ImportError as e: - logger.error(f"Failed to load module {possible_test_file}: {e}") + logger.error(f"Failed to load module {module_name}: {e}") continue - cases = get_registered_test_cases() + tests = _scrape_module_for_tests(module) + del sys.modules[module_name] - # print(cases) - provider_test_cases.extend(cases[(interface_name, version, Role.provider)]) - requirer_test_cases.extend(cases[(interface_name, version, Role.requirer)]) + tgt = provider_test_cases if role is Role.provider else requirer_test_cases + tgt.extend(tests) if not (requirer_test_cases or provider_test_cases): logger.error(f"no valid test case files found in {interface_tests_dir}") diff --git a/interface_tester/errors.py b/interface_tester/errors.py index b199340..297e99c 100644 --- a/interface_tester/errors.py +++ b/interface_tester/errors.py @@ -14,3 +14,6 @@ class InterfaceTestsFailed(RuntimeError): class NoTestsRun(RuntimeError): """Raised if no interface test was collected during a run() call.""" + +class SchemaValidationError(RuntimeError): + """Raised when schema validation fails on one or more relations.""" diff --git a/interface_tester/interface_test.py b/interface_tester/interface_test.py index a5b56f6..a20920e 100644 --- a/interface_tester/interface_test.py +++ b/interface_tester/interface_test.py @@ -3,21 +3,24 @@ import dataclasses import inspect import logging +import operator import re import typing -from collections import defaultdict +from contextlib import contextmanager from enum import Enum -from typing import Callable, Dict, List, Literal, Optional, Tuple, Union +from typing import Callable, Literal, Optional, Union, Any, Dict, List -from scenario import Event, State +from scenario import Event, State, Context, Relation -from .schema_base import DataBagSchema +from interface_tester.errors import InvalidTestCaseError, SchemaValidationError + +RoleLiteral = Literal["requirer", "provider"] if typing.TYPE_CHECKING: InterfaceNameStr = str VersionInt = int - RoleLiteral = Literal["requirer", "provider"] _SchemaConfigLiteral = Literal["default", "skip", "empty"] + from interface_tester import DataBagSchema INTF_NAME_AND_VERSION_REGEX = re.compile(r"/interfaces/(\w+)/v(\d+)/") @@ -28,94 +31,34 @@ class InvalidTestCase(RuntimeError): """Raised if a function decorated with interface_test_case is invalid.""" -class SchemaConfig(str, Enum): - """Class used to program the schema validator run that is paired with each test case.""" - - default = "default" - """Use this value if you want the test case to validate the output state's databags with the - default schema.""" - skip = "skip" - """Use this value if you want the test case skip schema validation altogether.""" - empty = "empty" - """Use this value if you want the databag validator to assert that the databags are empty.""" - - class Role(str, Enum): provider = "provider" requirer = "requirer" @dataclasses.dataclass -class _InterfaceTestCase: +class _InterfaceTestContext: """Data associated with a single interface test case.""" interface_name: str """The name of the interface that this test is about.""" version: int """The version of the interface that this test is about.""" - event: Union[Event, str] - """The event that this test is about.""" role: Role - """The role (provider|requirer) that this test is about.""" - name: str - """Human-readable name of what this test does.""" - validator: Callable[[State], None] - """The function that will be called on the output state to validate it.""" + # fixme + charm_type: type + supported_endpoints: dict + meta: Any + config: Any + actions: Any - schema: Union[DataBagSchema, SchemaConfig] = SchemaConfig.default - """Either a Pydantic schema for the unit/app databags of 'this side' of the relation, which - will be used to validate the relation databags in the output state, or: - - 'skip' to skip schema validation altogether - - 'empty' to have the schema validator assert that the databags should be empty. - """ + """The role (provider|requirer) that this test is about.""" + schema: Optional["DataBagSchema"] = None + """Databag schema to validate the output relation with.""" input_state: Optional[State] = None """Initial state that this test should be run with.""" - def run(self, output_state: State): - """Execute the test: that is, run the decorated validator against the output state.""" - return self.validator(output_state) - - -_TestCaseCacheType = Dict[Tuple["InterfaceNameStr", "VersionInt", Role], List[_InterfaceTestCase]] -# for each (interface_name, version, role) triplet: the list of all collected interface test cases. -REGISTERED_TEST_CASES: _TestCaseCacheType = defaultdict(list) - - -def get_registered_test_cases(clear: bool=False) -> _TestCaseCacheType: - """The test cases that have been registered so far.""" - tc = REGISTERED_TEST_CASES.copy() - if clear: - REGISTERED_TEST_CASES.clear() - return tc - - -def get_interface_name_and_version(fn: Callable) -> Tuple[str, int]: - f"""Return the interface name and version of a test case validator function. - - It assumes that the function is in a module whose path matches the following regex: - {INTF_NAME_AND_VERSION_REGEX} - - If that can't be matched, it will raise a InvalidTestCase. - """ - - file = inspect.getfile(fn) - match = INTF_NAME_AND_VERSION_REGEX.findall(file) - if len(match) != 1: - raise InvalidTestCase( - f"Can't determine interface name and version from test case location: {file}." - rf"expecting a file path matching '/interfaces/(\w+)/v(\d+)/' " - ) - interface_name, version_str = match[0] - try: - version_int = int(version_str) - except TypeError: - # overly cautious: the regex should already be only matching digits. - raise InvalidTestCase( - f"Unable to cast version {version_str} to integer. " f"Check file location: {file}." - ) - return interface_name, version_int - def check_test_case_validator_signature(fn: Callable): """Verify the signature of a test case validator function. @@ -148,48 +91,262 @@ def check_test_case_validator_signature(fn: Callable): ) -def interface_test_case( - role: Union[Role, "RoleLiteral"], - event: Union[str, Event], - input_state: Optional[State] = None, - name: str = None, - schema: Union[DataBagSchema, SchemaConfig, "_SchemaConfigLiteral"] = SchemaConfig.default, -): - """Decorator to register a function as an interface test case. - The decorated function must take exactly one positional argument of type State. - - Arguments: - :param name: the name of the test. Will default to the decorated function's identifier. - :param event: the event that this test is about. - :param role: the interface role this test is about. - :param input_state: the input state for this scenario test. Will default to the empty State(). - :param schema: the schema that the relation databags for the endpoint being tested should - satisfy **after** the event has been processed. - """ - if not isinstance(schema, DataBagSchema): - schema = SchemaConfig(schema) - - def wrapper(fn: Callable[[State], None]): - # validate that the function is a valid validator - check_test_case_validator_signature(fn) - - # derive from the module the function is defined in what the - # interface name and version are - interface_name, version = get_interface_name_and_version(fn) - - role_ = Role(role) - - REGISTERED_TEST_CASES[(interface_name, version, role_)].append( - _InterfaceTestCase( - interface_name=interface_name, - version=version, - event=event, - role=role_, - validator=fn, - name=name or fn.__name__, - input_state=input_state, - schema=schema, - ) +_TESTER_CTX: Optional[_InterfaceTestContext] = None + + +@contextmanager +def tester_context(ctx: _InterfaceTestContext): + global _TESTER_CTX + _TESTER_CTX = ctx + yield + _TESTER_CTX = None + if not Tester.__instance__: + raise NoTesterInstanceError('invalid tester_context usage: no Tester instance created') + Tester.__instance__._finalize() + + +class InvalidTesterRunError(RuntimeError): + """Raised if Tester is being used incorrectly.""" + +class NoTesterInstanceError(InvalidTesterRunError): + """Raised if no Tester is created within a tester_context scope.""" + + +class NoSchemaError(InvalidTesterRunError): + """Raised when schemas cannot be validated because there is no schema.""" + + +class Tester: + __instance__ = None + + def __init__(self, state_in: State=None, name: str = None): + """Initializer. + + :param state_in: the input state for this scenario test. Will default to the empty State(). + :param name: the name of the test. Will default to the function's identifier. + """ + # todo: pythonify + if Tester.__instance__: + raise RuntimeError("Tester is a singleton.") + Tester.__instance__ = self + + if not self.ctx: + raise RuntimeError('Tester can only be initialized inside a tester context.') + + self._state_template = None + self._state_in = state_in or State() + self._name = name + + self._state_out = None # will be State when has_run is true + self._has_run = False + self._has_checked_schema = False + + @property + def ctx(self): + return _TESTER_CTX + + def run(self, event: Union[str, Event]): + assert self.ctx, 'tester cannot run: no _TESTER_CTX set' + + state_out = self._run(event) + self._state_out = state_out + return state_out + + def assert_schema_valid(self, schema:"DataBagSchema" = None): + self._has_checked_schema = True + if not self._has_run: + raise RuntimeError('call Tester.run() first') + + if schema: + logger.info("running test with custom schema") + databag_schema = schema + else: + logger.info("running test with built-in schema") + databag_schema = self.ctx.schema + if not databag_schema: + raise NoSchemaError( + f"No schemas found for {self.ctx.interface_name}/{self.ctx.version}/{self.ctx.role};" + f"call skip_schema_validation() manually.") + + errors = [] + for relation in [r for r in self._state_out.relations if r.interface == self.ctx.interface_name]: + try: + databag_schema.validate( + { + "unit": relation.local_unit_data, + "app": relation.local_app_data, + } + ) + except RuntimeError as e: + errors.append(e.args[0]) + if errors: + raise SchemaValidationError(errors) + + def _check_has_run(self): + if not self._has_run: + raise InvalidTesterRunError('call Tester.run() first') + + def assert_relation_data_empty(self): + self._check_has_run() + # todo + self._has_checked_schema = True + + def skip_schema_validation(self): + self._check_has_run() + # todo + self._has_checked_schema = True + + def _finalize(self): + if not self._has_run: + raise InvalidTesterRunError("call .run() before returning") + if not self._has_checked_schema: + # todo: + raise InvalidTesterRunError("call .skip_schema_validation(), or ... before returning") + + # release singleton + Tester.__instance__ = None + + + def _run(self, event: Union[str, Event]): + logger.debug(f"running {event}") + self._has_run = True + + # this is the input state as specified by the interface tests writer. It can + # contain elements that are required for the relation interface test to work, + # typically relation data pertaining to the relation interface being tested. + input_state = self._state_in + + # state_template is state as specified by the charm being tested, which the charm + # requires to function properly. Consider it part of the mocking. For example: + # some required config, a "happy" status, network information, OTHER relations. + # Typically, should NOT touch the relation that this interface test is about + # -> so we overwrite and warn on conflict: state_template is the baseline, + state = (self._state_template or State()).copy() + + relations = self._generate_relations_state( + state, input_state, + self.ctx.supported_endpoints, + self.ctx.role ) + # State is frozen; replace + modified_state = state.replace(relations=relations) + + # the Relation instance this test is about: + relation = next(filter(lambda r: r.interface == self.ctx.interface_name, relations)) + # test.EVENT might be a string or an Event. Cast to Event. + evt: Event = self._coerce_event(event, relation) + + logger.info(f"collected test for {self.ctx.interface_name} with {evt.name}") + return self._run_scenario(evt, modified_state) + + def _run_scenario(self, event: Event, state: State): + logger.debug(f"running scenario with state={state}, event={event}") + + ctx = Context(self.ctx.charm_type, meta=self.ctx.meta, + actions=self.ctx.actions, config=self.ctx.config) + return ctx.run(event, state) + + def _coerce_event(self, raw_event: Union[str, Event], relation: Relation) -> Event: + # if the event being tested is a relation event, we need to inject some metadata + # or scenario.Runtime won't be able to guess what envvars need setting before ops.main + # takes over + if isinstance(raw_event, str): + ep_name, _, evt_kind = raw_event.rpartition("-relation-") + if ep_name and evt_kind: + # this is a relation event. + # we inject the relation metadata + # todo: if the user passes a relation event that is NOT about the relation + # interface that this test is about, at this point we are injecting the wrong + # Relation instance. + # e.g. if in interfaces/foo one wants to test that if 'bar-relation-joined' is + # fired... then one would have to pass an Event instance already with its + # own Relation. + return Event( + raw_event, + relation=relation.replace(endpoint=ep_name), + ) + + else: + return Event(raw_event) + + elif isinstance(raw_event, Event): + if raw_event._is_relation_event and not raw_event.relation: + raise InvalidTestCaseError( + "This test case was passed an Event representing a relation event." + "However it does not have a Relation. Please pass it to the Event like so: " + "evt = Event('my_relation_changed', relation=Relation(...))" + ) + + return raw_event + + else: + raise InvalidTestCaseError( + f"Expected Event or str, not {type(raw_event)}. " + f"Invalid test case: {self} cannot cast {raw_event} to Event." + ) - return wrapper + def _generate_relations_state( + self, state_template: State, input_state: State, supported_endpoints, role: Role + ) -> List[Relation]: + """Merge the relations from the input state and the state template into one. + + The charm being tested possibly provided a state_template to define some setup mocking data + The interface tests also have an input_state. Here we merge them into one relation list to + be passed to the 'final' State the test will run with. + """ + interface_name = self.ctx.interface_name + + for rel in state_template.relations: + if rel.interface == interface_name: + logger.warning( + f"relation with interface name = {interface_name} found in state template. " + f"This will be overwritten by the relation spec provided by the relation " + f"interface test case." + ) + + def filter_relations(rels: List[Relation], op: Callable): + return [r for r in rels if op(r.interface, interface_name)] + + # the baseline is: all relations whose interface IS NOT the interface we're testing. + relations = filter_relations(state_template.relations, op=operator.ne) + + if input_state: + # if the charm we're testing specified some relations in its input state, we add those + # whose interface IS the same as the one we're testing. If other relation interfaces + # were specified, they will be ignored. + relations.extend(filter_relations(input_state.relations, op=operator.eq)) + + if ignored := filter_relations(input_state.relations, op=operator.eq): + logger.warning( + f"irrelevant relations specified in input state for {interface_name}/{role}." + f"These will be ignored. details: {ignored}" + ) + + # if we still don't have any relation matching the interface we're testing, we generate + # one from scratch. + if not filter_relations(relations, op=operator.eq): + # if neither the charm nor the interface specified any custom relation spec for + # the interface we're testing, we will provide one. + endpoints_for_interface = supported_endpoints[role] + + if len(endpoints_for_interface) < 1: + raise ValueError(f"no endpoint found for {role}/{interface_name}.") + elif len(endpoints_for_interface) > 1: + raise ValueError( + f"Multiple endpoints found for {role}/{interface_name}: " + f"{endpoints_for_interface}: cannot guess which one it is " + f"we're supposed to be testing" + ) + else: + endpoint = endpoints_for_interface[0] + + relations.append( + Relation( + interface=interface_name, + endpoint=endpoint, + ) + ) + logger.debug( + f"{self}: merged {input_state} and {state_template} --> relations={relations}" + ) + return relations diff --git a/interface_tester/plugin.py b/interface_tester/plugin.py index 6e60dbb..966a960 100644 --- a/interface_tester/plugin.py +++ b/interface_tester/plugin.py @@ -1,12 +1,10 @@ # Copyright 2022 Canonical Ltd. # See LICENSE file for licensing details. import logging -import operator import tempfile from pathlib import Path from subprocess import PIPE, Popen from typing import ( - TYPE_CHECKING, Any, Callable, Dict, @@ -16,38 +14,34 @@ Optional, Tuple, Type, - Union, ) from ops.testing import CharmType -from scenario.state import Event, Relation, State, _CharmSpec +from scenario.state import Event, State, _CharmSpec -from .collector import InterfaceTestSpec, gather_test_spec_for_version -from .errors import ( +from interface_tester.schema_base import DataBagSchema +from interface_tester.collector import InterfaceTestSpec, gather_test_spec_for_version +from interface_tester.errors import ( InterfaceTesterValidationError, InterfaceTestsFailed, - InvalidTestCaseError, NoTestsRun, ) -from .runner import run_test_case -from .schema_base import DataBagSchema - -if TYPE_CHECKING: - from .interface_test import _InterfaceTestCase +from interface_tester.interface_test import _InterfaceTestContext, tester_context, RoleLiteral Callback = Callable[[State, Event], None] +ROLE_TO_ROLE_META = {"provider": "provides", "requirer": "requires"} logger = logging.getLogger("pytest_interface_tester") -ROLE_TO_ROLE_META = {"provider": "provides", "requirer": "requires"} -Role = Literal["provider", "requirer"] class InterfaceTester: + _RAISE_IMMEDIATELY = False + def __init__( - self, - repo: str = "https://github.com/canonical/charm-relation-interfaces", - branch: str = "main", - base_path: str = "interfaces", + self, + repo: str = "https://github.com/canonical/charm-relation-interfaces", + branch: str = "main", + base_path: str = "interfaces", ): self._repo = repo self._branch = branch @@ -65,18 +59,18 @@ def __init__( self._charm_spec_cache = None def configure( - self, - *, - charm_type: Optional[Type[CharmType]] = None, - repo: Optional[str] = None, - branch: Optional[str] = None, - base_path: Optional[str] = None, - interface_name: Optional[str] = None, - interface_version: Optional[int] = None, - state_template: Optional[State] = None, - meta: Optional[Dict[str, Any]] = None, - actions: Optional[Dict[str, Any]] = None, - config: Optional[Dict[str, Any]] = None, + self, + *, + charm_type: Optional[Type[CharmType]] = None, + repo: Optional[str] = None, + branch: Optional[str] = None, + base_path: Optional[str] = None, + interface_name: Optional[str] = None, + interface_version: Optional[int] = None, + state_template: Optional[State] = None, + meta: Optional[Dict[str, Any]] = None, + actions: Optional[Dict[str, Any]] = None, + config: Optional[Dict[str, Any]] = None, ): """ @@ -125,7 +119,7 @@ def _validate_config(self): ) if not self._charm_type: errors.append("Tester misconfigured: needs a charm_type set.") - if not self.meta: + elif not self.meta: errors.append("no metadata: it was not provided, and it cannot be autoloaded") if not self._repo: errors.append("repo missing") @@ -202,11 +196,11 @@ def _collect_interface_test_specs(self) -> InterfaceTestSpec: repo_name = self._repo.split("/")[-1] intf_spec_path = ( - Path(tempdir) - / repo_name - / self._base_path - / self._interface_name.replace("-", "_") - / f"v{self._interface_version}" + Path(tempdir) + / repo_name + / self._base_path + / self._interface_name.replace("-", "_") + / f"v{self._interface_version}" ) if not intf_spec_path.exists(): raise RuntimeError( @@ -223,13 +217,13 @@ def _collect_interface_test_specs(self) -> InterfaceTestSpec: return tests - def _gather_supported_endpoints(self) -> Dict[Literal[Role], List[str]]: + def _gather_supported_endpoints(self) -> Dict[RoleLiteral, List[str]]: """Given a relation interface name, return a list of supported endpoints as either role. These are collected from the charm's metadata.yaml. """ - supported_endpoints: Dict[Literal[Role], List[str]] = {} - role: Role + supported_endpoints: Dict["RoleLiteral", List[str]] = {} + role: RoleLiteral for role in ("provider", "requirer"): meta_role = ROLE_TO_ROLE_META[role] @@ -252,8 +246,8 @@ def _gather_supported_endpoints(self) -> Dict[Literal[Role], List[str]]: return supported_endpoints def _yield_tests( - self, - ) -> Generator[Tuple["_InterfaceTestCase", "DataBagSchema", Event, State], None, None]: + self, + ) -> Generator[Tuple[Callable, RoleLiteral, DataBagSchema], None, None]: """Yield all test cases applicable to this charm and interface. This means: @@ -280,41 +274,14 @@ def _yield_tests( if not supported_endpoints: raise RuntimeError(f"this charm does not declare any endpoint using {interface_name}.") - role: Role + role: RoleLiteral for role in supported_endpoints: logger.debug(f"collecting scenes for {role}") spec = tests[role] - test: "_InterfaceTestCase" + schema = spec["schema"] for test in spec["tests"]: - logger.debug(f"converting {test} to ") - - # this is the input state as specified by the interface tests writer. It can - # contain elements that are required for the relation interface test to work, - # typically relation data pertaining to the relation interface being tested. - input_state = test.input_state - - # state_template is state as specified by the charm being tested, which the charm - # requires to function properly. Consider it part of the mocking. For example: - # some required config, a "happy" status, network information, OTHER relations. - # Typically, should NOT touch the relation that this interface test is about - # -> so we overwrite and warn on conflict: state_template is the baseline, - state = (self._state_template or State()).copy() - - relations = self._generate_relations_state( - state, input_state, supported_endpoints, role - ) - # State is frozen; replace - modified_state = state.replace(relations=relations) - - # the Relation instance this test is about: - relation = next(filter(lambda r: r.interface == self._interface_name, relations)) - # test.EVENT might be a string or an Event. Cast to Event. - evt = self._coerce_event(test.event, relation) - - logger.info(f"collected test for {interface_name} with {evt.name}") - logger.debug(f"state={modified_state}, evt={evt}") - yield test, spec["schema"], evt, modified_state + yield test, role, schema def __repr__(self): return f""" bool: errors = [] ran_some = False - for test, schema, event, state in self._yield_tests(): - out = run_test_case( - test=test, + for test_fn, role, schema in self._yield_tests(): + ctx = _InterfaceTestContext( + role=role, schema=schema, - event=event, - state=state, interface_name=self._interface_name, + version=self._interface_version, charm_type=self._charm_type, meta=self.meta, config=self.config, actions=self.actions, + supported_endpoints=self._gather_supported_endpoints() ) - if out: - errors.extend(out) - + try: + with tester_context(ctx): + test_fn() + except Exception as e: + if self._RAISE_IMMEDIATELY: + raise e + errors.append(e) ran_some = True # todo: consider raising custom exceptions here. @@ -364,108 +335,3 @@ def run(self) -> bool: msg = f"no tests gathered for {self._interface_name}/v{self._interface_version}" logger.warning(msg) raise NoTestsRun(msg) - - def _coerce_event(self, raw_event: Union[str, Event], relation: Relation) -> Event: - # if the event being tested is a relation event, we need to inject some metadata - # or scenario.Runtime won't be able to guess what envvars need setting before ops.main - # takes over - if isinstance(raw_event, str): - ep_name, _, evt_kind = raw_event.rpartition("-relation-") - if ep_name and evt_kind: - # this is a relation event. - # we inject the relation metadata - # todo: if the user passes a relation event that is NOT about the relation - # interface that this test is about, at this point we are injecting the wrong - # Relation instance. - # e.g. if in interfaces/foo one wants to test that if 'bar-relation-joined' is - # fired... then one would have to pass an Event instance already with its - # own Relation. - return Event( - raw_event, - relation=relation.replace(endpoint=ep_name), - ) - - else: - return Event(raw_event) - - elif isinstance(raw_event, Event): - if raw_event._is_relation_event and not raw_event.relation: - raise InvalidTestCaseError( - "This test case was passed an Event representing a relation event." - "However it does not have a Relation. Please pass it to the Event like so: " - "evt = Event('my_relation_changed', relation=Relation(...))" - ) - - return raw_event - - else: - raise InvalidTestCaseError( - f"Expected Event or str, not {type(raw_event)}. " - f"Invalid test case: {self} cannot cast {raw_event} to Event." - ) - - def _generate_relations_state( - self, state_template: State, input_state: State, supported_endpoints, role: Role - ) -> List[Relation]: - """Merge the relations from the input state and the state template into one. - - The charm being tested possibly provided a state_template to define some setup mocking data - The interface tests also have an input_state. Here we merge them into one relation list to - be passed to the 'final' State the test will run with. - """ - interface_name = self._interface_name - - for rel in state_template.relations: - if rel.interface == interface_name: - logger.warning( - f"relation with interface name = {interface_name} found in state template. " - f"This will be overwritten by the relation spec provided by the relation " - f"interface test case." - ) - - def filter_relations(rels: List[Relation], op: Callable): - return [r for r in rels if op(r.interface, interface_name)] - - # the baseline is: all relations whose interface IS NOT the interface we're testing. - relations = filter_relations(state_template.relations, op=operator.ne) - - if input_state: - # if the charm we're testing specified some relations in its input state, we add those - # whose interface IS the same as the one we're testing. If other relation interfaces - # were specified, they will be ignored. - relations.extend(filter_relations(input_state.relations, op=operator.eq)) - - if ignored := filter_relations(input_state.relations, op=operator.eq): - logger.warning( - f"irrelevant relations specified in input state for {interface_name}/{role}." - f"These will be ignored. details: {ignored}" - ) - - # if we still don't have any relation matching the interface we're testing, we generate - # one from scratch. - if not filter_relations(relations, op=operator.eq): - # if neither the charm nor the interface specified any custom relation spec for - # the interface we're testing, we will provide one. - endpoints_for_interface = supported_endpoints[role] - - if len(endpoints_for_interface) < 1: - raise ValueError(f"no endpoint found for {role}/{interface_name}.") - elif len(endpoints_for_interface) > 1: - raise ValueError( - f"Multiple endpoints found for {role}/{interface_name}: " - f"{endpoints_for_interface}: cannot guess which one it is " - f"we're supposed to be testing" - ) - else: - endpoint = endpoints_for_interface[0] - - relations.append( - Relation( - interface=interface_name, - endpoint=endpoint, - ) - ) - logger.debug( - f"{self}: merged {input_state} and {state_template} --> relations={relations}" - ) - return relations diff --git a/interface_tester/runner.py b/interface_tester/runner.py deleted file mode 100644 index be9907b..0000000 --- a/interface_tester/runner.py +++ /dev/null @@ -1,146 +0,0 @@ -import typing -from typing import Dict, List, Optional, Type - -from ops.charm import CharmBase -from scenario import Context, Event, Relation, State - -from .errors import InvalidTestCaseError -from .interface_test import SchemaConfig, logger -from .schema_base import DataBagSchema - -if typing.TYPE_CHECKING: - from .interface_test import _InterfaceTestCase - - -def _assert_case_plays( - event: Event, state: State, charm_type: Type["CharmBase"], meta, actions, config -) -> State: - try: - ctx = Context(charm_type, meta=meta, actions=actions, config=config) - state_out = ctx.run(event, state) - except Exception as e: - msg = ( - f"Failed check 1: scenario errored out: ({type(e).__name__}){e}. Could not play scene." - ) - raise RuntimeError(msg) from e - return state_out - - -def _assert_state_out_valid(state_out: State, test: "_InterfaceTestCase"): - """Run the test's validator against the output state. - - Raise RuntimeError if any exception is raised by the validator. - """ - try: - test.run(state_out) - except Exception as e: - msg = f"Failed check 2: validating scene output: {e}" - raise RuntimeError(msg) from e - - -def _assert_schema_valid(schema: DataBagSchema, relation: Relation) -> None: - """Validate the relation databags against this schema. - - Raise RuntimeError if any exception is raised by the validator. - """ - try: - schema.validate( - { - "unit": relation.local_unit_data, - "app": relation.local_app_data, - } - ) - except Exception as e: - msg = f"Failed check 3: validating schema on scene output: {e}" - logger.error(msg) - raise RuntimeError(msg) from e - - -def _assert_schemas_valid( - test: "_InterfaceTestCase", state_out: State, schema: DataBagSchema, interface_name: str -) -> List[str]: - """Check that all relations using the interface comply with the provided schema.""" - test_schema = test.schema - if test_schema is SchemaConfig.skip: - logger.info("Schema validation skipped as per interface_test_case schema config.") - return [] - - if test_schema == SchemaConfig.default: - schema = schema - elif test_schema == SchemaConfig.empty: - schema = DataBagSchema() - elif isinstance(test_schema, DataBagSchema): - schema = test_schema - else: - raise InvalidTestCaseError( - "interface_test_case schema should be either a SchemaConfig instance or a " - f"DataBagSchema instance, not {type(test_schema)}." - ) - - errors = [] - for relation in [r for r in state_out.relations if r.interface == interface_name]: - try: - _assert_schema_valid(schema=schema, relation=relation) - except RuntimeError as e: - errors.append(e.args[0]) - return errors - - -def run_test_case( - test: "_InterfaceTestCase", - schema: Optional["DataBagSchema"], - event: Event, - state: State, - interface_name: str, - # the charm type we're testing - charm_type: Type["CharmBase"], - # charm metadata yamls - meta: Dict, - config: Dict, - actions: Dict, -) -> List[str]: - """Run an interface test case. - - This will run three checks in sequence: - - play the scenario (check that the charm runs without exceptions) and - obtain the output state - - validate the output state (by calling the test-case-provided validator with - the output state as argument) - - validate the schema against the relations in the output state. - - It will return a list of strings, representing any issues encountered in any of the checks. - """ - errors: List[str] = [] - logger.info(f"running test {test.name!r}") - logger.info("check 1/3: scenario play") - try: - state_out = _assert_case_plays( - event=event, - state=state, - charm_type=charm_type, - meta=meta, - config=config, - actions=actions, - ) - except RuntimeError as e: - errors.append(e.args[0]) - logger.error("scenario couldn't run: aborting test.", exc_info=True) - return errors - - logger.info("check 2/3: scenario output state validation") - # todo: consistency check? or should we rely on scenario's? - try: - _assert_state_out_valid(state_out=state_out, test=test) - except RuntimeError as e: - errors.append(e.args[0]) - - logger.info("check 3/3: databag schema validation") - if not schema: - logger.info("schema validation step skipped: no schema provided") - return errors - errors.extend( - _assert_schemas_valid( - test=test, state_out=state_out, schema=schema, interface_name=interface_name - ) - ) - return errors diff --git a/tests/resources/charm-like-path/tests/interface/conftest.py b/tests/resources/charm-like-path/tests/interface/conftest.py new file mode 100644 index 0000000..ab5133c --- /dev/null +++ b/tests/resources/charm-like-path/tests/interface/conftest.py @@ -0,0 +1,37 @@ +# Copyright 2022 Canonical Ltd. +# See LICENSE file for licensing details. + +import pytest +from ops import CharmBase +from scenario.state import State + +from interface_tester import InterfaceTester + +from interface_tester import InterfaceTester +from interface_tester.collector import gather_test_spec_for_version +from tests.unit.utils import CRI_LIKE_PATH + + +class CRILikePathTester(InterfaceTester): + def _collect_interface_test_specs(self): + tests = gather_test_spec_for_version( + CRI_LIKE_PATH, + interface_name=self._interface_name, + version=self._interface_version, + ) + +class DummiCharm(CharmBase): + pass + + +@pytest.fixture +def interface_tester(interface_tester: CRILikePathTester): + interface_tester.configure( + charm_type=DummiCharm, + meta={"name": "dummi", + "provides": {"tracing": {"interface": "tracing"}}, + "requires": {"tracing": {"interface": "tracing"}} + }, + state_template=State(leader=True), + ) + yield interface_tester diff --git a/tests/resources/cri-like-path/interfaces/tracing/v42/charms.yaml b/tests/resources/cri-like-path/interfaces/tracing/v42/charms.yaml new file mode 100644 index 0000000..bcc7677 --- /dev/null +++ b/tests/resources/cri-like-path/interfaces/tracing/v42/charms.yaml @@ -0,0 +1,5 @@ +providers: + - name: tempo-k8s + url: https://github.com/canonical/tempo-k8s-operator + +requirers: [] \ No newline at end of file diff --git a/tests/resources/cri-like-path/interfaces/tracing/v42/interface_tests/test_provider.py b/tests/resources/cri-like-path/interfaces/tracing/v42/interface_tests/test_provider.py new file mode 100644 index 0000000..671ff5f --- /dev/null +++ b/tests/resources/cri-like-path/interfaces/tracing/v42/interface_tests/test_provider.py @@ -0,0 +1,30 @@ +# Copyright 2023 Canonical +# See LICENSE file for licensing details. + +from scenario import State, Relation + +from interface_tester.interface_test import Tester + + +def test_no_data_on_created(): + t = Tester(State()) + state_out = t.run(event='tracing-relation-created') + t.assert_relation_data_empty() + + +def test_no_data_on_joined(): + t = Tester() + state_out = t.run(event='tracing-relation-joined') + t.assert_relation_data_empty() + + +def test_data_on_changed(): + t = Tester(State( + relations=[Relation( + endpoint='tracing', + interface='tracing', + remote_app_name='remote', + local_app_data={} + )] + )) + state_out = t.run("tracing-relation-changed") diff --git a/tests/resources/cri-like-path/interfaces/tracing/v42/interface_tests/test_requirer.py b/tests/resources/cri-like-path/interfaces/tracing/v42/interface_tests/test_requirer.py new file mode 100644 index 0000000..671ff5f --- /dev/null +++ b/tests/resources/cri-like-path/interfaces/tracing/v42/interface_tests/test_requirer.py @@ -0,0 +1,30 @@ +# Copyright 2023 Canonical +# See LICENSE file for licensing details. + +from scenario import State, Relation + +from interface_tester.interface_test import Tester + + +def test_no_data_on_created(): + t = Tester(State()) + state_out = t.run(event='tracing-relation-created') + t.assert_relation_data_empty() + + +def test_no_data_on_joined(): + t = Tester() + state_out = t.run(event='tracing-relation-joined') + t.assert_relation_data_empty() + + +def test_data_on_changed(): + t = Tester(State( + relations=[Relation( + endpoint='tracing', + interface='tracing', + remote_app_name='remote', + local_app_data={} + )] + )) + state_out = t.run("tracing-relation-changed") diff --git a/tests/resources/cri-like-path/interfaces/tracing/v42/schema.py b/tests/resources/cri-like-path/interfaces/tracing/v42/schema.py new file mode 100644 index 0000000..4479710 --- /dev/null +++ b/tests/resources/cri-like-path/interfaces/tracing/v42/schema.py @@ -0,0 +1,28 @@ +from enum import Enum +from typing import List + +from interface_tester.schema_base import DataBagSchema +from pydantic import BaseModel, Json + + +class IngesterProtocol(str, Enum): + otlp_grpc = "otlp_grpc" + otlp_http = "otlp_http" + zipkin = "zipkin" + tempo = "tempo" + + +class Ingester(BaseModel): + port: str + protocol: IngesterProtocol + + +class TracingRequirerData(BaseModel): + host: str + ingesters: Json[List[Ingester]] + + +class RequirerSchema(DataBagSchema): + """Requirer schema for Tracing.""" + app: TracingRequirerData + diff --git a/tests/unit/test_collect_interface_tests.py b/tests/unit/test_collect_interface_tests.py index 2df901b..1baa831 100644 --- a/tests/unit/test_collect_interface_tests.py +++ b/tests/unit/test_collect_interface_tests.py @@ -1,19 +1,11 @@ -import importlib -import random -import string -import sys -from textwrap import dedent - import pytest from interface_tester.collector import collect_tests from interface_tester.interface_test import ( InvalidTestCase, - Role, check_test_case_validator_signature, - get_interface_name_and_version, - get_registered_test_cases, ) +from tests.unit.utils import CRI_LIKE_PATH def test_signature_checker_too_many_params(): @@ -30,8 +22,8 @@ def _foo(a: int): check_test_case_validator_signature(_foo) assert ( - "interface test case validator will receive a State as first and " - "only positional argument." in caplog.text + "interface test case validator will receive a State as first and " + "only positional argument." in caplog.text ) @@ -43,102 +35,15 @@ def _foo(a, b=2, c="a"): check_test_case_validator_signature(_foo) -@pytest.mark.parametrize("role", list(Role)) -@pytest.mark.parametrize("event", ("start", "update-status", "foo-relation-joined")) -@pytest.mark.parametrize("input_state", ("State()", "State(leader=True)")) -@pytest.mark.parametrize("intf_name", ("foo", "bar")) -@pytest.mark.parametrize("version", (0, 42)) -def test_registered_test_cases_cache(tmp_path, role, event, input_state, intf_name, version): - unique_name = "".join(random.choices(string.ascii_letters + string.digits, k=16)) - - # if the module name is not unique, importing it multiple times will result in royal confusion - pth = ( - tmp_path - / "interfaces" - / intf_name - / f"v{version}" - / "interface_tests" - / f"mytestcase_{unique_name}.py" - ) - pth.parent.mkdir(parents=True) - - pth.write_text( - dedent( - f""" -from interface_tester.interface_test import interface_test_case, Role -from scenario import State - -@interface_test_case( - "{role.value}", - "{event}", - input_state={input_state}, - name="{unique_name}", - schema='skip' -) -def foo(state_out: State): - pass - """ - ) - ) - - collect_tests(tmp_path) - registered = get_registered_test_cases()[(intf_name, version, role)] - - # exactly one test found with the unique name we just created - assert len([x for x in registered if x.name == unique_name]) == 1 - +def test_load_from_mock_cri(): + tests = collect_tests(CRI_LIKE_PATH) + provider = tests['interface_name']["v42"]["provider"] + assert len(provider['tests']) == 3 + assert not provider['schema'] + assert provider['charms'][0].name == 'tempo-k8s' -@pytest.mark.parametrize("intf_name", ("foo", "bar")) -@pytest.mark.parametrize("version", (0, 42)) -def test_get_interface_name_and_version(tmp_path, intf_name, version): - unique_name = "".join(random.choices(string.ascii_letters + string.digits, k=16)) + requirer = tests['interface_name']["v42"]["requirer"] + assert len(requirer['tests']) == 3 + assert requirer['schema'] + assert not requirer['charms'] - pth = ( - tmp_path - / "interfaces" - / intf_name - / f"v{version}" - / "interface_tests" - / f"mytestcase_{unique_name}.py" - ) - pth.parent.mkdir(parents=True) - pth.write_text("def foo(): pass") - - # so we can import without tricks - sys.path.append(str(pth.parent)) - module_name = str(pth.with_suffix("").name) - module = importlib.import_module(module_name) - # cleanup - sys.path.pop(-1) - - foo_fn = getattr(module, "foo") - assert get_interface_name_and_version(foo_fn) == (intf_name, version) - - -@pytest.mark.parametrize("intf_name", ("foo", "bar")) -@pytest.mark.parametrize("version", (0, 42)) -def test_get_interface_name_and_version_raises(tmp_path, intf_name, version): - unique_name = "".join(random.choices(string.ascii_letters + string.digits, k=16)) - - pth = ( - tmp_path - / "gibber" - / "ish" - / intf_name - / f"v{version}" - / "boots" - / f"mytestcase_{unique_name}.py" - ) - pth.parent.mkdir(parents=True) - pth.write_text("def foo(): pass") - - # so we can import without tricks - sys.path.append(str(pth.parent)) - module_name = str(pth.with_suffix("").name) - module = importlib.import_module(module_name) - # cleanup - sys.path.pop(-1) - - foo_fn = getattr(module, "foo") - with pytest.raises(InvalidTestCase): - get_interface_name_and_version(foo_fn) diff --git a/tests/unit/test_e2e.py b/tests/unit/test_e2e.py new file mode 100644 index 0000000..d9b3fc2 --- /dev/null +++ b/tests/unit/test_e2e.py @@ -0,0 +1,267 @@ +import tempfile +from pathlib import Path +from textwrap import dedent + +import pytest +from ops import CharmBase +from scenario import State + +from interface_tester import InterfaceTester +from interface_tester.collector import InterfaceTestSpec, gather_test_spec_for_version +from interface_tester.errors import InterfaceTestsFailed +from interface_tester.interface_test import InvalidTesterRunError, NoSchemaError +from tests.unit.utils import CRI_LIKE_PATH + + + +class LocalTester(InterfaceTester): + _RAISE_IMMEDIATELY = True + def _collect_interface_test_specs(self): + return gather_test_spec_for_version( + CRI_LIKE_PATH/"interfaces"/self._interface_name/f"v{self._interface_version}", + interface_name=self._interface_name, + version=self._interface_version, + ) + + +class DummiCharm(CharmBase): + pass + + +@pytest.fixture +def interface_tester(): + interface_tester = LocalTester() + interface_tester.configure( + charm_type=DummiCharm, + meta={"name": "dummi", + "provides": {"tracing": {"interface": "tracing"}}, + "requires": {"tracing-req": {"interface": "tracing"}} + }, + state_template=State(leader=True), + ) + yield interface_tester + + +def test_local_run(interface_tester): + interface_tester.configure( + interface_name="tracing", + interface_version=42, + ) + interface_tester.run() + + +def _setup_with_test_file(contents:str): + td = tempfile.TemporaryDirectory() + temppath = Path(td.name) + + class TempDirTester(InterfaceTester): + _RAISE_IMMEDIATELY = True + + def _collect_interface_test_specs(self): + pth = temppath / "interfaces" / self._interface_name / f"v{self._interface_version}" + + test_dir = pth / 'interface_tests' + test_dir.mkdir(parents=True) + test_provider = test_dir / 'test_provider.py' + test_provider.write_text(contents) + + return gather_test_spec_for_version( + pth, + interface_name=self._interface_name, + version=self._interface_version, + ) + + interface_tester = TempDirTester() + interface_tester.configure( + interface_name="tracing", + charm_type=DummiCharm, + meta={"name": "dummi", + "provides": {"tracing": {"interface": "tracing"}}, + "requires": {"tracing-req": {"interface": "tracing"}} + }, + state_template=State(leader=True), + ) + + return interface_tester + +def test_error_if_skip_schema_before_run(): + tester = _setup_with_test_file(dedent(""" +from scenario import State, Relation + +from interface_tester.interface_test import Tester + +def test_data_on_changed(): + t = Tester(State( + relations=[Relation( + endpoint='tracing', + interface='tracing', + remote_app_name='remote', + local_app_data={} + )] + )) + # state_out = t.run("tracing-relation-changed") + t.skip_schema_validation() +""" +)) + + with pytest.raises(InterfaceTestsFailed): + tester.run() + + +def test_error_if_assert_relation_data_empty_before_run(): + tester = _setup_with_test_file(dedent(""" +from scenario import State, Relation + +from interface_tester.interface_test import Tester + +def test_data_on_changed(): + t = Tester(State( + relations=[Relation( + endpoint='tracing', + interface='tracing', + remote_app_name='remote', + local_app_data={} + )] + )) + # state_out = t.run("tracing-relation-changed") + t.assert_relation_data_empty() +""" +)) + + with pytest.raises(InterfaceTestsFailed): + tester.run() + + +def test_error_if_assert_schema_valid_before_run(): + tester = _setup_with_test_file(dedent(""" +from scenario import State, Relation + +from interface_tester.interface_test import Tester + +def test_data_on_changed(): + t = Tester(State( + relations=[Relation( + endpoint='tracing', + interface='tracing', + remote_app_name='remote', + local_app_data={} + )] + )) + # state_out = t.run("tracing-relation-changed") + t.assert_schema_valid() +""" +)) + + with pytest.raises(InterfaceTestsFailed): + tester.run() + + +def test_error_if_assert_schema_without_schema(): + tester = _setup_with_test_file(dedent(""" +from scenario import State, Relation + +from interface_tester.interface_test import Tester + +def test_data_on_changed(): + t = Tester(State( + relations=[Relation( + endpoint='tracing', + interface='tracing', + remote_app_name='remote', + local_app_data={} + )] + )) + state_out = t.run("tracing-relation-changed") + t.assert_schema_valid() +""" +)) + + with pytest.raises(InterfaceTestsFailed): + tester.run() + + +def test_error_if_return_before_schema_call(): + tester = _setup_with_test_file(dedent(""" +from scenario import State, Relation + +from interface_tester.interface_test import Tester + +def test_data_on_changed(): + t = Tester(State( + relations=[Relation( + endpoint='tracing', + interface='tracing', + remote_app_name='remote', + local_app_data={} + )] + )) + state_out = t.run("tracing-relation-changed") +""" +)) + + with pytest.raises(InterfaceTestsFailed): + tester.run() + + +def test_error_if_return_without_run(): + tester = _setup_with_test_file(dedent(""" +from scenario import State, Relation + +from interface_tester.interface_test import Tester + +def test_data_on_changed(): + t = Tester(State( + relations=[Relation( + endpoint='tracing', + interface='tracing', + remote_app_name='remote', + local_app_data={} + )] + )) + +""" +)) + + with pytest.raises(InterfaceTestsFailed): + tester.run() + + +def test_error_if_return_without_tester_init(): + tester = _setup_with_test_file(dedent(""" +from scenario import State, Relation + +from interface_tester.interface_test import Tester + +def test_data_on_changed(): + pass + +""" + )) + + with pytest.raises(InterfaceTestsFailed): + tester.run() + + +def test_valid_run(): + tester = _setup_with_test_file(dedent(""" + from scenario import State, Relation + + from interface_tester.interface_test import Tester + from interface_tester.schema_base import DataBagSchema + + def test_data_on_changed(): + t = Tester(State( + relations=[Relation( + endpoint='tracing', + interface='tracing', + remote_app_name='remote', + local_app_data={} + )] + )) + state_out = t.run("tracing-relation-changed") + t.assert_schema_valid(schema=DataBagSchema()) + """ + )) + + tester.run() + diff --git a/tests/unit/utils.py b/tests/unit/utils.py new file mode 100644 index 0000000..c107b6e --- /dev/null +++ b/tests/unit/utils.py @@ -0,0 +1,3 @@ +from pathlib import Path + +CRI_LIKE_PATH = Path(__file__).parent.parent/'resources' / "cri-like-path" From b77300744a090d7b55edc98ed40186fcd8a9fd09 Mon Sep 17 00:00:00 2001 From: Pietro Pasotti Date: Wed, 6 Sep 2023 11:38:41 +0200 Subject: [PATCH 02/10] tests green --- interface_tester/interface_test.py | 88 ++++++++++++++----- interface_tester/plugin.py | 3 +- .../v42/interface_tests/test_provider.py | 1 + .../v42/interface_tests/test_requirer.py | 1 + tests/unit/test_e2e.py | 20 ++--- 5 files changed, 81 insertions(+), 32 deletions(-) diff --git a/interface_tester/interface_test.py b/interface_tester/interface_test.py index a20920e..00f01d9 100644 --- a/interface_tester/interface_test.py +++ b/interface_tester/interface_test.py @@ -46,12 +46,18 @@ class _InterfaceTestContext: """The version of the interface that this test is about.""" role: Role - # fixme charm_type: type + """Charm class being tested""" supported_endpoints: dict + """Supported relation endpoints.""" meta: Any + """Charm metadata.yaml""" config: Any + """Charm config.yaml""" actions: Any + """Charm actions.yaml""" + test_fn: Callable + """Test function.""" """The role (provider|requirer) that this test is about.""" schema: Optional["DataBagSchema"] = None @@ -98,17 +104,41 @@ def check_test_case_validator_signature(fn: Callable): def tester_context(ctx: _InterfaceTestContext): global _TESTER_CTX _TESTER_CTX = ctx - yield - _TESTER_CTX = None - if not Tester.__instance__: - raise NoTesterInstanceError('invalid tester_context usage: no Tester instance created') - Tester.__instance__._finalize() + + try: + yield + except Exception as e: + tester = Tester.__instance__ + + if tester: + tester._detach() + + _TESTER_CTX = None + raise + + tester = Tester.__instance__ + + if not tester: + raise NoTesterInstanceError(f'Invalid test: {ctx.test_fn} did not instantiate Tester.') + + try: + tester._finalize() + finally: + tester._detach() + _TESTER_CTX = None + + if Tester.__instance__: + raise RuntimeError("cleanup failed, tester instance still bound") class InvalidTesterRunError(RuntimeError): """Raised if Tester is being used incorrectly.""" + def __init__(self, test_name, msg): + _msg = f"failed running {test_name}: invalid test. {msg}" + super().__init__(_msg) + -class NoTesterInstanceError(InvalidTesterRunError): +class NoTesterInstanceError(RuntimeError): """Raised if no Tester is created within a tester_context scope.""" @@ -135,12 +165,16 @@ def __init__(self, state_in: State=None, name: str = None): self._state_template = None self._state_in = state_in or State() - self._name = name + self._test_name = name or self.ctx.test_fn.__name__ self._state_out = None # will be State when has_run is true self._has_run = False self._has_checked_schema = False + @property + def _test_id(self): + return f"{self.ctx.interface_name}[{self.ctx.version}]/{self.ctx.role}:{self._test_name}" + @property def ctx(self): return _TESTER_CTX @@ -152,10 +186,14 @@ def run(self, event: Union[str, Event]): self._state_out = state_out return state_out + @property + def _relations(self) -> List[Relation]: + return [r for r in self._state_out.relations if r.interface == self.ctx.interface_name] + def assert_schema_valid(self, schema:"DataBagSchema" = None): self._has_checked_schema = True if not self._has_run: - raise RuntimeError('call Tester.run() first') + raise InvalidTesterRunError(self._test_id, 'call Tester.run() first') if schema: logger.info("running test with custom schema") @@ -165,11 +203,12 @@ def assert_schema_valid(self, schema:"DataBagSchema" = None): databag_schema = self.ctx.schema if not databag_schema: raise NoSchemaError( - f"No schemas found for {self.ctx.interface_name}/{self.ctx.version}/{self.ctx.role};" - f"call skip_schema_validation() manually.") + self._test_id, + "No schema found. If this is expected, call Tester.skip_schema_validation() instead." + ) errors = [] - for relation in [r for r in self._state_out.relations if r.interface == self.ctx.interface_name]: + for relation in self._relations: try: databag_schema.validate( { @@ -184,29 +223,38 @@ def assert_schema_valid(self, schema:"DataBagSchema" = None): def _check_has_run(self): if not self._has_run: - raise InvalidTesterRunError('call Tester.run() first') + raise InvalidTesterRunError(self._test_id, 'Call Tester.run() first.') def assert_relation_data_empty(self): self._check_has_run() - # todo + for relation in self._relations: + if relation.local_app_data: + raise SchemaValidationError(f"test {self._test_id}: local app databag not empty for {relation}") + if relation.local_unit_data: + raise SchemaValidationError(f"test {self._test_id}: local unit databag not empty for {relation}") self._has_checked_schema = True def skip_schema_validation(self): self._check_has_run() - # todo + logger.debug("skipping schema validation") self._has_checked_schema = True def _finalize(self): if not self._has_run: - raise InvalidTesterRunError("call .run() before returning") + raise InvalidTesterRunError(self._test_id, + "Test function must call Tester.run() before returning.") if not self._has_checked_schema: - # todo: - raise InvalidTesterRunError("call .skip_schema_validation(), or ... before returning") - + raise InvalidTesterRunError(self._test_id, + "Test function must call " + "Tester.skip_schema_validation(), or " + "Tester.assert_schema_valid(), or " + "Tester.assert_schema_empty() before returning.") + self._detach() + + def _detach(self): # release singleton Tester.__instance__ = None - def _run(self, event: Union[str, Event]): logger.debug(f"running {event}") self._has_run = True diff --git a/interface_tester/plugin.py b/interface_tester/plugin.py index 966a960..3abe424 100644 --- a/interface_tester/plugin.py +++ b/interface_tester/plugin.py @@ -316,7 +316,8 @@ def run(self) -> bool: meta=self.meta, config=self.config, actions=self.actions, - supported_endpoints=self._gather_supported_endpoints() + supported_endpoints=self._gather_supported_endpoints(), + test_fn=test_fn ) try: with tester_context(ctx): diff --git a/tests/resources/cri-like-path/interfaces/tracing/v42/interface_tests/test_provider.py b/tests/resources/cri-like-path/interfaces/tracing/v42/interface_tests/test_provider.py index 671ff5f..547be60 100644 --- a/tests/resources/cri-like-path/interfaces/tracing/v42/interface_tests/test_provider.py +++ b/tests/resources/cri-like-path/interfaces/tracing/v42/interface_tests/test_provider.py @@ -28,3 +28,4 @@ def test_data_on_changed(): )] )) state_out = t.run("tracing-relation-changed") + t.assert_relation_data_empty() diff --git a/tests/resources/cri-like-path/interfaces/tracing/v42/interface_tests/test_requirer.py b/tests/resources/cri-like-path/interfaces/tracing/v42/interface_tests/test_requirer.py index 671ff5f..547be60 100644 --- a/tests/resources/cri-like-path/interfaces/tracing/v42/interface_tests/test_requirer.py +++ b/tests/resources/cri-like-path/interfaces/tracing/v42/interface_tests/test_requirer.py @@ -28,3 +28,4 @@ def test_data_on_changed(): )] )) state_out = t.run("tracing-relation-changed") + t.assert_relation_data_empty() diff --git a/tests/unit/test_e2e.py b/tests/unit/test_e2e.py index d9b3fc2..a0b046f 100644 --- a/tests/unit/test_e2e.py +++ b/tests/unit/test_e2e.py @@ -9,7 +9,7 @@ from interface_tester import InterfaceTester from interface_tester.collector import InterfaceTestSpec, gather_test_spec_for_version from interface_tester.errors import InterfaceTestsFailed -from interface_tester.interface_test import InvalidTesterRunError, NoSchemaError +from interface_tester.interface_test import InvalidTesterRunError, NoSchemaError, Tester, NoTesterInstanceError from tests.unit.utils import CRI_LIKE_PATH @@ -99,12 +99,11 @@ def test_data_on_changed(): local_app_data={} )] )) - # state_out = t.run("tracing-relation-changed") t.skip_schema_validation() """ )) - with pytest.raises(InterfaceTestsFailed): + with pytest.raises(InvalidTesterRunError): tester.run() @@ -123,13 +122,13 @@ def test_data_on_changed(): local_app_data={} )] )) - # state_out = t.run("tracing-relation-changed") t.assert_relation_data_empty() """ )) - with pytest.raises(InterfaceTestsFailed): + with pytest.raises(InvalidTesterRunError): tester.run() + assert not Tester.__instance__ def test_error_if_assert_schema_valid_before_run(): @@ -147,12 +146,11 @@ def test_data_on_changed(): local_app_data={} )] )) - # state_out = t.run("tracing-relation-changed") t.assert_schema_valid() """ )) - with pytest.raises(InterfaceTestsFailed): + with pytest.raises(InvalidTesterRunError): tester.run() @@ -176,7 +174,7 @@ def test_data_on_changed(): """ )) - with pytest.raises(InterfaceTestsFailed): + with pytest.raises(NoSchemaError): tester.run() @@ -199,7 +197,7 @@ def test_data_on_changed(): """ )) - with pytest.raises(InterfaceTestsFailed): + with pytest.raises(InvalidTesterRunError): tester.run() @@ -222,7 +220,7 @@ def test_data_on_changed(): """ )) - with pytest.raises(InterfaceTestsFailed): + with pytest.raises(InvalidTesterRunError): tester.run() @@ -238,7 +236,7 @@ def test_data_on_changed(): """ )) - with pytest.raises(InterfaceTestsFailed): + with pytest.raises(NoTesterInstanceError): tester.run() From 24f1e5d59a8eebc6aaac5f4e9523db6320ab9ef8 Mon Sep 17 00:00:00 2001 From: Pietro Pasotti Date: Wed, 6 Sep 2023 11:38:59 +0200 Subject: [PATCH 03/10] lint --- interface_tester/collector.py | 2 +- interface_tester/errors.py | 1 + interface_tester/interface_test.py | 54 ++++++----- interface_tester/plugin.py | 66 ++++++------- .../tests/interface/conftest.py | 12 +-- .../v42/interface_tests/test_provider.py | 26 +++--- .../v42/interface_tests/test_requirer.py | 26 +++--- .../interfaces/tracing/v42/schema.py | 5 +- tests/unit/test_collect_interface_tests.py | 23 +++-- tests/unit/test_e2e.py | 93 ++++++++++++------- tests/unit/utils.py | 2 +- 11 files changed, 177 insertions(+), 133 deletions(-) diff --git a/interface_tester/collector.py b/interface_tester/collector.py index aaccf4c..caa1822 100644 --- a/interface_tester/collector.py +++ b/interface_tester/collector.py @@ -17,7 +17,7 @@ import sys import types from pathlib import Path -from typing import Dict, List, Literal, Optional, Type, TypedDict, Callable +from typing import Callable, Dict, List, Literal, Optional, Type, TypedDict import pydantic import yaml diff --git a/interface_tester/errors.py b/interface_tester/errors.py index 297e99c..b23d648 100644 --- a/interface_tester/errors.py +++ b/interface_tester/errors.py @@ -15,5 +15,6 @@ class InterfaceTestsFailed(RuntimeError): class NoTestsRun(RuntimeError): """Raised if no interface test was collected during a run() call.""" + class SchemaValidationError(RuntimeError): """Raised when schema validation fails on one or more relations.""" diff --git a/interface_tester/interface_test.py b/interface_tester/interface_test.py index 00f01d9..1b0fd26 100644 --- a/interface_tester/interface_test.py +++ b/interface_tester/interface_test.py @@ -8,9 +8,9 @@ import typing from contextlib import contextmanager from enum import Enum -from typing import Callable, Literal, Optional, Union, Any, Dict, List +from typing import Any, Callable, Dict, List, Literal, Optional, Union -from scenario import Event, State, Context, Relation +from scenario import Context, Event, Relation, State from interface_tester.errors import InvalidTestCaseError, SchemaValidationError @@ -119,7 +119,7 @@ def tester_context(ctx: _InterfaceTestContext): tester = Tester.__instance__ if not tester: - raise NoTesterInstanceError(f'Invalid test: {ctx.test_fn} did not instantiate Tester.') + raise NoTesterInstanceError(f"Invalid test: {ctx.test_fn} did not instantiate Tester.") try: tester._finalize() @@ -133,6 +133,7 @@ def tester_context(ctx: _InterfaceTestContext): class InvalidTesterRunError(RuntimeError): """Raised if Tester is being used incorrectly.""" + def __init__(self, test_name, msg): _msg = f"failed running {test_name}: invalid test. {msg}" super().__init__(_msg) @@ -149,7 +150,7 @@ class NoSchemaError(InvalidTesterRunError): class Tester: __instance__ = None - def __init__(self, state_in: State=None, name: str = None): + def __init__(self, state_in: State = None, name: str = None): """Initializer. :param state_in: the input state for this scenario test. Will default to the empty State(). @@ -161,7 +162,7 @@ def __init__(self, state_in: State=None, name: str = None): Tester.__instance__ = self if not self.ctx: - raise RuntimeError('Tester can only be initialized inside a tester context.') + raise RuntimeError("Tester can only be initialized inside a tester context.") self._state_template = None self._state_in = state_in or State() @@ -180,7 +181,7 @@ def ctx(self): return _TESTER_CTX def run(self, event: Union[str, Event]): - assert self.ctx, 'tester cannot run: no _TESTER_CTX set' + assert self.ctx, "tester cannot run: no _TESTER_CTX set" state_out = self._run(event) self._state_out = state_out @@ -190,10 +191,10 @@ def run(self, event: Union[str, Event]): def _relations(self) -> List[Relation]: return [r for r in self._state_out.relations if r.interface == self.ctx.interface_name] - def assert_schema_valid(self, schema:"DataBagSchema" = None): + def assert_schema_valid(self, schema: "DataBagSchema" = None): self._has_checked_schema = True if not self._has_run: - raise InvalidTesterRunError(self._test_id, 'call Tester.run() first') + raise InvalidTesterRunError(self._test_id, "call Tester.run() first") if schema: logger.info("running test with custom schema") @@ -204,7 +205,7 @@ def assert_schema_valid(self, schema:"DataBagSchema" = None): if not databag_schema: raise NoSchemaError( self._test_id, - "No schema found. If this is expected, call Tester.skip_schema_validation() instead." + "No schema found. If this is expected, call Tester.skip_schema_validation() instead.", ) errors = [] @@ -223,15 +224,19 @@ def assert_schema_valid(self, schema:"DataBagSchema" = None): def _check_has_run(self): if not self._has_run: - raise InvalidTesterRunError(self._test_id, 'Call Tester.run() first.') + raise InvalidTesterRunError(self._test_id, "Call Tester.run() first.") def assert_relation_data_empty(self): self._check_has_run() for relation in self._relations: if relation.local_app_data: - raise SchemaValidationError(f"test {self._test_id}: local app databag not empty for {relation}") + raise SchemaValidationError( + f"test {self._test_id}: local app databag not empty for {relation}" + ) if relation.local_unit_data: - raise SchemaValidationError(f"test {self._test_id}: local unit databag not empty for {relation}") + raise SchemaValidationError( + f"test {self._test_id}: local unit databag not empty for {relation}" + ) self._has_checked_schema = True def skip_schema_validation(self): @@ -241,14 +246,17 @@ def skip_schema_validation(self): def _finalize(self): if not self._has_run: - raise InvalidTesterRunError(self._test_id, - "Test function must call Tester.run() before returning.") + raise InvalidTesterRunError( + self._test_id, "Test function must call Tester.run() before returning." + ) if not self._has_checked_schema: - raise InvalidTesterRunError(self._test_id, + raise InvalidTesterRunError( + self._test_id, "Test function must call " "Tester.skip_schema_validation(), or " "Tester.assert_schema_valid(), or " - "Tester.assert_schema_empty() before returning.") + "Tester.assert_schema_empty() before returning.", + ) self._detach() def _detach(self): @@ -272,9 +280,7 @@ def _run(self, event: Union[str, Event]): state = (self._state_template or State()).copy() relations = self._generate_relations_state( - state, input_state, - self.ctx.supported_endpoints, - self.ctx.role + state, input_state, self.ctx.supported_endpoints, self.ctx.role ) # State is frozen; replace modified_state = state.replace(relations=relations) @@ -290,8 +296,12 @@ def _run(self, event: Union[str, Event]): def _run_scenario(self, event: Event, state: State): logger.debug(f"running scenario with state={state}, event={event}") - ctx = Context(self.ctx.charm_type, meta=self.ctx.meta, - actions=self.ctx.actions, config=self.ctx.config) + ctx = Context( + self.ctx.charm_type, + meta=self.ctx.meta, + actions=self.ctx.actions, + config=self.ctx.config, + ) return ctx.run(event, state) def _coerce_event(self, raw_event: Union[str, Event], relation: Relation) -> Event: @@ -334,7 +344,7 @@ def _coerce_event(self, raw_event: Union[str, Event], relation: Relation) -> Eve ) def _generate_relations_state( - self, state_template: State, input_state: State, supported_endpoints, role: Role + self, state_template: State, input_state: State, supported_endpoints, role: Role ) -> List[Relation]: """Merge the relations from the input state and the state template into one. diff --git a/interface_tester/plugin.py b/interface_tester/plugin.py index 3abe424..606835f 100644 --- a/interface_tester/plugin.py +++ b/interface_tester/plugin.py @@ -4,29 +4,23 @@ import tempfile from pathlib import Path from subprocess import PIPE, Popen -from typing import ( - Any, - Callable, - Dict, - Generator, - List, - Literal, - Optional, - Tuple, - Type, -) +from typing import Any, Callable, Dict, Generator, List, Literal, Optional, Tuple, Type from ops.testing import CharmType from scenario.state import Event, State, _CharmSpec -from interface_tester.schema_base import DataBagSchema from interface_tester.collector import InterfaceTestSpec, gather_test_spec_for_version from interface_tester.errors import ( InterfaceTesterValidationError, InterfaceTestsFailed, NoTestsRun, ) -from interface_tester.interface_test import _InterfaceTestContext, tester_context, RoleLiteral +from interface_tester.interface_test import ( + RoleLiteral, + _InterfaceTestContext, + tester_context, +) +from interface_tester.schema_base import DataBagSchema Callback = Callable[[State, Event], None] ROLE_TO_ROLE_META = {"provider": "provides", "requirer": "requires"} @@ -38,10 +32,10 @@ class InterfaceTester: _RAISE_IMMEDIATELY = False def __init__( - self, - repo: str = "https://github.com/canonical/charm-relation-interfaces", - branch: str = "main", - base_path: str = "interfaces", + self, + repo: str = "https://github.com/canonical/charm-relation-interfaces", + branch: str = "main", + base_path: str = "interfaces", ): self._repo = repo self._branch = branch @@ -59,18 +53,18 @@ def __init__( self._charm_spec_cache = None def configure( - self, - *, - charm_type: Optional[Type[CharmType]] = None, - repo: Optional[str] = None, - branch: Optional[str] = None, - base_path: Optional[str] = None, - interface_name: Optional[str] = None, - interface_version: Optional[int] = None, - state_template: Optional[State] = None, - meta: Optional[Dict[str, Any]] = None, - actions: Optional[Dict[str, Any]] = None, - config: Optional[Dict[str, Any]] = None, + self, + *, + charm_type: Optional[Type[CharmType]] = None, + repo: Optional[str] = None, + branch: Optional[str] = None, + base_path: Optional[str] = None, + interface_name: Optional[str] = None, + interface_version: Optional[int] = None, + state_template: Optional[State] = None, + meta: Optional[Dict[str, Any]] = None, + actions: Optional[Dict[str, Any]] = None, + config: Optional[Dict[str, Any]] = None, ): """ @@ -196,11 +190,11 @@ def _collect_interface_test_specs(self) -> InterfaceTestSpec: repo_name = self._repo.split("/")[-1] intf_spec_path = ( - Path(tempdir) - / repo_name - / self._base_path - / self._interface_name.replace("-", "_") - / f"v{self._interface_version}" + Path(tempdir) + / repo_name + / self._base_path + / self._interface_name.replace("-", "_") + / f"v{self._interface_version}" ) if not intf_spec_path.exists(): raise RuntimeError( @@ -246,7 +240,7 @@ def _gather_supported_endpoints(self) -> Dict[RoleLiteral, List[str]]: return supported_endpoints def _yield_tests( - self, + self, ) -> Generator[Tuple[Callable, RoleLiteral, DataBagSchema], None, None]: """Yield all test cases applicable to this charm and interface. @@ -317,7 +311,7 @@ def run(self) -> bool: config=self.config, actions=self.actions, supported_endpoints=self._gather_supported_endpoints(), - test_fn=test_fn + test_fn=test_fn, ) try: with tester_context(ctx): diff --git a/tests/resources/charm-like-path/tests/interface/conftest.py b/tests/resources/charm-like-path/tests/interface/conftest.py index ab5133c..b20117b 100644 --- a/tests/resources/charm-like-path/tests/interface/conftest.py +++ b/tests/resources/charm-like-path/tests/interface/conftest.py @@ -5,8 +5,6 @@ from ops import CharmBase from scenario.state import State -from interface_tester import InterfaceTester - from interface_tester import InterfaceTester from interface_tester.collector import gather_test_spec_for_version from tests.unit.utils import CRI_LIKE_PATH @@ -20,6 +18,7 @@ def _collect_interface_test_specs(self): version=self._interface_version, ) + class DummiCharm(CharmBase): pass @@ -28,10 +27,11 @@ class DummiCharm(CharmBase): def interface_tester(interface_tester: CRILikePathTester): interface_tester.configure( charm_type=DummiCharm, - meta={"name": "dummi", - "provides": {"tracing": {"interface": "tracing"}}, - "requires": {"tracing": {"interface": "tracing"}} - }, + meta={ + "name": "dummi", + "provides": {"tracing": {"interface": "tracing"}}, + "requires": {"tracing": {"interface": "tracing"}}, + }, state_template=State(leader=True), ) yield interface_tester diff --git a/tests/resources/cri-like-path/interfaces/tracing/v42/interface_tests/test_provider.py b/tests/resources/cri-like-path/interfaces/tracing/v42/interface_tests/test_provider.py index 547be60..31e25d2 100644 --- a/tests/resources/cri-like-path/interfaces/tracing/v42/interface_tests/test_provider.py +++ b/tests/resources/cri-like-path/interfaces/tracing/v42/interface_tests/test_provider.py @@ -1,31 +1,35 @@ # Copyright 2023 Canonical # See LICENSE file for licensing details. -from scenario import State, Relation +from scenario import Relation, State from interface_tester.interface_test import Tester def test_no_data_on_created(): t = Tester(State()) - state_out = t.run(event='tracing-relation-created') + state_out = t.run(event="tracing-relation-created") t.assert_relation_data_empty() def test_no_data_on_joined(): t = Tester() - state_out = t.run(event='tracing-relation-joined') + state_out = t.run(event="tracing-relation-joined") t.assert_relation_data_empty() def test_data_on_changed(): - t = Tester(State( - relations=[Relation( - endpoint='tracing', - interface='tracing', - remote_app_name='remote', - local_app_data={} - )] - )) + t = Tester( + State( + relations=[ + Relation( + endpoint="tracing", + interface="tracing", + remote_app_name="remote", + local_app_data={}, + ) + ] + ) + ) state_out = t.run("tracing-relation-changed") t.assert_relation_data_empty() diff --git a/tests/resources/cri-like-path/interfaces/tracing/v42/interface_tests/test_requirer.py b/tests/resources/cri-like-path/interfaces/tracing/v42/interface_tests/test_requirer.py index 547be60..31e25d2 100644 --- a/tests/resources/cri-like-path/interfaces/tracing/v42/interface_tests/test_requirer.py +++ b/tests/resources/cri-like-path/interfaces/tracing/v42/interface_tests/test_requirer.py @@ -1,31 +1,35 @@ # Copyright 2023 Canonical # See LICENSE file for licensing details. -from scenario import State, Relation +from scenario import Relation, State from interface_tester.interface_test import Tester def test_no_data_on_created(): t = Tester(State()) - state_out = t.run(event='tracing-relation-created') + state_out = t.run(event="tracing-relation-created") t.assert_relation_data_empty() def test_no_data_on_joined(): t = Tester() - state_out = t.run(event='tracing-relation-joined') + state_out = t.run(event="tracing-relation-joined") t.assert_relation_data_empty() def test_data_on_changed(): - t = Tester(State( - relations=[Relation( - endpoint='tracing', - interface='tracing', - remote_app_name='remote', - local_app_data={} - )] - )) + t = Tester( + State( + relations=[ + Relation( + endpoint="tracing", + interface="tracing", + remote_app_name="remote", + local_app_data={}, + ) + ] + ) + ) state_out = t.run("tracing-relation-changed") t.assert_relation_data_empty() diff --git a/tests/resources/cri-like-path/interfaces/tracing/v42/schema.py b/tests/resources/cri-like-path/interfaces/tracing/v42/schema.py index 4479710..d172c93 100644 --- a/tests/resources/cri-like-path/interfaces/tracing/v42/schema.py +++ b/tests/resources/cri-like-path/interfaces/tracing/v42/schema.py @@ -1,9 +1,10 @@ from enum import Enum from typing import List -from interface_tester.schema_base import DataBagSchema from pydantic import BaseModel, Json +from interface_tester.schema_base import DataBagSchema + class IngesterProtocol(str, Enum): otlp_grpc = "otlp_grpc" @@ -24,5 +25,5 @@ class TracingRequirerData(BaseModel): class RequirerSchema(DataBagSchema): """Requirer schema for Tracing.""" - app: TracingRequirerData + app: TracingRequirerData diff --git a/tests/unit/test_collect_interface_tests.py b/tests/unit/test_collect_interface_tests.py index 1baa831..4ba8d51 100644 --- a/tests/unit/test_collect_interface_tests.py +++ b/tests/unit/test_collect_interface_tests.py @@ -22,8 +22,8 @@ def _foo(a: int): check_test_case_validator_signature(_foo) assert ( - "interface test case validator will receive a State as first and " - "only positional argument." in caplog.text + "interface test case validator will receive a State as first and " + "only positional argument." in caplog.text ) @@ -37,13 +37,12 @@ def _foo(a, b=2, c="a"): def test_load_from_mock_cri(): tests = collect_tests(CRI_LIKE_PATH) - provider = tests['interface_name']["v42"]["provider"] - assert len(provider['tests']) == 3 - assert not provider['schema'] - assert provider['charms'][0].name == 'tempo-k8s' - - requirer = tests['interface_name']["v42"]["requirer"] - assert len(requirer['tests']) == 3 - assert requirer['schema'] - assert not requirer['charms'] - + provider = tests["interface_name"]["v42"]["provider"] + assert len(provider["tests"]) == 3 + assert not provider["schema"] + assert provider["charms"][0].name == "tempo-k8s" + + requirer = tests["interface_name"]["v42"]["requirer"] + assert len(requirer["tests"]) == 3 + assert requirer["schema"] + assert not requirer["charms"] diff --git a/tests/unit/test_e2e.py b/tests/unit/test_e2e.py index a0b046f..74e5a63 100644 --- a/tests/unit/test_e2e.py +++ b/tests/unit/test_e2e.py @@ -9,16 +9,21 @@ from interface_tester import InterfaceTester from interface_tester.collector import InterfaceTestSpec, gather_test_spec_for_version from interface_tester.errors import InterfaceTestsFailed -from interface_tester.interface_test import InvalidTesterRunError, NoSchemaError, Tester, NoTesterInstanceError +from interface_tester.interface_test import ( + InvalidTesterRunError, + NoSchemaError, + NoTesterInstanceError, + Tester, +) from tests.unit.utils import CRI_LIKE_PATH - class LocalTester(InterfaceTester): _RAISE_IMMEDIATELY = True + def _collect_interface_test_specs(self): return gather_test_spec_for_version( - CRI_LIKE_PATH/"interfaces"/self._interface_name/f"v{self._interface_version}", + CRI_LIKE_PATH / "interfaces" / self._interface_name / f"v{self._interface_version}", interface_name=self._interface_name, version=self._interface_version, ) @@ -33,10 +38,11 @@ def interface_tester(): interface_tester = LocalTester() interface_tester.configure( charm_type=DummiCharm, - meta={"name": "dummi", - "provides": {"tracing": {"interface": "tracing"}}, - "requires": {"tracing-req": {"interface": "tracing"}} - }, + meta={ + "name": "dummi", + "provides": {"tracing": {"interface": "tracing"}}, + "requires": {"tracing-req": {"interface": "tracing"}}, + }, state_template=State(leader=True), ) yield interface_tester @@ -50,7 +56,7 @@ def test_local_run(interface_tester): interface_tester.run() -def _setup_with_test_file(contents:str): +def _setup_with_test_file(contents: str): td = tempfile.TemporaryDirectory() temppath = Path(td.name) @@ -60,9 +66,9 @@ class TempDirTester(InterfaceTester): def _collect_interface_test_specs(self): pth = temppath / "interfaces" / self._interface_name / f"v{self._interface_version}" - test_dir = pth / 'interface_tests' + test_dir = pth / "interface_tests" test_dir.mkdir(parents=True) - test_provider = test_dir / 'test_provider.py' + test_provider = test_dir / "test_provider.py" test_provider.write_text(contents) return gather_test_spec_for_version( @@ -75,17 +81,21 @@ def _collect_interface_test_specs(self): interface_tester.configure( interface_name="tracing", charm_type=DummiCharm, - meta={"name": "dummi", - "provides": {"tracing": {"interface": "tracing"}}, - "requires": {"tracing-req": {"interface": "tracing"}} - }, + meta={ + "name": "dummi", + "provides": {"tracing": {"interface": "tracing"}}, + "requires": {"tracing-req": {"interface": "tracing"}}, + }, state_template=State(leader=True), ) return interface_tester + def test_error_if_skip_schema_before_run(): - tester = _setup_with_test_file(dedent(""" + tester = _setup_with_test_file( + dedent( + """ from scenario import State, Relation from interface_tester.interface_test import Tester @@ -101,14 +111,17 @@ def test_data_on_changed(): )) t.skip_schema_validation() """ -)) + ) + ) with pytest.raises(InvalidTesterRunError): tester.run() def test_error_if_assert_relation_data_empty_before_run(): - tester = _setup_with_test_file(dedent(""" + tester = _setup_with_test_file( + dedent( + """ from scenario import State, Relation from interface_tester.interface_test import Tester @@ -124,7 +137,8 @@ def test_data_on_changed(): )) t.assert_relation_data_empty() """ -)) + ) + ) with pytest.raises(InvalidTesterRunError): tester.run() @@ -132,7 +146,9 @@ def test_data_on_changed(): def test_error_if_assert_schema_valid_before_run(): - tester = _setup_with_test_file(dedent(""" + tester = _setup_with_test_file( + dedent( + """ from scenario import State, Relation from interface_tester.interface_test import Tester @@ -148,14 +164,17 @@ def test_data_on_changed(): )) t.assert_schema_valid() """ -)) + ) + ) with pytest.raises(InvalidTesterRunError): tester.run() def test_error_if_assert_schema_without_schema(): - tester = _setup_with_test_file(dedent(""" + tester = _setup_with_test_file( + dedent( + """ from scenario import State, Relation from interface_tester.interface_test import Tester @@ -172,14 +191,17 @@ def test_data_on_changed(): state_out = t.run("tracing-relation-changed") t.assert_schema_valid() """ -)) + ) + ) with pytest.raises(NoSchemaError): tester.run() def test_error_if_return_before_schema_call(): - tester = _setup_with_test_file(dedent(""" + tester = _setup_with_test_file( + dedent( + """ from scenario import State, Relation from interface_tester.interface_test import Tester @@ -195,14 +217,17 @@ def test_data_on_changed(): )) state_out = t.run("tracing-relation-changed") """ -)) + ) + ) with pytest.raises(InvalidTesterRunError): tester.run() def test_error_if_return_without_run(): - tester = _setup_with_test_file(dedent(""" + tester = _setup_with_test_file( + dedent( + """ from scenario import State, Relation from interface_tester.interface_test import Tester @@ -218,14 +243,17 @@ def test_data_on_changed(): )) """ -)) + ) + ) with pytest.raises(InvalidTesterRunError): tester.run() def test_error_if_return_without_tester_init(): - tester = _setup_with_test_file(dedent(""" + tester = _setup_with_test_file( + dedent( + """ from scenario import State, Relation from interface_tester.interface_test import Tester @@ -234,14 +262,17 @@ def test_data_on_changed(): pass """ - )) + ) + ) with pytest.raises(NoTesterInstanceError): tester.run() def test_valid_run(): - tester = _setup_with_test_file(dedent(""" + tester = _setup_with_test_file( + dedent( + """ from scenario import State, Relation from interface_tester.interface_test import Tester @@ -259,7 +290,7 @@ def test_data_on_changed(): state_out = t.run("tracing-relation-changed") t.assert_schema_valid(schema=DataBagSchema()) """ - )) + ) + ) tester.run() - diff --git a/tests/unit/utils.py b/tests/unit/utils.py index c107b6e..851deae 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -1,3 +1,3 @@ from pathlib import Path -CRI_LIKE_PATH = Path(__file__).parent.parent/'resources' / "cri-like-path" +CRI_LIKE_PATH = Path(__file__).parent.parent / "resources" / "cri-like-path" From 06996984d831d4b0dc9b393626a53b58029bc6be Mon Sep 17 00:00:00 2001 From: Pietro Pasotti Date: Wed, 6 Sep 2023 13:08:03 +0200 Subject: [PATCH 04/10] tests green --- interface_tester/interface_test.py | 3 +- interface_tester/plugin.py | 5 +- tests/unit/test_collect_interface_tests.py | 6 +- tests/unit/test_e2e.py | 175 ++++++++++++++++++++- tox.ini | 2 +- 5 files changed, 178 insertions(+), 13 deletions(-) diff --git a/interface_tester/interface_test.py b/interface_tester/interface_test.py index 1b0fd26..6060d65 100644 --- a/interface_tester/interface_test.py +++ b/interface_tester/interface_test.py @@ -10,6 +10,7 @@ from enum import Enum from typing import Any, Callable, Dict, List, Literal, Optional, Union +from pydantic import ValidationError from scenario import Context, Event, Relation, State from interface_tester.errors import InvalidTestCaseError, SchemaValidationError @@ -217,7 +218,7 @@ def assert_schema_valid(self, schema: "DataBagSchema" = None): "app": relation.local_app_data, } ) - except RuntimeError as e: + except ValidationError as e: errors.append(e.args[0]) if errors: raise SchemaValidationError(errors) diff --git a/interface_tester/plugin.py b/interface_tester/plugin.py index 606835f..2ca745d 100644 --- a/interface_tester/plugin.py +++ b/interface_tester/plugin.py @@ -7,7 +7,7 @@ from typing import Any, Callable, Dict, Generator, List, Literal, Optional, Tuple, Type from ops.testing import CharmType -from scenario.state import Event, State, _CharmSpec +from scenario.state import Event, MetadataNotFoundError, State, _CharmSpec from interface_tester.collector import InterfaceTestSpec, gather_test_spec_for_version from interface_tester.errors import ( @@ -141,8 +141,7 @@ def _charm_spec(self) -> _CharmSpec: spec = _CharmSpec.autoload(self._charm_type) # if no metadata.yaml can be found in the charm type module's parent directory, # this call will raise: - - except FileNotFoundError as e: + except MetadataNotFoundError as e: # if we have _meta set, we're good, otherwise we're misconfigured. if self._meta and self._charm_type: spec = _CharmSpec( diff --git a/tests/unit/test_collect_interface_tests.py b/tests/unit/test_collect_interface_tests.py index 4ba8d51..34ac9b8 100644 --- a/tests/unit/test_collect_interface_tests.py +++ b/tests/unit/test_collect_interface_tests.py @@ -1,11 +1,11 @@ import pytest +from utils import CRI_LIKE_PATH from interface_tester.collector import collect_tests from interface_tester.interface_test import ( InvalidTestCase, check_test_case_validator_signature, ) -from tests.unit.utils import CRI_LIKE_PATH def test_signature_checker_too_many_params(): @@ -37,12 +37,12 @@ def _foo(a, b=2, c="a"): def test_load_from_mock_cri(): tests = collect_tests(CRI_LIKE_PATH) - provider = tests["interface_name"]["v42"]["provider"] + provider = tests["tracing"]["v42"]["provider"] assert len(provider["tests"]) == 3 assert not provider["schema"] assert provider["charms"][0].name == "tempo-k8s" - requirer = tests["interface_name"]["v42"]["requirer"] + requirer = tests["tracing"]["v42"]["requirer"] assert len(requirer["tests"]) == 3 assert requirer["schema"] assert not requirer["charms"] diff --git a/tests/unit/test_e2e.py b/tests/unit/test_e2e.py index 74e5a63..5a021ed 100644 --- a/tests/unit/test_e2e.py +++ b/tests/unit/test_e2e.py @@ -5,17 +5,17 @@ import pytest from ops import CharmBase from scenario import State +from utils import CRI_LIKE_PATH from interface_tester import InterfaceTester -from interface_tester.collector import InterfaceTestSpec, gather_test_spec_for_version -from interface_tester.errors import InterfaceTestsFailed +from interface_tester.collector import gather_test_spec_for_version +from interface_tester.errors import SchemaValidationError from interface_tester.interface_test import ( InvalidTesterRunError, NoSchemaError, NoTesterInstanceError, Tester, ) -from tests.unit.utils import CRI_LIKE_PATH class LocalTester(InterfaceTester): @@ -56,7 +56,7 @@ def test_local_run(interface_tester): interface_tester.run() -def _setup_with_test_file(contents: str): +def _setup_with_test_file(test_file: str, schema_file: str = None): td = tempfile.TemporaryDirectory() temppath = Path(td.name) @@ -69,7 +69,11 @@ def _collect_interface_test_specs(self): test_dir = pth / "interface_tests" test_dir.mkdir(parents=True) test_provider = test_dir / "test_provider.py" - test_provider.write_text(contents) + test_provider.write_text(test_file) + + if schema_file: + schema_path = pth / "schema.py" + schema_path.write_text(schema_file) return gather_test_spec_for_version( pth, @@ -294,3 +298,164 @@ def test_data_on_changed(): ) tester.run() + + +def test_valid_run_default_schema(): + tester = _setup_with_test_file( + dedent( + """ + from scenario import State, Relation + + from interface_tester.interface_test import Tester + from interface_tester.schema_base import DataBagSchema + + def test_data_on_changed(): + t = Tester(State( + relations=[Relation( + endpoint='tracing', + interface='tracing', + remote_app_name='remote', + local_app_data={"foo":"1"}, + local_unit_data={"bar": "smackbeef"} + )] + )) + state_out = t.run("tracing-relation-changed") + t.assert_schema_valid() + """ + ), + schema_file=dedent( + """ +from interface_tester.interface_test import Tester +from interface_tester.schema_base import DataBagSchema, BaseModel + +class Foo(BaseModel): + foo:int=1 +class Bar(BaseModel): + bar:str + +class ProviderSchema(DataBagSchema): + unit: Bar + app: Foo +""" + ), + ) + + tester.run() + + +def test_default_schema_validation_failure(): + tester = _setup_with_test_file( + dedent( + """ + from scenario import State, Relation + + from interface_tester.interface_test import Tester + from interface_tester.schema_base import DataBagSchema + + def test_data_on_changed(): + t = Tester(State( + relations=[Relation( + endpoint='tracing', + interface='tracing', + remote_app_name='remote', + local_app_data={"foo":"abc"}, + local_unit_data={"bar": "smackbeef"} + )] + )) + state_out = t.run("tracing-relation-changed") + t.assert_schema_valid() + """ + ), + schema_file=dedent( + """ + from interface_tester.interface_test import Tester + from interface_tester.schema_base import DataBagSchema, BaseModel + + class Foo(BaseModel): + foo:int=1 + class Bar(BaseModel): + bar:str + + class ProviderSchema(DataBagSchema): + unit: Bar + app: Foo + """ + ), + ) + + with pytest.raises(SchemaValidationError): + tester.run() + + +def test_valid_run_custom_schema(): + tester = _setup_with_test_file( + dedent( + """ + from scenario import State, Relation + + from interface_tester.interface_test import Tester + from interface_tester.schema_base import DataBagSchema, BaseModel + + class Foo(BaseModel): + foo:int=1 + class Bar(BaseModel): + bar:str + + class FooBarSchema(DataBagSchema): + unit: Bar + app: Foo + + def test_data_on_changed(): + t = Tester(State( + relations=[Relation( + endpoint='tracing', + interface='tracing', + remote_app_name='remote', + local_app_data={"foo":"1"}, + local_unit_data={"bar": "smackbeef"} + )] + )) + state_out = t.run("tracing-relation-changed") + t.assert_schema_valid(schema=FooBarSchema) + """ + ) + ) + + tester.run() + + +def test_invalid_custom_schema(): + tester = _setup_with_test_file( + dedent( + """ + from scenario import State, Relation + + from interface_tester.interface_test import Tester + from interface_tester.schema_base import DataBagSchema, BaseModel + + class Foo(BaseModel): + foo:int=1 + class Bar(BaseModel): + bar:str + + class FooBarSchema(DataBagSchema): + unit: Bar + app: Foo + + def test_data_on_changed(): + t = Tester(State( + relations=[Relation( + endpoint='tracing', + interface='tracing', + remote_app_name='remote', + local_app_data={"foo":"abc"}, + local_unit_data={"bar": "smackbeef"} + )] + )) + state_out = t.run("tracing-relation-changed") + t.assert_schema_valid(schema=FooBarSchema) + """ + ) + ) + with pytest.raises(SchemaValidationError): + tester.run() diff --git a/tox.ini b/tox.ini index 63345f7..d82e28a 100644 --- a/tox.ini +++ b/tox.ini @@ -18,7 +18,7 @@ description = run unittests deps = .[unit_tests] commands = - pytest {[vars]tst_path}/unit/ + pytest -v --tb native {[vars]tst_path}/unit --log-cli-level=INFO -s {posargs} [testenv:lint] From e221ca4ae8beb71791f351f5e6ea352ecf288e8d Mon Sep 17 00:00:00 2001 From: Pietro Pasotti Date: Wed, 6 Sep 2023 14:03:22 +0200 Subject: [PATCH 05/10] vbump --- interface_tester/cli/discover.py | 8 +++----- pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/interface_tester/cli/discover.py b/interface_tester/cli/discover.py index d4685b2..26d9f29 100644 --- a/interface_tester/cli/discover.py +++ b/interface_tester/cli/discover.py @@ -1,9 +1,9 @@ from pathlib import Path +from typing import Callable import typer from interface_tester.collector import _CharmTestConfig, collect_tests -from interface_tester.interface_test import SchemaConfig, _InterfaceTestCase def pprint_tests( @@ -22,10 +22,8 @@ def _pprint_tests(path: Path = Path(), include="*"): tests = collect_tests(path=path, include=include) print("Discovered:") - def pprint_case(case: "_InterfaceTestCase"): - state = "yes" if case.input_state else "no" - schema_config = case.schema if isinstance(case.schema, SchemaConfig) else "custom" - print(f" - {case.name}:: {case.event} (state={state}, schema={schema_config})") + def pprint_case(case: Callable): + print(f" - {case.__name__}") for interface, versions in tests.items(): if not versions: diff --git a/pyproject.toml b/pyproject.toml index 31ebe11..8da2688 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ build-backend = "setuptools.build_meta" [project] name = "pytest-interface-tester" -version = "0.3.3" +version = "1.0" authors = [ { name = "Pietro Pasotti", email = "pietro.pasotti@canonical.com" }, ] From 2fe561f74dafcebbacda2c5ceb4c6a9fe76b0a01 Mon Sep 17 00:00:00 2001 From: Pietro Pasotti Date: Wed, 6 Sep 2023 14:11:02 +0200 Subject: [PATCH 06/10] lint --- interface_tester/__init__.py | 1 - interface_tester/interface_test.py | 7 ++++--- interface_tester/plugin.py | 2 +- .../resources/charm-like-path/tests/interface/conftest.py | 2 +- .../tracing/v42/interface_tests/test_provider.py | 6 +++--- .../tracing/v42/interface_tests/test_requirer.py | 6 +++--- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/interface_tester/__init__.py b/interface_tester/__init__.py index 71bae33..599293e 100644 --- a/interface_tester/__init__.py +++ b/interface_tester/__init__.py @@ -2,7 +2,6 @@ # See LICENSE file for licensing details. import pytest -from interface_tester.interface_test import Tester from interface_tester.plugin import InterfaceTester from interface_tester.schema_base import DataBagSchema # noqa: F401 diff --git a/interface_tester/interface_test.py b/interface_tester/interface_test.py index 6060d65..65356dc 100644 --- a/interface_tester/interface_test.py +++ b/interface_tester/interface_test.py @@ -8,7 +8,7 @@ import typing from contextlib import contextmanager from enum import Enum -from typing import Any, Callable, Dict, List, Literal, Optional, Union +from typing import Any, Callable, List, Literal, Optional, Union from pydantic import ValidationError from scenario import Context, Event, Relation, State @@ -108,7 +108,7 @@ def tester_context(ctx: _InterfaceTestContext): try: yield - except Exception as e: + except Exception: tester = Tester.__instance__ if tester: @@ -206,7 +206,8 @@ def assert_schema_valid(self, schema: "DataBagSchema" = None): if not databag_schema: raise NoSchemaError( self._test_id, - "No schema found. If this is expected, call Tester.skip_schema_validation() instead.", + "No schema found. If this is expected, " + "call Tester.skip_schema_validation() instead.", ) errors = [] diff --git a/interface_tester/plugin.py b/interface_tester/plugin.py index 2ca745d..11c3d4a 100644 --- a/interface_tester/plugin.py +++ b/interface_tester/plugin.py @@ -4,7 +4,7 @@ import tempfile from pathlib import Path from subprocess import PIPE, Popen -from typing import Any, Callable, Dict, Generator, List, Literal, Optional, Tuple, Type +from typing import Any, Callable, Dict, Generator, List, Optional, Tuple, Type from ops.testing import CharmType from scenario.state import Event, MetadataNotFoundError, State, _CharmSpec diff --git a/tests/resources/charm-like-path/tests/interface/conftest.py b/tests/resources/charm-like-path/tests/interface/conftest.py index b20117b..f3d950e 100644 --- a/tests/resources/charm-like-path/tests/interface/conftest.py +++ b/tests/resources/charm-like-path/tests/interface/conftest.py @@ -12,7 +12,7 @@ class CRILikePathTester(InterfaceTester): def _collect_interface_test_specs(self): - tests = gather_test_spec_for_version( + gather_test_spec_for_version( CRI_LIKE_PATH, interface_name=self._interface_name, version=self._interface_version, diff --git a/tests/resources/cri-like-path/interfaces/tracing/v42/interface_tests/test_provider.py b/tests/resources/cri-like-path/interfaces/tracing/v42/interface_tests/test_provider.py index 31e25d2..61d32a2 100644 --- a/tests/resources/cri-like-path/interfaces/tracing/v42/interface_tests/test_provider.py +++ b/tests/resources/cri-like-path/interfaces/tracing/v42/interface_tests/test_provider.py @@ -8,13 +8,13 @@ def test_no_data_on_created(): t = Tester(State()) - state_out = t.run(event="tracing-relation-created") + t.run(event="tracing-relation-created") t.assert_relation_data_empty() def test_no_data_on_joined(): t = Tester() - state_out = t.run(event="tracing-relation-joined") + t.run(event="tracing-relation-joined") t.assert_relation_data_empty() @@ -31,5 +31,5 @@ def test_data_on_changed(): ] ) ) - state_out = t.run("tracing-relation-changed") + t.run("tracing-relation-changed") t.assert_relation_data_empty() diff --git a/tests/resources/cri-like-path/interfaces/tracing/v42/interface_tests/test_requirer.py b/tests/resources/cri-like-path/interfaces/tracing/v42/interface_tests/test_requirer.py index 31e25d2..61d32a2 100644 --- a/tests/resources/cri-like-path/interfaces/tracing/v42/interface_tests/test_requirer.py +++ b/tests/resources/cri-like-path/interfaces/tracing/v42/interface_tests/test_requirer.py @@ -8,13 +8,13 @@ def test_no_data_on_created(): t = Tester(State()) - state_out = t.run(event="tracing-relation-created") + t.run(event="tracing-relation-created") t.assert_relation_data_empty() def test_no_data_on_joined(): t = Tester() - state_out = t.run(event="tracing-relation-joined") + t.run(event="tracing-relation-joined") t.assert_relation_data_empty() @@ -31,5 +31,5 @@ def test_data_on_changed(): ] ) ) - state_out = t.run("tracing-relation-changed") + t.run("tracing-relation-changed") t.assert_relation_data_empty() From 16c3cc5382eff57ee272faac88bb79aff255cdd0 Mon Sep 17 00:00:00 2001 From: Pietro Pasotti Date: Wed, 13 Sep 2023 10:10:35 +0200 Subject: [PATCH 07/10] tests for discover --- interface_tester/cli/discover.py | 12 +++--- .../interfaces/database/v1/charms.yaml | 5 +++ .../v1/interface_tests/test_provider.py | 35 ++++++++++++++++ .../v1/interface_tests/test_requirer.py | 35 ++++++++++++++++ .../interfaces/database/v1/schema.py | 14 +++++++ tests/unit/test_discover.py | 42 +++++++++++++++++++ 6 files changed, 138 insertions(+), 5 deletions(-) create mode 100644 tests/resources/cri-like-path/interfaces/database/v1/charms.yaml create mode 100644 tests/resources/cri-like-path/interfaces/database/v1/interface_tests/test_provider.py create mode 100644 tests/resources/cri-like-path/interfaces/database/v1/interface_tests/test_requirer.py create mode 100644 tests/resources/cri-like-path/interfaces/database/v1/schema.py create mode 100644 tests/unit/test_discover.py diff --git a/interface_tester/cli/discover.py b/interface_tester/cli/discover.py index 26d9f29..4ea0dc2 100644 --- a/interface_tester/cli/discover.py +++ b/interface_tester/cli/discover.py @@ -25,7 +25,8 @@ def _pprint_tests(path: Path = Path(), include="*"): def pprint_case(case: Callable): print(f" - {case.__name__}") - for interface, versions in tests.items(): + # sorted by interface first, version then + for interface, versions in sorted(tests.items()): if not versions: print(f"{interface}: ") print() @@ -33,19 +34,20 @@ def pprint_case(case: Callable): print(f"{interface}:") - for version, roles in versions.items(): + for version, roles in sorted(versions.items()): print(f" - {version}:") by_role = {role: roles[role] for role in {"requirer", "provider"}} - for role, test_spec in by_role.items(): + for role, test_spec in sorted(by_role.items()): print(f" - {role}:") tests = test_spec["tests"] schema = test_spec["schema"] - for test_cls in tests: + for test_cls in sorted(tests, key=lambda fn: fn.__name__): pprint_case(test_cls) + if not tests: print(" - ") @@ -59,7 +61,7 @@ def pprint_case(case: Callable): if charms: print(" - charms:") charm: _CharmTestConfig - for charm in charms: + for charm in sorted(charms): if isinstance(charm, str): print(" - ") continue diff --git a/tests/resources/cri-like-path/interfaces/database/v1/charms.yaml b/tests/resources/cri-like-path/interfaces/database/v1/charms.yaml new file mode 100644 index 0000000..335bd83 --- /dev/null +++ b/tests/resources/cri-like-path/interfaces/database/v1/charms.yaml @@ -0,0 +1,5 @@ +providers: + - name: foo-k8s + url: https://github.com/canonical/foo-k8s-operator + +requirers: [] \ No newline at end of file diff --git a/tests/resources/cri-like-path/interfaces/database/v1/interface_tests/test_provider.py b/tests/resources/cri-like-path/interfaces/database/v1/interface_tests/test_provider.py new file mode 100644 index 0000000..2ef0a9d --- /dev/null +++ b/tests/resources/cri-like-path/interfaces/database/v1/interface_tests/test_provider.py @@ -0,0 +1,35 @@ +# Copyright 2023 Canonical +# See LICENSE file for licensing details. + +from scenario import Relation, State + +from interface_tester.interface_test import Tester + + +def test_no_data_on_created(): + t = Tester(State()) + t.run(event="database-relation-created") + t.assert_relation_data_empty() + + +def test_no_data_on_joined(): + t = Tester() + t.run(event="database-relation-joined") + t.assert_relation_data_empty() + + +def test_data_on_changed(): + t = Tester( + State( + relations=[ + Relation( + endpoint="database", + interface="database", + remote_app_name="remote", + local_app_data={}, + ) + ] + ) + ) + t.run("database-relation-changed") + t.assert_relation_data_empty() diff --git a/tests/resources/cri-like-path/interfaces/database/v1/interface_tests/test_requirer.py b/tests/resources/cri-like-path/interfaces/database/v1/interface_tests/test_requirer.py new file mode 100644 index 0000000..2ef0a9d --- /dev/null +++ b/tests/resources/cri-like-path/interfaces/database/v1/interface_tests/test_requirer.py @@ -0,0 +1,35 @@ +# Copyright 2023 Canonical +# See LICENSE file for licensing details. + +from scenario import Relation, State + +from interface_tester.interface_test import Tester + + +def test_no_data_on_created(): + t = Tester(State()) + t.run(event="database-relation-created") + t.assert_relation_data_empty() + + +def test_no_data_on_joined(): + t = Tester() + t.run(event="database-relation-joined") + t.assert_relation_data_empty() + + +def test_data_on_changed(): + t = Tester( + State( + relations=[ + Relation( + endpoint="database", + interface="database", + remote_app_name="remote", + local_app_data={}, + ) + ] + ) + ) + t.run("database-relation-changed") + t.assert_relation_data_empty() diff --git a/tests/resources/cri-like-path/interfaces/database/v1/schema.py b/tests/resources/cri-like-path/interfaces/database/v1/schema.py new file mode 100644 index 0000000..bf655c2 --- /dev/null +++ b/tests/resources/cri-like-path/interfaces/database/v1/schema.py @@ -0,0 +1,14 @@ +from pydantic import BaseModel + +from interface_tester.schema_base import DataBagSchema + + +class DBRequirerData(BaseModel): + foo: str + bar: int + + +class RequirerSchema(DataBagSchema): + """Requirer schema for Tracing.""" + + app: DBRequirerData diff --git a/tests/unit/test_discover.py b/tests/unit/test_discover.py new file mode 100644 index 0000000..71b8103 --- /dev/null +++ b/tests/unit/test_discover.py @@ -0,0 +1,42 @@ +from interface_tester.cli.discover import pprint_tests +from tests.unit.utils import CRI_LIKE_PATH + + +def test_discover(capsys): + pprint_tests(CRI_LIKE_PATH, "*") + out = capsys.readouterr().out + assert out.strip() == """ +collecting tests for * from root = /home/pietro/canonical/pytest-interface-tester/tests/resources/cri-like-path +Discovered: +database: + - v1: + - provider: + - test_data_on_changed + - test_no_data_on_created + - test_no_data_on_joined + - schema NOT OK + - charms: + - foo-k8s (https://github.com/canonical/foo-k8s-operator) custom_test_setup=no + - requirer: + - test_data_on_changed + - test_no_data_on_created + - test_no_data_on_joined + - schema OK + - + +tracing: + - v42: + - provider: + - test_data_on_changed + - test_no_data_on_created + - test_no_data_on_joined + - schema NOT OK + - charms: + - tempo-k8s (https://github.com/canonical/tempo-k8s-operator) custom_test_setup=no + - requirer: + - test_data_on_changed + - test_no_data_on_created + - test_no_data_on_joined + - schema OK + - +""".strip() From d840670fadb32ea2492817e88c5d7a8db5a0d85f Mon Sep 17 00:00:00 2001 From: Pietro Pasotti Date: Wed, 13 Sep 2023 10:53:17 +0200 Subject: [PATCH 08/10] pr comments --- interface_tester/collector.py | 32 ++--- interface_tester/interface_test.py | 119 ++++++++++++++---- .../tests/interface/conftest.py | 2 +- tests/unit/test_discover.py | 9 +- 4 files changed, 120 insertions(+), 42 deletions(-) diff --git a/interface_tester/collector.py b/interface_tester/collector.py index caa1822..71089e0 100644 --- a/interface_tester/collector.py +++ b/interface_tester/collector.py @@ -124,13 +124,13 @@ def load_schema_module(schema_path: Path) -> types.ModuleType: def get_schemas(file: Path) -> Dict[Literal["requirer", "provider"], Type[DataBagSchema]]: """Load databag schemas from schema.py file.""" if not file.exists(): - logger.warning(f"File does not exist: {file}") + logger.warning("File does not exist: %s" % file) return {} try: module = load_schema_module(file) except ImportError as e: - logger.error(f"Failed to load module {file}: {e}") + logger.error("Failed to load module %s: %s" % (file, e)) return {} out = {} @@ -139,12 +139,12 @@ def get_schemas(file: Path) -> Dict[Literal["requirer", "provider"], Type[DataBa out[role] = get_schema_from_module(module, name) except NameError: logger.warning( - f"Failed to load {name} from {file}: " f"schema not defined for role: {role}." + "Failed to load %s from %s: schema not defined for role: %s." % (name, file, role) ) except TypeError as e: logger.error( - f"Found object called {name!r} in {file}; " - f"expecting a DataBagSchema subclass, not {e.args[0]!r}." + "Found object called %s in %s; expecting a DataBagSchema subclass, not %s." + % (name, file, e.args[0]) ) return out @@ -162,9 +162,9 @@ def _gather_charms_for_version(version_dir: Path) -> Optional[_CharmsDotYamlSpec try: charms = yaml.safe_load(charms_yaml.read_text()) except (json.JSONDecodeError, yaml.YAMLError) as e: - logger.error(f"failed to decode {charms_yaml}: " f"verify that it is valid: {e}") + logger.error("failed to decode %s: verify that it is valid yaml: %s" % (charms_yaml, e)) except FileNotFoundError as e: - logger.error(f"not found: {e}") + logger.error("not found: %s" % e) if not charms: return None @@ -187,8 +187,8 @@ def _gather_charms_for_version(version_dir: Path) -> Optional[_CharmsDotYamlSpec cfg = _CharmTestConfig(**item) except TypeError: logger.error( - f"failure parsing {item} to _CharmTestConfig; invalid charm test " - f"configuration in {version_dir}/charms.yaml:providers" + "failure parsing %s to _CharmTestConfig; invalid charm test " + "configuration in %s/charms.yaml:providers" % (item, version_dir) ) continue destination.append(cfg) @@ -197,7 +197,7 @@ def _gather_charms_for_version(version_dir: Path) -> Optional[_CharmsDotYamlSpec return spec -def _scrape_module_for_tests(module) -> List[Callable[[None], None]]: +def _scrape_module_for_tests(module: types.ModuleType) -> List[Callable[[None], None]]: tests = [] for name, obj in inspect.getmembers(module): if inspect.isfunction(obj): @@ -222,7 +222,7 @@ def _gather_test_cases_for_version(version_dir: Path, interface_name: str, versi try: module = importlib.import_module(module_name) except ImportError as e: - logger.error(f"Failed to load module {module_name}: {e}") + logger.error("Failed to load module %s: %s" % (module_name, e)) continue tests = _scrape_module_for_tests(module) @@ -233,7 +233,7 @@ def _gather_test_cases_for_version(version_dir: Path, interface_name: str, versi tgt.extend(tests) if not (requirer_test_cases or provider_test_cases): - logger.error(f"no valid test case files found in {interface_tests_dir}") + logger.error("no valid test case files found in %s" % interface_tests_dir) # remove from import search path sys.path.pop(-1) @@ -283,7 +283,9 @@ def _gather_tests_for_interface( try: version_n = int(version_dir.name[1:]) except TypeError: - logger.error(f"Unable to parse version {version_dir.name} as an integer. Skipping...") + logger.error( + "Unable to parse version %s as an integer. Skipping..." % version_dir.name + ) continue tests[version_dir.name] = gather_test_spec_for_version( version_dir, interface_name, version_n @@ -304,14 +306,14 @@ def collect_tests(path: Path, include: str = "*") -> Dict[str, Dict[str, Interfa - name: foo url: www.github.com/canonical/foo """ - logger.info(f"collecting tests from {path}:{include}") + logger.info("collecting tests from %s: %s" % (path, include)) tests = {} for interface_dir in (path / "interfaces").glob(include): interface_dir_name = interface_dir.name if interface_dir_name.startswith("__"): # ignore __template__ and python-dirs continue # skip - logger.info(f"collecting tests for interface {interface_dir_name}") + logger.info("collecting tests for interface %s" % interface_dir_name) interface_name = interface_dir_name.replace("-", "_") tests[interface_name] = _gather_tests_for_interface(interface_dir, interface_name) diff --git a/interface_tester/interface_test.py b/interface_tester/interface_test.py index 65356dc..01be7f2 100644 --- a/interface_tester/interface_test.py +++ b/interface_tester/interface_test.py @@ -10,6 +10,7 @@ from enum import Enum from typing import Any, Callable, List, Literal, Optional, Union +from ops.testing import CharmType from pydantic import ValidationError from scenario import Context, Event, Relation, State @@ -47,7 +48,7 @@ class _InterfaceTestContext: """The version of the interface that this test is about.""" role: Role - charm_type: type + charm_type: CharmType """Charm class being tested""" supported_endpoints: dict """Supported relation endpoints.""" @@ -135,7 +136,7 @@ def tester_context(ctx: _InterfaceTestContext): class InvalidTesterRunError(RuntimeError): """Raised if Tester is being used incorrectly.""" - def __init__(self, test_name, msg): + def __init__(self, test_name: str, msg: str): _msg = f"failed running {test_name}: invalid test. {msg}" super().__init__(_msg) @@ -151,11 +152,51 @@ class NoSchemaError(InvalidTesterRunError): class Tester: __instance__ = None - def __init__(self, state_in: State = None, name: str = None): - """Initializer. - - :param state_in: the input state for this scenario test. Will default to the empty State(). - :param name: the name of the test. Will default to the function's identifier. + def __init__(self, state_in: Optional[State] = None, name: Optional[str] = None): + """Core interface test specification tool. + + This class is essential to defining an interface test to be used in the + ``charm-relation-interfaces`` repository. In order to define a valid interface + test you will need to: + + a) Initialize this class in the scope of an interface test to specify the scenario's + initial state. Then b) call its ``run`` method to execute scenario, and finally + c) validate the schema. + + Failing to take any of these three steps will result in an invalid test. + If an error is raised during execution of these steps, or manually from elsewhere in the + test scope, the test will fail. + + usage: + >>> def test_foo_relation_joined(): + >>> t = Tester(state_in=State()) # specify the initial state + >>> state_out = t.run('foo-relation-joined') # run Scenario and get the output state + >>> t.assert_schema_valid() # check that the schema is valid + + You can run assertions on ``state_out``, if the interface specification makes + claims on its contents. + + Alternatively to calling ``assert_schema_valid``, you can: + 1) define your own schema subclassing ``DataBagSchema`` + >>> from interface_tester.schema_base import DataBagSchema, BaseModel + >>> class CustomAppModel(BaseModel): + >>> foo: int + >>> bar: str + >>> + >>> class SomeCustomSchema(DataBagSchema): + >>> app: CustomAppModel + And then pass it to assert_schema_valid to override the default schema + (the one defined in ``schema.py``). + >>> t.assert_schema_valid(SomeCustomSchema) + 2) check that all local databags are empty (same as ``assert_schema_valid(DataBagSchema)``) + >>> t.assert_relation_data_empty() + 3) skip schema validation altogether if you know better + >>> t.skip_schema_validation() + + :param state_in: the input state for this scenario test. + Will default to the empty ``State()``. + :param name: the name of the test. Will default to the function's + identifier (``__name__``). """ # todo: pythonify if Tester.__instance__: @@ -163,7 +204,7 @@ def __init__(self, state_in: State = None, name: str = None): Tester.__instance__ = self if not self.ctx: - raise RuntimeError("Tester can only be initialized inside a tester context.") + raise RuntimeError("Tester can only be initialized inside an interface test context.") self._state_template = None self._state_in = state_in or State() @@ -174,15 +215,33 @@ def __init__(self, state_in: State = None, name: str = None): self._has_checked_schema = False @property - def _test_id(self): + def _test_id(self) -> str: + """A name for this test, as descriptive and unique as possible.""" return f"{self.ctx.interface_name}[{self.ctx.version}]/{self.ctx.role}:{self._test_name}" @property - def ctx(self): + def ctx(self) -> Optional[_InterfaceTestContext]: + """The test context, defined by the test caller. + + It exposes information about the charm that is using this test. + You probably won't need to call this from inside the test definition. + + When called from an interface test scope, is guaranteed(^tm) to return + ``_InterfaceTestContext``. + """ return _TESTER_CTX - def run(self, event: Union[str, Event]): - assert self.ctx, "tester cannot run: no _TESTER_CTX set" + def run(self, event: Union[str, Event]) -> "State": + """Simulate the emission on an event in the initial state you passed to the initializer. + + Calling this method will run scenario and verify that the charm being tested can handle + the ``event`` without raising exceptions. + + It returns the output state resulting from this execution, should you want to + write assertions against it. + """ + if not self.ctx: + raise InvalidTesterRunError("tester cannot run: no _TESTER_CTX set") state_out = self._run(event) self._state_out = state_out @@ -190,9 +249,16 @@ def run(self, event: Union[str, Event]): @property def _relations(self) -> List[Relation]: + """The relations that this test is about.""" return [r for r in self._state_out.relations if r.interface == self.ctx.interface_name] - def assert_schema_valid(self, schema: "DataBagSchema" = None): + def assert_schema_valid(self, schema: Optional["DataBagSchema"] = None): + """Check that the local databags of the relations being tested satisfy the default schema. + + Default schema is defined in this-interface/vX/schema.py. + Override the schema being checked against by passing your own DataBagSchema subclass. + """ + self._has_checked_schema = True if not self._has_run: raise InvalidTesterRunError(self._test_id, "call Tester.run() first") @@ -229,6 +295,7 @@ def _check_has_run(self): raise InvalidTesterRunError(self._test_id, "Call Tester.run() first.") def assert_relation_data_empty(self): + """Assert that all local databags are empty for the relations being tested.""" self._check_has_run() for relation in self._relations: if relation.local_app_data: @@ -242,11 +309,16 @@ def assert_relation_data_empty(self): self._has_checked_schema = True def skip_schema_validation(self): + """Skip schema validation for this test run. + + Only use if you really have to. + """ self._check_has_run() logger.debug("skipping schema validation") self._has_checked_schema = True def _finalize(self): + """Verify that .run() has been called, as well as some schema validation method.""" if not self._has_run: raise InvalidTesterRunError( self._test_id, "Test function must call Tester.run() before returning." @@ -257,7 +329,7 @@ def _finalize(self): "Test function must call " "Tester.skip_schema_validation(), or " "Tester.assert_schema_valid(), or " - "Tester.assert_schema_empty() before returning.", + "Tester.assert_relation_data_empty() before returning.", ) self._detach() @@ -266,7 +338,7 @@ def _detach(self): Tester.__instance__ = None def _run(self, event: Union[str, Event]): - logger.debug(f"running {event}") + logger.debug("running %s" % event) self._has_run = True # this is the input state as specified by the interface tests writer. It can @@ -292,11 +364,11 @@ def _run(self, event: Union[str, Event]): # test.EVENT might be a string or an Event. Cast to Event. evt: Event = self._coerce_event(event, relation) - logger.info(f"collected test for {self.ctx.interface_name} with {evt.name}") + logger.info("collected test for %s with %s" % (self.ctx.interface_name, evt.name)) return self._run_scenario(evt, modified_state) def _run_scenario(self, event: Event, state: State): - logger.debug(f"running scenario with state={state}, event={event}") + logger.debug("running scenario with state=%s, event=%s" % (state, event)) ctx = Context( self.ctx.charm_type, @@ -359,9 +431,9 @@ def _generate_relations_state( for rel in state_template.relations: if rel.interface == interface_name: logger.warning( - f"relation with interface name = {interface_name} found in state template. " - f"This will be overwritten by the relation spec provided by the relation " - f"interface test case." + "relation with interface name =%s found in state template. " + "This will be overwritten by the relation spec provided by the relation " + "interface test case." % interface_name ) def filter_relations(rels: List[Relation], op: Callable): @@ -378,8 +450,8 @@ def filter_relations(rels: List[Relation], op: Callable): if ignored := filter_relations(input_state.relations, op=operator.eq): logger.warning( - f"irrelevant relations specified in input state for {interface_name}/{role}." - f"These will be ignored. details: {ignored}" + "irrelevant relations specified in input state for %s/%s." + "These will be ignored. details: %s" % (interface_name, role, ignored) ) # if we still don't have any relation matching the interface we're testing, we generate @@ -407,6 +479,7 @@ def filter_relations(rels: List[Relation], op: Callable): ) ) logger.debug( - f"{self}: merged {input_state} and {state_template} --> relations={relations}" + "%s: merged %s and %s --> relations=%s" + % (self, input_state, state_template, relations) ) return relations diff --git a/tests/resources/charm-like-path/tests/interface/conftest.py b/tests/resources/charm-like-path/tests/interface/conftest.py index f3d950e..7026569 100644 --- a/tests/resources/charm-like-path/tests/interface/conftest.py +++ b/tests/resources/charm-like-path/tests/interface/conftest.py @@ -1,4 +1,4 @@ -# Copyright 2022 Canonical Ltd. +# Copyright 2023 Canonical Ltd. # See LICENSE file for licensing details. import pytest diff --git a/tests/unit/test_discover.py b/tests/unit/test_discover.py index 71b8103..c3601c0 100644 --- a/tests/unit/test_discover.py +++ b/tests/unit/test_discover.py @@ -1,12 +1,14 @@ from interface_tester.cli.discover import pprint_tests -from tests.unit.utils import CRI_LIKE_PATH +from utils import CRI_LIKE_PATH def test_discover(capsys): pprint_tests(CRI_LIKE_PATH, "*") out = capsys.readouterr().out - assert out.strip() == """ -collecting tests for * from root = /home/pietro/canonical/pytest-interface-tester/tests/resources/cri-like-path + assert ( + out.strip() + == f""" +collecting tests for * from root = {CRI_LIKE_PATH} Discovered: database: - v1: @@ -40,3 +42,4 @@ def test_discover(capsys): - schema OK - """.strip() + ) From 5135cfecfd3f8eef9323aa2f469afae88d89bb41 Mon Sep 17 00:00:00 2001 From: Pietro Pasotti Date: Wed, 13 Sep 2023 10:58:16 +0200 Subject: [PATCH 09/10] pin linter versions --- tests/unit/test_discover.py | 3 ++- tox.ini | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/unit/test_discover.py b/tests/unit/test_discover.py index c3601c0..27788ef 100644 --- a/tests/unit/test_discover.py +++ b/tests/unit/test_discover.py @@ -1,6 +1,7 @@ -from interface_tester.cli.discover import pprint_tests from utils import CRI_LIKE_PATH +from interface_tester.cli.discover import pprint_tests + def test_discover(capsys): pprint_tests(CRI_LIKE_PATH, "*") diff --git a/tox.ini b/tox.ini index d82e28a..b256842 100644 --- a/tox.ini +++ b/tox.ini @@ -25,9 +25,9 @@ commands = skip_install = True description = run linters (check only) deps = - black - isort - ruff + black==23.9.1 + isort==5.12.0 + ruff==0.0.289 commands = black --check {[vars]all_path} isort --profile black --check-only {[vars]all_path} @@ -38,8 +38,8 @@ commands = skip_install=True description = run formatters deps = - black - isort + black==23.9.1 + isort==5.12.0 commands = black {[vars]all_path} isort --profile black {[vars]all_path} From 54397b89ef77ed01cb915a6c7b6e70698100a184 Mon Sep 17 00:00:00 2001 From: Pietro Pasotti Date: Thu, 14 Sep 2023 13:26:25 +0200 Subject: [PATCH 10/10] dequoted state --- interface_tester/interface_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface_tester/interface_test.py b/interface_tester/interface_test.py index 01be7f2..e4ebd77 100644 --- a/interface_tester/interface_test.py +++ b/interface_tester/interface_test.py @@ -231,7 +231,7 @@ def ctx(self) -> Optional[_InterfaceTestContext]: """ return _TESTER_CTX - def run(self, event: Union[str, Event]) -> "State": + def run(self, event: Union[str, Event]) -> State: """Simulate the emission on an event in the initial state you passed to the initializer. Calling this method will run scenario and verify that the charm being tested can handle