diff --git a/.travis.yml b/.travis.yml index cbd38224..3955bb49 100644 --- a/.travis.yml +++ b/.travis.yml @@ -102,6 +102,9 @@ jobs: - python: 3.7 env: TOXENV=py37-advanced TOXCFG=tox-integration.ini stage: full-tests + - python: 3.7 + env: TOXENV=py37-hooks TOXCFG=tox-integration.ini + stage: full-tests - stage: deploy script: skip deploy: diff --git a/docs/source/basics.md b/docs/source/basics.md index 8f7033a5..4e8c1ca6 100644 --- a/docs/source/basics.md +++ b/docs/source/basics.md @@ -1424,7 +1424,7 @@ values. Say we want to convert a value from an included file to an integer: ```yaml request: json: - an_integer: !!int "{my_integer:d}" # Error + # an_integer: !!int "{my_integer:d}" # Error an_integer: !int "{my_integer:d}" # Works ``` @@ -1939,3 +1939,62 @@ There are some limitations on fixtures: - Fixtures should be 'function' or 'session' scoped. 'module' scoped fixtures will raise an error and 'class' scoped fixtures may not behave as you expect. - Parametrizing fixtures does not work - this is a limitation in Pytest. + + +## Hooks + +As well as fixtures as mentioned in the previous section, since version 0.28.0 +there is a couple of hooks which can be used to extract more information from +tests. + +These hooks are used by defining a function with the name of the hook in your +`conftest.py` that take the same arguments _with the same names_ - these hooks +will then be picked up at runtime and called appropriately. + +**NOTE**: These hooks should be considered a 'beta' feature, they are ready to +use but the names and arguments they take should be considered unstable and may +change in a future release (and more may also be added). + +### Before every test run + +This hook is called after fixtures, global configuration, and plugins have been +loaded, but _before_ formatting is done on the test and the schema of the test +is checked. This can be used to 'inject' extra things into the test before it is +run, such as configurations blocks for a plugin, or just for some kind of +logging. + +Args: +- test_dict (dict): Test to run +- variables (dict): Available variables + +Example usage: + +```python +import logging + +def pytest_tavern_beta_before_every_test_run(test_dict, variables): + logging.info("Starting test %s", test_dict["test_name"]) + + variables["extra_var"] = "abc123" +``` + +### After every response + +This hook is called after every _response_ for each _stage_ - this includes HTTP +responses, but also MQTT responses if you are using MQTT. This means if you are +using MQTT it might be called multiple times for each stage! + +Args: +- response (object): Response object. Could be whatever kind of response object + is returned by whatever plugin is used - if using the default HTTP + implementation which uses Requests, this will be a `requests.Response` object. + If using MQTT, this will be a paho-mqtt `Message` object. +- expected (dict): Formatted response block from the stage. + +Example usage: + +```python +def pytest_tavern_beta_after_every_response(expected, response): + with open("logfile.txt", "a") as logfile: + logfile.write("Got response: {}".format(response.json())) +``` diff --git a/example/hooks/Dockerfile b/example/hooks/Dockerfile new file mode 100644 index 00000000..57a81fcb --- /dev/null +++ b/example/hooks/Dockerfile @@ -0,0 +1,9 @@ +FROM python:3.5-alpine + +RUN pip install flask pyjwt + +COPY server.py / + +ENV FLASK_APP=/server.py + +CMD ["flask", "run", "--host=0.0.0.0"] diff --git a/example/hooks/conftest.py b/example/hooks/conftest.py new file mode 100644 index 00000000..76164c7b --- /dev/null +++ b/example/hooks/conftest.py @@ -0,0 +1,42 @@ +import os +import tempfile +import logging + +import pytest + +logger = logging.getLogger(__name__) + +name = None + + +@pytest.fixture(autouse=True) +def setup_logging(): + logging.basicConfig(level=logging.INFO) + + +def pytest_tavern_beta_after_every_response(expected, response): + global name + logging.debug(expected) + logging.debug(response) + with open(name, "a") as tfile: + tfile.write("abc\n") + + +@pytest.fixture(autouse=True) +def after_check_result(): + """Create a temporary file for the duration of the test, and make sure the above hook was called""" + global name + with tempfile.NamedTemporaryFile(delete=False) as tfile: + try: + tfile.close() + + name = tfile.name + yield + + with open(tfile.name, "r") as opened: + contents = opened.readlines() + assert len(contents) == 2 + assert all(i.strip() == "abc" for i in contents) + + finally: + os.remove(tfile.name) diff --git a/example/hooks/docker-compose.yaml b/example/hooks/docker-compose.yaml new file mode 100644 index 00000000..3d636985 --- /dev/null +++ b/example/hooks/docker-compose.yaml @@ -0,0 +1,10 @@ +--- +version: '2' + +services: + server: + build: + context: . + dockerfile: Dockerfile + ports: + - "5000:5000" diff --git a/example/hooks/server.py b/example/hooks/server.py new file mode 100644 index 00000000..9f965f9b --- /dev/null +++ b/example/hooks/server.py @@ -0,0 +1,21 @@ +from flask import Flask, jsonify, request + + +app = Flask(__name__) + + +@app.route("/double", methods=["POST"]) +def double_number(): + r = request.get_json() + + try: + number = r["number"] + except (KeyError, TypeError): + return jsonify({"error": "no number passed"}), 400 + + try: + double = int(number) * 2 + except ValueError: + return jsonify({"error": "a number was not passed"}), 400 + + return jsonify({"double": double}), 200 diff --git a/example/hooks/test_server.tavern.yaml b/example/hooks/test_server.tavern.yaml new file mode 100644 index 00000000..24366099 --- /dev/null +++ b/example/hooks/test_server.tavern.yaml @@ -0,0 +1,30 @@ +--- + +test_name: Make sure server doubles number properly + +stages: + - name: Make sure number is returned correctly + request: + url: http://localhost:5000/double + json: + number: 5 + method: POST + headers: + content-type: application/json + response: + status_code: 200 + body: + double: 10 + + - name: Make sure number is returned correctly again + request: + url: http://localhost:5000/double + json: + number: 10 + method: POST + headers: + content-type: application/json + response: + status_code: 200 + body: + double: 20 diff --git a/tavern/_plugins/mqtt/response.py b/tavern/_plugins/mqtt/response.py index 0f233a7c..e61f1805 100644 --- a/tavern/_plugins/mqtt/response.py +++ b/tavern/_plugins/mqtt/response.py @@ -6,6 +6,7 @@ from tavern.response.base import BaseResponse from tavern.util.dict_util import check_keys_match_recursive from tavern.util.loader import ANYTHING +from tavern.testutils.pytesthook.newhooks import call_hook try: LoadException = json.decoder.JSONDecodeError @@ -18,10 +19,9 @@ class MQTTResponse(BaseResponse): def __init__(self, client, name, expected, test_block_config): - # pylint: disable=unused-argument - super(MQTTResponse, self).__init__() + self.test_block_config = test_block_config self.name = name self._check_for_validate_functions(expected.get("payload", {})) @@ -78,6 +78,13 @@ def _await_response(self): # timed out break + call_hook( + self.test_block_config, + "pytest_tavern_beta_after_every_response", + expected=self.expected, + response=msg, + ) + self.received_messages.append(msg) msg.payload = msg.payload.decode("utf8") diff --git a/tavern/_plugins/rest/response.py b/tavern/_plugins/rest/response.py index 3f854d09..8f3759ff 100644 --- a/tavern/_plugins/rest/response.py +++ b/tavern/_plugins/rest/response.py @@ -9,6 +9,7 @@ from requests.status_codes import _codes +from tavern.testutils.pytesthook.newhooks import call_hook from tavern.schemas.extensions import get_wrapped_response_function from tavern.util.dict_util import recurse_access_key, deep_dict_merge from tavern.util import exceptions @@ -149,6 +150,13 @@ def verify(self, response): """ self._verbose_log_response(response) + call_hook( + self.test_block_config, + "pytest_tavern_beta_after_every_response", + expected=self.expected, + response=response, + ) + self.response = response self.status_code = response.status_code diff --git a/tavern/testutils/pytesthook/__init__.py b/tavern/testutils/pytesthook/__init__.py index 5028b0b5..fc3c6921 100644 --- a/tavern/testutils/pytesthook/__init__.py +++ b/tavern/testutils/pytesthook/__init__.py @@ -1,4 +1,9 @@ -from .hooks import pytest_collect_file, pytest_addoption +from .hooks import pytest_collect_file, pytest_addoption, pytest_addhooks from .util import add_parser_options -__all__ = ["pytest_addoption", "pytest_collect_file", "add_parser_options"] +__all__ = [ + "pytest_addoption", + "pytest_collect_file", + "pytest_addhooks", + "add_parser_options", +] diff --git a/tavern/testutils/pytesthook/hooks.py b/tavern/testutils/pytesthook/hooks.py index 4d991dd1..d45f477c 100644 --- a/tavern/testutils/pytesthook/hooks.py +++ b/tavern/testutils/pytesthook/hooks.py @@ -54,3 +54,10 @@ def pytest_collect_file(parent, path): return YamlFile(path, parent) return None + + +def pytest_addhooks(pluginmanager): + """Add our custom tavern hooks""" + from . import newhooks + + pluginmanager.add_hookspecs(newhooks) diff --git a/tavern/testutils/pytesthook/item.py b/tavern/testutils/pytesthook/item.py index b6f6d376..4cc61ead 100644 --- a/tavern/testutils/pytesthook/item.py +++ b/tavern/testutils/pytesthook/item.py @@ -10,6 +10,7 @@ from tavern.plugins import load_plugins from tavern.schemas.files import verify_tests from tavern.util import exceptions +from tavern.testutils.pytesthook.newhooks import call_hook from .error import ReprdError from .util import load_global_cfg @@ -139,6 +140,8 @@ def runtest(self): load_plugins(self.global_cfg) + self.global_cfg["tavern_internal"] = {"pytest_hook_caller": self.config.hook} + # INTERNAL # NOTE - now that we can 'mark' tests, we could use pytest.mark.xfail # instead. This doesn't differentiate between an error in verification @@ -146,11 +149,18 @@ def runtest(self): xfail = self.spec.get("_xfail", False) try: - verify_tests(self.spec) - fixture_values = self._load_fixture_values() self.global_cfg["variables"].update(fixture_values) + call_hook( + self.global_cfg, + "pytest_tavern_beta_before_every_test_run", + test_dict=self.spec, + variables=self.global_cfg["variables"], + ) + + verify_tests(self.spec) + run_test(self.path, self.spec, self.global_cfg) except exceptions.BadSchemaError: if xfail == "verify": diff --git a/tavern/testutils/pytesthook/newhooks.py b/tavern/testutils/pytesthook/newhooks.py new file mode 100644 index 00000000..078e0263 --- /dev/null +++ b/tavern/testutils/pytesthook/newhooks.py @@ -0,0 +1,51 @@ +# pylint: disable=unused-argument +import logging + +logger = logging.getLogger(__name__) + + +def pytest_tavern_beta_before_every_test_run(test_dict, variables): + """Called: + + - directly after fixtures are loaded for a test + - directly before verifying the schema of the file + - Before formatting is done on values + - After global configuration has been loaded + - After plugins have been loaded + + Modify the test in-place if you want to do something to it. + + Args: + test_dict (dict): Test to run + variables (dict): Available variables + """ + + +def pytest_tavern_beta_after_every_response(expected, response): + """Called after every _response_ - including MQTT/HTTP/etc + + Note: + - The response object type and the expected dict depends on what plugin you're using, and which kind of response it is! + - MQTT responses will call this hook multiple times if multiple messages are received + + Args: + response (object): Response object. + expected (dict): Response block in stage + """ + + +def call_hook(test_block_config, hookname, **kwargs): + """Utility to call the hooks""" + try: + hook = getattr( + test_block_config["tavern_internal"]["pytest_hook_caller"], hookname + ) + except AttributeError: + logger.critical("Error getting tavern hook!") + raise + + try: + hook(**kwargs) + except AttributeError: + logger.error("Unexpected error calling tavern hook") + raise diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 3a6a1a50..6c0d7bd8 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -1,3 +1,4 @@ +from mock import Mock import pytest @@ -13,6 +14,7 @@ def fix_example_includes(): }, "backends": {"mqtt": "paho-mqtt", "http": "requests"}, "strict": True, + "tavern_internal": {"pytest_hook_caller": Mock()}, } return includes.copy() diff --git a/tests/unit/response/test_mqtt_response.py b/tests/unit/response/test_mqtt_response.py index 630359d8..4e7bd4b1 100644 --- a/tests/unit/response/test_mqtt_response.py +++ b/tests/unit/response/test_mqtt_response.py @@ -27,7 +27,7 @@ def __init__(self, returned): class TestResponse(object): - def _get_fake_verifier(self, expected, fake_messages): + def _get_fake_verifier(self, expected, fake_messages, includes): """Given a list of messages, return a mocked version of the MQTT response verifier which will take messages off the front of this list as if they were published @@ -51,16 +51,16 @@ def inner(timeout): fake_client = Mock(spec=MQTTClient, message_received=yield_all_messages()) - return MQTTResponse(fake_client, "Test stage", expected, {}) + return MQTTResponse(fake_client, "Test stage", expected, includes) - def test_message_on_same_topic_fails(self): + def test_message_on_same_topic_fails(self, includes): """Correct topic, wrong message""" expected = {"topic": "/a/b/c", "payload": "hello"} fake_message = FakeMessage({"topic": "/a/b/c", "payload": "goodbye"}) - verifier = self._get_fake_verifier(expected, [fake_message]) + verifier = self._get_fake_verifier(expected, [fake_message], includes) with pytest.raises(exceptions.TestFailError): verifier.verify(expected) @@ -68,21 +68,21 @@ def test_message_on_same_topic_fails(self): assert len(verifier.received_messages) == 1 assert verifier.received_messages[0].topic == fake_message.topic - def test_correct_message(self): + def test_correct_message(self, includes): """Both correct matches""" expected = {"topic": "/a/b/c", "payload": "hello"} fake_message = FakeMessage(expected) - verifier = self._get_fake_verifier(expected, [fake_message]) + verifier = self._get_fake_verifier(expected, [fake_message], includes) verifier.verify(expected) assert len(verifier.received_messages) == 1 assert verifier.received_messages[0].topic == fake_message.topic - def test_correct_message_eventually(self): + def test_correct_message_eventually(self, includes): """One wrong messge, then the correct one""" expected = {"topic": "/a/b/c", "payload": "hello"} @@ -91,7 +91,7 @@ def test_correct_message_eventually(self): fake_message_bad = FakeMessage({"topic": "/a/b/c", "payload": "goodbye"}) verifier = self._get_fake_verifier( - expected, [fake_message_bad, fake_message_good] + expected, [fake_message_bad, fake_message_good], includes ) verifier.verify(expected) diff --git a/tox-integration.ini b/tox-integration.ini index 44e8ba58..94e60fe4 100644 --- a/tox-integration.ini +++ b/tox-integration.ini @@ -1,5 +1,5 @@ [tox] -envlist = {py27,py34,py35,py36,py37,pypy,pypy3}-{generic,cookies,mqtt,advanced,components,noextra} +envlist = {py27,py34,py35,py36,py37,pypy,pypy3}-{generic,cookies,mqtt,advanced,components,noextra,hooks} skip_missing_interpreters = true [testenv] @@ -13,6 +13,7 @@ changedir = cookies: example/cookies advanced: example/advanced components: example/components + hooks: example/hooks generic: tests/integration noextra: tests/integration deps = @@ -24,10 +25,10 @@ deps = paho-mqtt mqtt: fluent-logger commands = - docker-compose stop +; docker-compose stop docker-compose build docker-compose up -d - python -m pytest --tb=short --tavern-global-cfg={toxinidir}/tests/integration/global_cfg.yaml + python -m pytest --tb=short --tavern-global-cfg={toxinidir}/tests/integration/global_cfg.yaml --tavern-beta-new-traceback cookies: tavern-ci --stdout test_server.tavern.yaml cookies: python -c "from tavern.core import run; exit(run('test_server.tavern.yaml', pytest_args=[]))"