Add hook which runs before every test

.travis.yml

@@ -102,6 +102,9 @@ jobs:
     - python: 3.7
       env: TOXENV=py37-advanced TOXCFG=tox-integration.ini
       stage: full-tests
+    - python: 3.7
+      env: TOXENV=py37-hooks TOXCFG=tox-integration.ini
+      stage: full-tests
     - stage: deploy
       script: skip
       deploy:

docs/source/basics.md

@@ -1424,7 +1424,7 @@ values. Say we want to convert a value from an included file to an integer:
 ```yaml
 request:
   json:
-    an_integer: !!int "{my_integer:d}" # Error
+    # an_integer: !!int "{my_integer:d}" # Error
     an_integer: !int "{my_integer:d}" # Works
 ```
 
@@ -1939,3 +1939,62 @@ There are some limitations on fixtures:
 - Fixtures should be 'function' or 'session' scoped. 'module' scoped fixtures
   will raise an error and 'class' scoped fixtures may not behave as you expect.
 - Parametrizing fixtures does not work - this is a limitation in Pytest.
+
+
+## Hooks
+
+As well as fixtures as mentioned in the previous section, since version 0.28.0
+there is a couple of hooks which can be used to extract more information from
+tests.
+
+These hooks are used by defining a function with the name of the hook in your
+`conftest.py` that take the same arguments _with the same names_ - these hooks
+will then be picked up at runtime and called appropriately.
+
+**NOTE**: These hooks should be considered a 'beta' feature, they are ready to
+use but the names and arguments they take should be considered unstable and may
+change in a future release (and more may also be added).
+
+### Before every test run
+
+This hook is called after fixtures, global configuration, and plugins have been
+loaded, but _before_ formatting is done on the test and the schema of the test
+is checked. This can be used to 'inject' extra things into the test before it is
+run, such as configurations blocks for a plugin, or just for some kind of
+logging.
+
+Args: 
+- test_dict (dict): Test to run
+- variables (dict): Available variables
+
+Example usage:
+
+```python
+import logging
+
+def pytest_tavern_beta_before_every_test_run(test_dict, variables):
+    logging.info("Starting test %s", test_dict["test_name"])
+
+    variables["extra_var"] = "abc123"
+```
+
+### After every response
+
+This hook is called after every _response_ for each _stage_ - this includes HTTP
+responses, but also MQTT responses if you are using MQTT. This means if you are
+using MQTT it might be called multiple times for each stage!
+
+Args: 
+- response (object): Response object. Could be whatever kind of response object
+  is returned by whatever plugin is used - if using the default HTTP
+  implementation which uses Requests, this will be a `requests.Response` object.
+  If using MQTT, this will be a paho-mqtt `Message` object.
+- expected (dict): Formatted response block from the stage.
+
+Example usage:
+
+```python
+def pytest_tavern_beta_after_every_response(expected, response):
+    with open("logfile.txt", "a") as logfile:
+        logfile.write("Got response: {}".format(response.json()))
+```

example/hooks/Dockerfile

@@ -0,0 +1,9 @@
+FROM python:3.5-alpine
+
+RUN pip install flask pyjwt
+
+COPY server.py /
+
+ENV FLASK_APP=/server.py
+
+CMD ["flask", "run", "--host=0.0.0.0"]

example/hooks/conftest.py

@@ -0,0 +1,42 @@
+import os
+import tempfile
+import logging
+
+import pytest
+
+logger = logging.getLogger(__name__)
+
+name = None
+
+
+@pytest.fixture(autouse=True)
+def setup_logging():
+    logging.basicConfig(level=logging.INFO)
+
+
+def pytest_tavern_beta_after_every_response(expected, response):
+    global name
+    logging.debug(expected)
+    logging.debug(response)
+    with open(name, "a") as tfile:
+        tfile.write("abc\n")
+
+
+@pytest.fixture(autouse=True)
+def after_check_result():
+    """Create a temporary file for the duration of the test, and make sure the above hook was called"""
+    global name
+    with tempfile.NamedTemporaryFile(delete=False) as tfile:
+        try:
+            tfile.close()
+
+            name = tfile.name
+            yield
+
+            with open(tfile.name, "r") as opened:
+                contents = opened.readlines()
+                assert len(contents) == 2
+                assert all(i.strip() == "abc" for i in contents)
+
+        finally:
+            os.remove(tfile.name)

example/hooks/docker-compose.yaml

@@ -0,0 +1,10 @@
+---
+version: '2'
+
+services:
+  server:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    ports:
+      - "5000:5000"

example/hooks/server.py

@@ -0,0 +1,21 @@
+from flask import Flask, jsonify, request
+
+
+app = Flask(__name__)
+
+
+@app.route("/double", methods=["POST"])
+def double_number():
+    r = request.get_json()
+
+    try:
+        number = r["number"]
+    except (KeyError, TypeError):
+        return jsonify({"error": "no number passed"}), 400
+
+    try:
+        double = int(number) * 2
+    except ValueError:
+        return jsonify({"error": "a number was not passed"}), 400
+
+    return jsonify({"double": double}), 200

example/hooks/test_server.tavern.yaml

@@ -0,0 +1,30 @@
+---
+
+test_name: Make sure server doubles number properly
+
+stages:
+  - name: Make sure number is returned correctly
+    request:
+      url: http://localhost:5000/double
+      json:
+        number: 5
+      method: POST
+      headers:
+        content-type: application/json
+    response:
+      status_code: 200
+      body:
+        double: 10
+
+  - name: Make sure number is returned correctly again
+    request:
+      url: http://localhost:5000/double
+      json:
+        number: 10
+      method: POST
+      headers:
+        content-type: application/json
+    response:
+      status_code: 200
+      body:
+        double: 20

tavern/_plugins/mqtt/response.py

@@ -6,6 +6,7 @@
 from tavern.response.base import BaseResponse
 from tavern.util.dict_util import check_keys_match_recursive
 from tavern.util.loader import ANYTHING
+from tavern.testutils.pytesthook.newhooks import call_hook
 
 try:
     LoadException = json.decoder.JSONDecodeError
@@ -18,10 +19,9 @@
 
 class MQTTResponse(BaseResponse):
     def __init__(self, client, name, expected, test_block_config):
-        # pylint: disable=unused-argument
-
         super(MQTTResponse, self).__init__()
 
+        self.test_block_config = test_block_config
         self.name = name
 
         self._check_for_validate_functions(expected.get("payload", {}))
@@ -78,6 +78,13 @@ def _await_response(self):
                 # timed out
                 break
 
+            call_hook(
+                self.test_block_config,
+                "pytest_tavern_beta_after_every_response",
+                expected=self.expected,
+                response=msg,
+            )
+
             self.received_messages.append(msg)
 
             msg.payload = msg.payload.decode("utf8")

tavern/_plugins/rest/response.py

@@ -9,6 +9,7 @@
 
 from requests.status_codes import _codes
 
+from tavern.testutils.pytesthook.newhooks import call_hook
 from tavern.schemas.extensions import get_wrapped_response_function
 from tavern.util.dict_util import recurse_access_key, deep_dict_merge
 from tavern.util import exceptions
@@ -149,6 +150,13 @@ def verify(self, response):
         """
         self._verbose_log_response(response)
 
+        call_hook(
+            self.test_block_config,
+            "pytest_tavern_beta_after_every_response",
+            expected=self.expected,
+            response=response,
+        )
+
         self.response = response
         self.status_code = response.status_code
 

tavern/testutils/pytesthook/__init__.py

@@ -1,4 +1,9 @@
-from .hooks import pytest_collect_file, pytest_addoption
+from .hooks import pytest_collect_file, pytest_addoption, pytest_addhooks
 from .util import add_parser_options
 
-__all__ = ["pytest_addoption", "pytest_collect_file", "add_parser_options"]
+__all__ = [
+    "pytest_addoption",
+    "pytest_collect_file",
+    "pytest_addhooks",
+    "add_parser_options",
+]

tavern/testutils/pytesthook/hooks.py

@@ -54,3 +54,10 @@ def pytest_collect_file(parent, path):
         return YamlFile(path, parent)
 
     return None
+
+
+def pytest_addhooks(pluginmanager):
+    """Add our custom tavern hooks"""
+    from . import newhooks
+
+    pluginmanager.add_hookspecs(newhooks)

tavern/testutils/pytesthook/item.py

@@ -10,6 +10,7 @@
 from tavern.plugins import load_plugins
 from tavern.schemas.files import verify_tests
 from tavern.util import exceptions
+from tavern.testutils.pytesthook.newhooks import call_hook
 
 from .error import ReprdError
 from .util import load_global_cfg
@@ -139,18 +140,27 @@ def runtest(self):
 
         load_plugins(self.global_cfg)
 
+        self.global_cfg["tavern_internal"] = {"pytest_hook_caller": self.config.hook}
+
         # INTERNAL
         # NOTE - now that we can 'mark' tests, we could use pytest.mark.xfail
         # instead. This doesn't differentiate between an error in verification
         # and an error when running the test though.
         xfail = self.spec.get("_xfail", False)
 
         try:
-            verify_tests(self.spec)
-
             fixture_values = self._load_fixture_values()
             self.global_cfg["variables"].update(fixture_values)
 
+            call_hook(
+                self.global_cfg,
+                "pytest_tavern_beta_before_every_test_run",
+                test_dict=self.spec,
+                variables=self.global_cfg["variables"],
+            )
+
+            verify_tests(self.spec)
+
             run_test(self.path, self.spec, self.global_cfg)
         except exceptions.BadSchemaError:
             if xfail == "verify":

tavern/testutils/pytesthook/newhooks.py

@@ -0,0 +1,51 @@
+# pylint: disable=unused-argument
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def pytest_tavern_beta_before_every_test_run(test_dict, variables):
+    """Called:
+
+    - directly after fixtures are loaded for a test
+    - directly before verifying the schema of the file
+    - Before formatting is done on values
+    - After global configuration has been loaded
+    - After plugins have been loaded
+
+    Modify the test in-place if you want to do something to it.
+
+    Args:
+        test_dict (dict): Test to run
+        variables (dict): Available variables
+    """
+
+
+def pytest_tavern_beta_after_every_response(expected, response):
+    """Called after every _response_ - including MQTT/HTTP/etc
+
+    Note:
+        - The response object type and the expected dict depends on what plugin you're using, and which kind of response it is!
+        - MQTT responses will call this hook multiple times if multiple messages are received
+
+    Args:
+        response (object): Response object.
+        expected (dict): Response block in stage
+    """
+
+
+def call_hook(test_block_config, hookname, **kwargs):
+    """Utility to call the hooks"""
+    try:
+        hook = getattr(
+            test_block_config["tavern_internal"]["pytest_hook_caller"], hookname
+        )
+    except AttributeError:
+        logger.critical("Error getting tavern hook!")
+        raise
+
+    try:
+        hook(**kwargs)
+    except AttributeError:
+        logger.error("Unexpected error calling tavern hook")
+        raise

tests/unit/conftest.py

@@ -1,3 +1,4 @@
+from mock import Mock
 import pytest
 
 
@@ -13,6 +14,7 @@ def fix_example_includes():
         },
         "backends": {"mqtt": "paho-mqtt", "http": "requests"},
         "strict": True,
+        "tavern_internal": {"pytest_hook_caller": Mock()},
     }
 
     return includes.copy()