Merge pull request #102 from dhlavac/perf

Kuadrant performance test framework setup

Makefile

@@ -41,6 +41,10 @@ test: ## Run test
 test pytest tests: pipenv
 	$(PYTEST) -n4 -m 'not flaky' --dist loadfile $(flags) testsuite
 
+# Run performance tests
+performance: pipenv
+	$(PYTEST) --performance $(flags) testsuite/tests/kuadrant/authorino/performance
+
 Pipfile.lock: Pipfile
 	pipenv lock
 

config/settings.local.yaml.tpl

@@ -25,6 +25,8 @@
 #  mockserver:
 #    url: "MOCKSERVER_URL"
 #  cfssl: "cfssl"  # Path to the CFSSL library for TLS tests
+#  hyperfoil:
+#    url: "HYPERFOIL_URL"
 #  authorino:
 #    image: "quay.io/kuadrant/authorino:latest"  # If specified will override the authorino image
 #    deploy: false                               # If false, the testsuite will use already deployed authorino for testing

mypy.ini

@@ -11,3 +11,6 @@ ignore_missing_imports = True
 
 [mypy-openshift.*]
 ignore_missing_imports = True
+
+[mypy-hyperfoil.*]
+ignore_missing_imports = True

pytest.ini

@@ -1,6 +1,7 @@
 [pytest]
 markers =
     issue: Reference to covered issue
+    performance: Performance tests have unique needs
 filterwarnings =
     ignore: WARNING the new order is not taken into account:UserWarning
     ignore::urllib3.exceptions.InsecureRequestWarning

testsuite/oidc/rhsso/__init__.py

@@ -85,3 +85,11 @@ def refresh_token(self, refresh_token):
     def get_token(self, username=None, password=None) -> Token:
         data = self.oidc_client.token(username or self.test_username, password or self.test_password)
         return Token(data["access_token"], self.refresh_token, data["refresh_token"])
+
+    def token_params(self) -> str:
+        """
+        Returns token parameters that can be added to request url
+        """
+        return f"grant_type=password&client_id={self.oidc_client.client_id}&" \
+               f"client_secret={self.oidc_client.client_secret_key}&username={self.test_username}&" \
+               f"password={self.test_password}"

testsuite/perf_utils.py

@@ -0,0 +1,110 @@
+"""
+This file contains methods that are used in performance testing
+"""
+import os
+from urllib.parse import urlparse, ParseResult
+from importlib import resources
+
+import yaml
+from hyperfoil.factories import HyperfoilFactory, Benchmark
+
+from testsuite.objects import LifecycleObject
+
+
+def _load_benchmark(filename):
+    """Loads benchmark"""
+    with open(filename, encoding="utf8") as file:
+        benchmark = Benchmark(yaml.load(file, Loader=yaml.Loader))
+    return benchmark
+
+
+def authority(url: str):
+    """Returns hyperfoil authority format of URL <hostname>:<port> from given URL."""
+    parsed_url = urlparse(url)
+    return f"{parsed_url.hostname}:{parsed_url.port}"
+
+
+def prepare_url(url: ParseResult) -> ParseResult:
+    """ Adds port number to url if it is not set"""
+    if not url.hostname:
+        raise ValueError("Missing hostname part of url")
+    if not url.port:
+        url_port = 80 if url.scheme == 'http' else 443
+        url = url._replace(netloc=url.hostname + f":{url_port}")
+    return url
+
+
+class HyperfoilUtils(LifecycleObject):
+    """
+        Setup class for hyperfoil test and wrapper of Hyperfoil-python-client.
+    """
+    message_1kb = resources.files('testsuite.resources.performance.files').joinpath('message_1kb.txt')
+
+    def __init__(self, hyperfoil_client, template_filename):
+        self.hyperfoil_client = hyperfoil_client
+        self.factory = HyperfoilFactory(hyperfoil_client)
+        self.benchmark = None
+        self.template_filename = template_filename
+
+    def commit(self):
+        """Open file streams for Hyperfoil benchmark"""
+        self.benchmark = _load_benchmark(self.template_filename)
+
+    def create_benchmark(self):
+        """Creates benchmark"""
+        benchmark = self.benchmark.create()
+        return self.factory.benchmark(benchmark).create()
+
+    def update_benchmark(self, benchmark):
+        """Updates benchmark"""
+        self.benchmark.update(benchmark=benchmark)
+
+    def add_shared_template(self, agents_number: int):
+        """Updates benchmark with shared template for hyperfoil agents setup"""
+        agents: dict = {'agents': {}}
+        for i in range(1, agents_number + 1):
+            agent = {'host': 'localhost', 'port': 22, 'stop': True}
+            agents['agents'][f'agent-{i}'] = agent
+        self.benchmark.update(agents)
+
+    def delete(self):
+        """Hyperfoil factory opens a lot of file streams, we need to ensure that they are closed."""
+        self.factory.close()
+
+    def add_host(self, url: str, shared_connections: int, **kwargs):
+        """Adds specific url host to the benchmark"""
+        self.benchmark.add_host(url, shared_connections, **kwargs)
+
+    # pylint: disable=consider-using-with
+    def add_file(self, path):
+        """Adds file to the benchmark"""
+        filename = os.path.basename(path)
+        self.factory.file(filename, open(path, 'r', encoding="utf8"))
+
+    def generate_random_file(self, filename: str, size: int):
+        """Generates and adds file with such filename and size to the benchmark"""
+        self.factory.generate_random_file(filename, size)
+
+    def generate_random_files(self, files: dict):
+        """Generates and adds files to the benchmark"""
+        for filename, size in files.items():
+            self.factory.generate_random_file(filename, size)
+
+    def add_user_key_auth(self, user_key, url, filename):
+        """
+        TODO: add method for user key authentication
+        """
+
+    def add_rhsso_auth_token(self, rhsso, client_url, filename):
+        """
+        Adds csv file with data for access token creation. Each row consists of following columns:
+        [authority url, rhsso url, rhsso path, body for token creation]
+        :param rhsso: rhsso service fixture
+        :param client_url: url of desired endpoint to be tested
+        :param filename: name of csv file
+        """
+        rows = []
+        token_url_obj = prepare_url(urlparse(rhsso.well_known['token_endpoint']))
+        rows.append([client_url, f"{token_url_obj.hostname}:{token_url_obj.port}", token_url_obj.path,
+                     rhsso.token_params()])
+        self.factory.csv_data(filename, rows)

testsuite/resources/performance/files/message_1kb.txt

@@ -0,0 +1 @@
+U0BX97dqR9WWhDxUYSE7WoucYzpFegYWqq7Kt6wA4iUjdSh2ztbIRrE5qO5SDWqb3UBqnEHMMgFEbtiFMyewWSngePLOScW3gvZNxgp8qdfiWYeZkctEgkjGZpSHtna6vbu1Uu4QnTkvqDDcV85n3Sz7WdqQz734LNKT7Ft5IqdUytbPfcTV6mLCcHQoqG5ZX1fC60Coyc0QuipTlzZW3EbxOWqehkWqZuA1BC2tK6FjCI9TErc0tEpRre5s0mMwBOtfVVjylk0uyGL41EaRnRwanH69u84PamVnhPr33LaVBJ7zo9R2MNVR1DnORJuul8ahgxVpxblj8nuybiPOTdRRR9TUbulPoinIOvitk56e4ihQPmHMJx0EQ456PnSyrdZy1k3BS9fmw8VDIiqwDobokpRcxaKJdPqjwkHESgRU4Adcx1MYTdaSkKxbcLK9szojv1k944u7yZ7qyPQrSxUQSir8kKvkvJSSAkPxOcwRzt7K6qzw1R8xQKTqG9bo9b01qta1dZZGauL4MbD9jCgRaTbM7cqCS4jv0osJxyioGPd9tlGn8RyWhvwdBpO7nuS9LqF1vskgtuqU2LtOMEgHYDIikOxsIRjUjbcw5jaznlPOVnRF7A5NTBGiAZMihm6dWyrTiUN9fssSIUkOnAXaWBDBz5idsnl16hsym1GtsOCYRh9vjFAk2ayNL4uvGmrFBXbHur2F1rWldjaGGq1koHMarasC9acbUJ6SumM36mytLSI4TqDo7KEnsRk6gejIKCPitltTSMN9bCjPXB7vUXVGjBPaQIfWkRxHdZspZ1gj9E3FRqs0SrAkYX7stuHznQnqhGDbHmYDXOiSchxEEaxPf7tRwJp7oXbynMNgTGimyAKIKe82ugdvDSCAX1XHTWanFnndKXvB4qH71dUHSoZa4GHXdistqbQCd8bca3IUCcku9kb1HzMqy0I4mQVWFpVLXUy2a7lOo1L8FCaFROvLcbzRkIfQFSzf7xfuCQSwPYPtQpNLLNT14p7N

testsuite/tests/conftest.py

@@ -17,6 +17,19 @@
 from testsuite.utils import randomize, _whoami
 
 
+def pytest_addoption(parser):
+    """Add option to include performance tests in testrun"""
+    parser.addoption(
+        "--performance", action="store_true", default=False, help="Run also performance tests (default: False)")
+
+
+def pytest_runtest_setup(item):
+    """Exclude performance tests by default, require explicit option"""
+    marks = [i.name for i in item.iter_markers()]
+    if "performance" in marks and not item.config.getoption("--performance"):
+        pytest.skip("Excluding performance tests")
+
+
 @pytest.fixture(scope='session', autouse=True)
 def term_handler():
     """

testsuite/tests/kuadrant/authorino/performance/conftest.py

@@ -0,0 +1,39 @@
+"""
+Conftest for performance tests
+"""
+import pytest
+from hyperfoil import HyperfoilClient
+from dynaconf import ValidationError
+
+from testsuite.perf_utils import HyperfoilUtils
+from testsuite.httpx.auth import HttpxOidcClientAuth
+
+
+@pytest.fixture(scope='session')
+def hyperfoil_client(testconfig):
+    """Hyperfoil client"""
+    try:
+        return HyperfoilClient(testconfig['hyperfoil']['url'])
+    except (KeyError, ValidationError) as exc:
+        return pytest.skip(f"Hyperfoil configuration item is missing: {exc}")
+
+
+@pytest.fixture(scope='module')
+def hyperfoil_utils(hyperfoil_client, template, request):
+    """Init of hyperfoil utils"""
+    utils = HyperfoilUtils(hyperfoil_client, template)
+    request.addfinalizer(utils.delete)
+    utils.commit()
+    return utils
+
+
+@pytest.fixture(scope="module")
+def rhsso_auth(rhsso):
+    """Returns RHSSO authentication object for HTTPX"""
+    return HttpxOidcClientAuth(rhsso.get_token)
+
+
+@pytest.fixture(scope='module')
+def number_of_agents():
+    """Number of spawned HyperFoil agents"""
+    return 1

testsuite/tests/kuadrant/authorino/performance/templates/template_perf_basic_query_rhsso.hf.yaml

@@ -0,0 +1,60 @@
+name: test_perf_basic
+# http endpoints will be added via test
+phases:
+  - rampUp:
+      increasingRate:
+        duration: 1m
+        maxDuration: 3m
+        initialUsersPerSec: 8
+        targetUsersPerSec: 20
+        scenario:
+          - loadCsv: &loadCsv
+            - randomCsvRow:
+                file: 'rhsso_auth.csv'
+                skipComments: true
+                removeQuotes: true
+                columns:
+                  0: 0 #hostname
+                  1: 1 #rhsso_url
+                  2: 2 #path
+                  3: 3 #data
+          - createToken: &createToken
+            - httpRequest:
+                  authority:
+                    fromVar: 1
+                  POST:
+                    fromVar: 2
+                  headers:
+                    Content-Type: application/x-www-form-urlencoded
+                  body:
+                    fromVar: 3
+                  handler:
+                    body:
+                     json:
+                       query: .access_token
+                       toVar: access_token
+          - postLargeData: &postLargeData
+            - template:
+                    pattern: Bearer ${access_token}
+                    toVar: authorization
+            - httpRequest:
+                authority:
+                  fromVar: 0
+                POST: /post
+                sync: true
+                headers:
+                  authorization:
+                    fromVar: authorization
+                body:
+                  fromFile: message_1kb.txt
+  - steadyLoad:
+      constantRate:
+        duration: 2m
+        maxDuration: 4m
+        usersPerSec: 12
+        startAfter:
+          phase: rampUp
+        scenario:
+          - loadCsv: *loadCsv
+          - createToken: *createToken
+          - postLargeData: *postLargeData

testsuite/tests/kuadrant/authorino/performance/test_perf_basic.py

@@ -0,0 +1,86 @@
+"""
+    Test that will set up authorino and prepares objects for performance testing.
+    Fill necessary data to benchmark template.
+    Run the test and assert results.
+"""
+from urllib.parse import urlparse
+from importlib import resources
+
+import backoff
+import pytest
+
+from testsuite.perf_utils import HyperfoilUtils, prepare_url
+
+# Maximal runtime of test (need to cover all performance stages)
+MAX_RUN_TIME = 10 * 60
+# Number of Hyperfoil agents to be spawned
+AGENTS = 2
+
+pytestmark = [pytest.mark.performance]
+
+
+@pytest.fixture(scope='module')
+def number_of_agents():
+    """Number of spawned HyperFoil agents"""
+    return AGENTS
+
+
+@pytest.fixture(scope='module')
+def template():
+    """Path to template"""
+    return resources.files("testsuite.tests.kuadrant.authorino.performance.templates")\
+        .joinpath('template_perf_basic_query_rhsso.hf.yaml')
+
+
+@pytest.fixture(scope='module')
+def hyperfoil_utils(hyperfoil_client, template, request):
+    """Init of hyperfoil utils"""
+    utils = HyperfoilUtils(hyperfoil_client, template)
+    request.addfinalizer(utils.delete)
+    utils.commit()
+    return utils
+
+
+@pytest.fixture(scope='module')
+def setup_benchmark_rhsso(hyperfoil_utils, client, rhsso, number_of_agents):
+    """Setup of benchmark. It will add necessary host connections, csv data and files."""
+    # currently number of shared connections is set as a placeholder and later should be determined by test results
+    url_pool = [{'url': rhsso.server_url, 'connections': 100}, {'url': str(client.base_url), 'connections': 20}]
+    for url in url_pool:
+        complete_url = prepare_url(urlparse(url['url']))
+        hyperfoil_utils.add_host(complete_url._replace(path="").geturl(), shared_connections=url['connections'])
+
+    hyperfoil_utils.add_rhsso_auth_token(rhsso, prepare_url(urlparse(str(client.base_url))).netloc, 'rhsso_auth.csv')
+    hyperfoil_utils.add_file(HyperfoilUtils.message_1kb)
+    hyperfoil_utils.add_shared_template(number_of_agents)
+    return hyperfoil_utils
+
+
+@backoff.on_predicate(backoff.constant, lambda x: not x.is_finished(), interval=5, max_time=MAX_RUN_TIME)
+def wait_run(run):
+    """Waits for the run to end"""
+    return run.reload()
+
+
+def test_basic_perf_rhsso(client, rhsso_auth, setup_benchmark_rhsso):
+    """
+       Test checks that authorino is set up correctly.
+       Runs the created benchmark.
+       Asserts it was successful.
+    """
+    get_response = client.get("/get", auth=rhsso_auth)
+    post_response = client.post("/post", auth=rhsso_auth)
+    assert get_response.status_code == 200
+    assert post_response.status_code == 200
+
+    benchmark = setup_benchmark_rhsso.create_benchmark()
+    run = benchmark.start()
+
+    run = wait_run(run)
+
+    stats = run.all_stats()
+
+    assert stats
+    assert stats.get('info', {}).get('errors') == []
+    assert stats.get('failures') == []
+    assert stats.get('stats', []) != []