diff --git a/Makefile b/Makefile index 5476099c..74c9864e 100644 --- a/Makefile +++ b/Makefile @@ -41,6 +41,10 @@ test: ## Run test test pytest tests: pipenv $(PYTEST) -n4 -m 'not flaky' --dist loadfile $(flags) testsuite +# Run performance tests +performance: pipenv + $(PYTEST) --performance $(flags) testsuite/tests/kuadrant/authorino/performance + Pipfile.lock: Pipfile pipenv lock diff --git a/config/settings.local.yaml.tpl b/config/settings.local.yaml.tpl index 3f339ec5..43343633 100644 --- a/config/settings.local.yaml.tpl +++ b/config/settings.local.yaml.tpl @@ -25,6 +25,8 @@ # mockserver: # url: "MOCKSERVER_URL" # cfssl: "cfssl" # Path to the CFSSL library for TLS tests +# hyperfoil: +# url: "HYPERFOIL_URL" # authorino: # image: "quay.io/kuadrant/authorino:latest" # If specified will override the authorino image # deploy: false # If false, the testsuite will use already deployed authorino for testing diff --git a/mypy.ini b/mypy.ini index 69eb07c1..28418de4 100644 --- a/mypy.ini +++ b/mypy.ini @@ -11,3 +11,6 @@ ignore_missing_imports = True [mypy-openshift.*] ignore_missing_imports = True + +[mypy-hyperfoil.*] +ignore_missing_imports = True diff --git a/pytest.ini b/pytest.ini index c9e86a64..250f2c7a 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,6 +1,7 @@ [pytest] markers = issue: Reference to covered issue + performance: Performance tests have unique needs filterwarnings = ignore: WARNING the new order is not taken into account:UserWarning ignore::urllib3.exceptions.InsecureRequestWarning diff --git a/testsuite/oidc/rhsso/__init__.py b/testsuite/oidc/rhsso/__init__.py index 2bc4f4a0..4b0e59f2 100644 --- a/testsuite/oidc/rhsso/__init__.py +++ b/testsuite/oidc/rhsso/__init__.py @@ -85,3 +85,11 @@ def refresh_token(self, refresh_token): def get_token(self, username=None, password=None) -> Token: data = self.oidc_client.token(username or self.test_username, password or self.test_password) return Token(data["access_token"], self.refresh_token, data["refresh_token"]) + + def token_params(self) -> str: + """ + Returns token parameters that can be added to request url + """ + return f"grant_type=password&client_id={self.oidc_client.client_id}&" \ + f"client_secret={self.oidc_client.client_secret_key}&username={self.test_username}&" \ + f"password={self.test_password}" diff --git a/testsuite/perf_utils.py b/testsuite/perf_utils.py new file mode 100644 index 00000000..f5c554ea --- /dev/null +++ b/testsuite/perf_utils.py @@ -0,0 +1,110 @@ +""" +This file contains methods that are used in performance testing +""" +import os +from urllib.parse import urlparse, ParseResult +from importlib import resources + +import yaml +from hyperfoil.factories import HyperfoilFactory, Benchmark + +from testsuite.objects import LifecycleObject + + +def _load_benchmark(filename): + """Loads benchmark""" + with open(filename, encoding="utf8") as file: + benchmark = Benchmark(yaml.load(file, Loader=yaml.Loader)) + return benchmark + + +def authority(url: str): + """Returns hyperfoil authority format of URL : from given URL.""" + parsed_url = urlparse(url) + return f"{parsed_url.hostname}:{parsed_url.port}" + + +def prepare_url(url: ParseResult) -> ParseResult: + """ Adds port number to url if it is not set""" + if not url.hostname: + raise ValueError("Missing hostname part of url") + if not url.port: + url_port = 80 if url.scheme == 'http' else 443 + url = url._replace(netloc=url.hostname + f":{url_port}") + return url + + +class HyperfoilUtils(LifecycleObject): + """ + Setup class for hyperfoil test and wrapper of Hyperfoil-python-client. + """ + message_1kb = resources.files('testsuite.resources.performance.files').joinpath('message_1kb.txt') + + def __init__(self, hyperfoil_client, template_filename): + self.hyperfoil_client = hyperfoil_client + self.factory = HyperfoilFactory(hyperfoil_client) + self.benchmark = None + self.template_filename = template_filename + + def commit(self): + """Open file streams for Hyperfoil benchmark""" + self.benchmark = _load_benchmark(self.template_filename) + + def create_benchmark(self): + """Creates benchmark""" + benchmark = self.benchmark.create() + return self.factory.benchmark(benchmark).create() + + def update_benchmark(self, benchmark): + """Updates benchmark""" + self.benchmark.update(benchmark=benchmark) + + def add_shared_template(self, agents_number: int): + """Updates benchmark with shared template for hyperfoil agents setup""" + agents: dict = {'agents': {}} + for i in range(1, agents_number + 1): + agent = {'host': 'localhost', 'port': 22, 'stop': True} + agents['agents'][f'agent-{i}'] = agent + self.benchmark.update(agents) + + def delete(self): + """Hyperfoil factory opens a lot of file streams, we need to ensure that they are closed.""" + self.factory.close() + + def add_host(self, url: str, shared_connections: int, **kwargs): + """Adds specific url host to the benchmark""" + self.benchmark.add_host(url, shared_connections, **kwargs) + + # pylint: disable=consider-using-with + def add_file(self, path): + """Adds file to the benchmark""" + filename = os.path.basename(path) + self.factory.file(filename, open(path, 'r', encoding="utf8")) + + def generate_random_file(self, filename: str, size: int): + """Generates and adds file with such filename and size to the benchmark""" + self.factory.generate_random_file(filename, size) + + def generate_random_files(self, files: dict): + """Generates and adds files to the benchmark""" + for filename, size in files.items(): + self.factory.generate_random_file(filename, size) + + def add_user_key_auth(self, user_key, url, filename): + """ + TODO: add method for user key authentication + """ + + def add_rhsso_auth_token(self, rhsso, client_url, filename): + """ + Adds csv file with data for access token creation. Each row consists of following columns: + [authority url, rhsso url, rhsso path, body for token creation] + :param rhsso: rhsso service fixture + :param client_url: url of desired endpoint to be tested + :param filename: name of csv file + """ + rows = [] + token_url_obj = prepare_url(urlparse(rhsso.well_known['token_endpoint'])) + rows.append([client_url, f"{token_url_obj.hostname}:{token_url_obj.port}", token_url_obj.path, + rhsso.token_params()]) + self.factory.csv_data(filename, rows) diff --git a/testsuite/resources/performance/files/message_1kb.txt b/testsuite/resources/performance/files/message_1kb.txt new file mode 100644 index 00000000..ea4a20e2 --- /dev/null +++ b/testsuite/resources/performance/files/message_1kb.txt @@ -0,0 +1 @@ +U0BX97dqR9WWhDxUYSE7WoucYzpFegYWqq7Kt6wA4iUjdSh2ztbIRrE5qO5SDWqb3UBqnEHMMgFEbtiFMyewWSngePLOScW3gvZNxgp8qdfiWYeZkctEgkjGZpSHtna6vbu1Uu4QnTkvqDDcV85n3Sz7WdqQz734LNKT7Ft5IqdUytbPfcTV6mLCcHQoqG5ZX1fC60Coyc0QuipTlzZW3EbxOWqehkWqZuA1BC2tK6FjCI9TErc0tEpRre5s0mMwBOtfVVjylk0uyGL41EaRnRwanH69u84PamVnhPr33LaVBJ7zo9R2MNVR1DnORJuul8ahgxVpxblj8nuybiPOTdRRR9TUbulPoinIOvitk56e4ihQPmHMJx0EQ456PnSyrdZy1k3BS9fmw8VDIiqwDobokpRcxaKJdPqjwkHESgRU4Adcx1MYTdaSkKxbcLK9szojv1k944u7yZ7qyPQrSxUQSir8kKvkvJSSAkPxOcwRzt7K6qzw1R8xQKTqG9bo9b01qta1dZZGauL4MbD9jCgRaTbM7cqCS4jv0osJxyioGPd9tlGn8RyWhvwdBpO7nuS9LqF1vskgtuqU2LtOMEgHYDIikOxsIRjUjbcw5jaznlPOVnRF7A5NTBGiAZMihm6dWyrTiUN9fssSIUkOnAXaWBDBz5idsnl16hsym1GtsOCYRh9vjFAk2ayNL4uvGmrFBXbHur2F1rWldjaGGq1koHMarasC9acbUJ6SumM36mytLSI4TqDo7KEnsRk6gejIKCPitltTSMN9bCjPXB7vUXVGjBPaQIfWkRxHdZspZ1gj9E3FRqs0SrAkYX7stuHznQnqhGDbHmYDXOiSchxEEaxPf7tRwJp7oXbynMNgTGimyAKIKe82ugdvDSCAX1XHTWanFnndKXvB4qH71dUHSoZa4GHXdistqbQCd8bca3IUCcku9kb1HzMqy0I4mQVWFpVLXUy2a7lOo1L8FCaFROvLcbzRkIfQFSzf7xfuCQSwPYPtQpNLLNT14p7N \ No newline at end of file diff --git a/testsuite/tests/conftest.py b/testsuite/tests/conftest.py index 3fb69b54..32fab04e 100644 --- a/testsuite/tests/conftest.py +++ b/testsuite/tests/conftest.py @@ -17,6 +17,19 @@ from testsuite.utils import randomize, _whoami +def pytest_addoption(parser): + """Add option to include performance tests in testrun""" + parser.addoption( + "--performance", action="store_true", default=False, help="Run also performance tests (default: False)") + + +def pytest_runtest_setup(item): + """Exclude performance tests by default, require explicit option""" + marks = [i.name for i in item.iter_markers()] + if "performance" in marks and not item.config.getoption("--performance"): + pytest.skip("Excluding performance tests") + + @pytest.fixture(scope='session', autouse=True) def term_handler(): """ diff --git a/testsuite/tests/kuadrant/authorino/performance/__init__.py b/testsuite/tests/kuadrant/authorino/performance/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testsuite/tests/kuadrant/authorino/performance/conftest.py b/testsuite/tests/kuadrant/authorino/performance/conftest.py new file mode 100644 index 00000000..5370b397 --- /dev/null +++ b/testsuite/tests/kuadrant/authorino/performance/conftest.py @@ -0,0 +1,39 @@ +""" +Conftest for performance tests +""" +import pytest +from hyperfoil import HyperfoilClient +from dynaconf import ValidationError + +from testsuite.perf_utils import HyperfoilUtils +from testsuite.httpx.auth import HttpxOidcClientAuth + + +@pytest.fixture(scope='session') +def hyperfoil_client(testconfig): + """Hyperfoil client""" + try: + return HyperfoilClient(testconfig['hyperfoil']['url']) + except (KeyError, ValidationError) as exc: + return pytest.skip(f"Hyperfoil configuration item is missing: {exc}") + + +@pytest.fixture(scope='module') +def hyperfoil_utils(hyperfoil_client, template, request): + """Init of hyperfoil utils""" + utils = HyperfoilUtils(hyperfoil_client, template) + request.addfinalizer(utils.delete) + utils.commit() + return utils + + +@pytest.fixture(scope="module") +def rhsso_auth(rhsso): + """Returns RHSSO authentication object for HTTPX""" + return HttpxOidcClientAuth(rhsso.get_token) + + +@pytest.fixture(scope='module') +def number_of_agents(): + """Number of spawned HyperFoil agents""" + return 1 diff --git a/testsuite/tests/kuadrant/authorino/performance/templates/__init__.py b/testsuite/tests/kuadrant/authorino/performance/templates/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testsuite/tests/kuadrant/authorino/performance/templates/template_perf_basic_query_rhsso.hf.yaml b/testsuite/tests/kuadrant/authorino/performance/templates/template_perf_basic_query_rhsso.hf.yaml new file mode 100644 index 00000000..3cd2a283 --- /dev/null +++ b/testsuite/tests/kuadrant/authorino/performance/templates/template_perf_basic_query_rhsso.hf.yaml @@ -0,0 +1,60 @@ +name: test_perf_basic +# http endpoints will be added via test +phases: + - rampUp: + increasingRate: + duration: 1m + maxDuration: 3m + initialUsersPerSec: 8 + targetUsersPerSec: 20 + scenario: + - loadCsv: &loadCsv + - randomCsvRow: + file: 'rhsso_auth.csv' + skipComments: true + removeQuotes: true + columns: + 0: 0 #hostname + 1: 1 #rhsso_url + 2: 2 #path + 3: 3 #data + - createToken: &createToken + - httpRequest: + authority: + fromVar: 1 + POST: + fromVar: 2 + headers: + Content-Type: application/x-www-form-urlencoded + body: + fromVar: 3 + handler: + body: + json: + query: .access_token + toVar: access_token + - postLargeData: &postLargeData + - template: + pattern: Bearer ${access_token} + toVar: authorization + - httpRequest: + authority: + fromVar: 0 + POST: /post + sync: true + headers: + authorization: + fromVar: authorization + body: + fromFile: message_1kb.txt + - steadyLoad: + constantRate: + duration: 2m + maxDuration: 4m + usersPerSec: 12 + startAfter: + phase: rampUp + scenario: + - loadCsv: *loadCsv + - createToken: *createToken + - postLargeData: *postLargeData \ No newline at end of file diff --git a/testsuite/tests/kuadrant/authorino/performance/test_perf_basic.py b/testsuite/tests/kuadrant/authorino/performance/test_perf_basic.py new file mode 100644 index 00000000..ad4a6597 --- /dev/null +++ b/testsuite/tests/kuadrant/authorino/performance/test_perf_basic.py @@ -0,0 +1,86 @@ +""" + Test that will set up authorino and prepares objects for performance testing. + Fill necessary data to benchmark template. + Run the test and assert results. +""" +from urllib.parse import urlparse +from importlib import resources + +import backoff +import pytest + +from testsuite.perf_utils import HyperfoilUtils, prepare_url + +# Maximal runtime of test (need to cover all performance stages) +MAX_RUN_TIME = 10 * 60 +# Number of Hyperfoil agents to be spawned +AGENTS = 2 + +pytestmark = [pytest.mark.performance] + + +@pytest.fixture(scope='module') +def number_of_agents(): + """Number of spawned HyperFoil agents""" + return AGENTS + + +@pytest.fixture(scope='module') +def template(): + """Path to template""" + return resources.files("testsuite.tests.kuadrant.authorino.performance.templates")\ + .joinpath('template_perf_basic_query_rhsso.hf.yaml') + + +@pytest.fixture(scope='module') +def hyperfoil_utils(hyperfoil_client, template, request): + """Init of hyperfoil utils""" + utils = HyperfoilUtils(hyperfoil_client, template) + request.addfinalizer(utils.delete) + utils.commit() + return utils + + +@pytest.fixture(scope='module') +def setup_benchmark_rhsso(hyperfoil_utils, client, rhsso, number_of_agents): + """Setup of benchmark. It will add necessary host connections, csv data and files.""" + # currently number of shared connections is set as a placeholder and later should be determined by test results + url_pool = [{'url': rhsso.server_url, 'connections': 100}, {'url': str(client.base_url), 'connections': 20}] + for url in url_pool: + complete_url = prepare_url(urlparse(url['url'])) + hyperfoil_utils.add_host(complete_url._replace(path="").geturl(), shared_connections=url['connections']) + + hyperfoil_utils.add_rhsso_auth_token(rhsso, prepare_url(urlparse(str(client.base_url))).netloc, 'rhsso_auth.csv') + hyperfoil_utils.add_file(HyperfoilUtils.message_1kb) + hyperfoil_utils.add_shared_template(number_of_agents) + return hyperfoil_utils + + +@backoff.on_predicate(backoff.constant, lambda x: not x.is_finished(), interval=5, max_time=MAX_RUN_TIME) +def wait_run(run): + """Waits for the run to end""" + return run.reload() + + +def test_basic_perf_rhsso(client, rhsso_auth, setup_benchmark_rhsso): + """ + Test checks that authorino is set up correctly. + Runs the created benchmark. + Asserts it was successful. + """ + get_response = client.get("/get", auth=rhsso_auth) + post_response = client.post("/post", auth=rhsso_auth) + assert get_response.status_code == 200 + assert post_response.status_code == 200 + + benchmark = setup_benchmark_rhsso.create_benchmark() + run = benchmark.start() + + run = wait_run(run) + + stats = run.all_stats() + + assert stats + assert stats.get('info', {}).get('errors') == [] + assert stats.get('failures') == [] + assert stats.get('stats', []) != []