From e2a01578614f256256198ffddee3b93ef536fc00 Mon Sep 17 00:00:00 2001 From: Christoph Kuhnke Date: Mon, 27 May 2024 15:59:14 +0200 Subject: [PATCH] #10 add fixtures (#13) * #10: Added pytest fixtures for usage in integration tests of external projects * Use pyyaml for reading project short tag from error_code_config.yml * Added ip-whitelisting to fixture operational_saas_database_id * Verified fixture in test_operational_database * Replaced bash script in justfile by python * Added developer instructions * Updated README file * moved developer instructions to Developer guide. Co-authored-by: Torsten Kilias --- .github/workflows/ci-main.yml | 5 +- .github/workflows/ci-pr.yml | 5 +- .github/workflows/ci.yml | 22 ++- README.md | 44 ++++++ README.rst | 55 -------- doc/developer-guide.md | 23 +++ justfile | 17 ++- pytest-itde/pyproject.toml | 4 + pytest-saas/README.md | 40 ++++++ pytest-saas/doc/changes/unreleased.md | 1 + pytest-saas/exasol/pytest_saas/__init__.py | 101 ++++++++++++++ .../exasol/pytest_saas/project_short_tag.py | 28 ++++ pytest-saas/poetry.lock | 2 +- pytest-saas/pyproject.toml | 6 +- .../test/integration/pytest_saas_test.py | 132 ++++++++++++++++++ pytest-saas/test/integration/smoke_test.py | 2 - 16 files changed, 418 insertions(+), 69 deletions(-) create mode 100644 README.md delete mode 100644 README.rst create mode 100644 doc/developer-guide.md create mode 100644 pytest-saas/exasol/pytest_saas/__init__.py create mode 100644 pytest-saas/exasol/pytest_saas/project_short_tag.py create mode 100644 pytest-saas/test/integration/pytest_saas_test.py delete mode 100644 pytest-saas/test/integration/smoke_test.py diff --git a/.github/workflows/ci-main.yml b/.github/workflows/ci-main.yml index 427dfee..01efa10 100644 --- a/.github/workflows/ci-main.yml +++ b/.github/workflows/ci-main.yml @@ -1,4 +1,4 @@ -name: Continues Integration (Master) +name: Continuous Integration (Main) on: workflow_dispatch: @@ -13,3 +13,6 @@ jobs: CI: uses: ./.github/workflows/ci.yml + secrets: inherit + with: + slow-tests: true diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml index e00b9df..d22b714 100644 --- a/.github/workflows/ci-pr.yml +++ b/.github/workflows/ci-pr.yml @@ -1,4 +1,4 @@ -name: Continues Integration (PR) +name: Continuous Integration (PR) on: pull_request: @@ -7,3 +7,6 @@ jobs: CI: uses: ./.github/workflows/ci.yml + secrets: inherit + with: + slow-tests: true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f4eb3db..22fe0db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,6 +2,11 @@ name: CI on: workflow_call: + inputs: + slow-tests: + type: boolean + required: false + default: false secrets: ALTERNATIVE_GITHUB_TOKEN: required: false @@ -13,6 +18,11 @@ jobs: runs-on: ubuntu-20.04 steps: + - name: Set pytest markers + id: pytest-markers + if: ${{ ! inputs.slow-tests }} + run: echo slow-tests='-m "not slow"' >> "$GITHUB_OUTPUT" + - name: SCM Checkout uses: actions/checkout@v4 with: @@ -21,6 +31,12 @@ jobs: - name: Setup Development Environment uses: ./.github/actions/pytest-plugins-environment - - name: Run Tests of all plugins - run: just test - + - name: Run Tests of All Plugins + run: | + echo "PYTEST_ADDOPTS = $PYTEST_ADDOPTS" + just test + env: + SAAS_HOST: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_HOST }} + SAAS_ACCOUNT_ID: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_ACCOUNT_ID }} + SAAS_PAT: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_PAT }} + PYTEST_ADDOPTS: '-o log_cli=true -o log_cli_level=INFO ${{ steps.pytest-markers.outputs.slow-tests }}' diff --git a/README.md b/README.md new file mode 100644 index 0000000..18e9070 --- /dev/null +++ b/README.md @@ -0,0 +1,44 @@ +# Pytest-Plugins for Exasol + +Welcome to the official repository for Exasol pytest-plugins! + +This collection of plugins is designed to enhance and simplify the testing experience for projects related to Exasol. + +By providing a centralized location for pytest plugins, we aim to foster collaboration, ensure consistency, and improve the quality of testing practices within the organization. + +## Introduction + +[pytest](https://pytest.org) is a powerful testing framework for [python](https://www.python.org), and with the help of these plugins, developers can extend its functionality to better suit the testing requirements of Exasol-related projects. + +Whether you're looking to use database interactions, enhance test reporting, or streamline your testing pipeline, our plugins are here to help. + +## Plugins + +| Plugin | Description | PYPI | +|----------------------|----------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------| +| `pytest-exasol-itde` | Fixture to enable simple usage with Exasol's project [ITDE](https://github.com/exasol/integration-test-docker-environment) | [pytest-exasol-itde](https://pypi.org/project/pytest-exasol-itde/) | +| `pytest-exasol-saas` | Fixture to enable simple usage with Exasol's project [saas-api-python](https://github.com/exasol/saas-api-python/) | [pytest-exasol-saas](https://pypi.org/project/pytest-exasol-saas/) | + + +## Installation + +To ensure you're using the latest features and bug fixes, we recommend installing the plugins directly from PyPI using your preferred package manager. This approach simplifies the process of keeping your testing environment up-to-date. + +For example, to install the `pytest-exasol-itde` plugin, you could use the following command: + + +```shell +pip install pytest-exasol-itde +``` + +To install a specific version of a plugin, simply specify the version number: + +```shell +pip install "pytest-exasol-itde==x.y.z" +``` + +Replace x.y.z with the desired version number. + +## Development + +See [Developer Guide](doc/developer-guide.md). diff --git a/README.rst b/README.rst deleted file mode 100644 index 7a331a1..0000000 --- a/README.rst +++ /dev/null @@ -1,55 +0,0 @@ -.. _pytest-plugins-exasol: - -Pytest-Plugins for Exasol -========================== - -Welcome to the official repository for Exasol pytest-plugins! -This collection of plugins is designed to enhance and simplify the testing experience for projects related to Exasol. -By providing a centralized location for pytest plugins, we aim to foster collaboration, ensure consistency, and improve the quality of testing practices within the organization. - -Introduction ------------- - -`pytest `_ is a powerful testing framework for `python `_, and with the help of these plugins, developers can extend its functionality to better suit the testing requirements of Exasol-related projects. -Whether you're looking to use database interactions, enhance test reporting, or streamline your testing pipeline, our plugins are here to help. - -Plugins -------- - -.. list-table:: - :header-rows: 1 - - * - Plugin - - Description - - PYPI - * - pytest-itde - - fixture to enable simple usage with exasols itde proejct - - TBD - -Installation ------------- - -To ensure you're using the latest features and bug fixes, we recommend installing the plugins directly from PyPI using your preferred package manager. This approach simplifies the process of keeping your testing environment up-to-date. - -For example, to install the ``pytest-itde`` plugin, you could use the following command: - -.. code-block:: bash - - pip install pytest-itde - -To install a specific version of a plugin, simply specify the version number: - -.. code-block:: bash - - pip install "pytest-itde==x.y.z" - -Replace x.y.z with the desired version number. - -Development ------------ - -Before you can start developing in this workspace, please ensure you have the following tools installed either globally or at a workspace level. - -- `Python `_ -- `Just `_ - diff --git a/doc/developer-guide.md b/doc/developer-guide.md new file mode 100644 index 0000000..cbc0d0a --- /dev/null +++ b/doc/developer-guide.md @@ -0,0 +1,23 @@ +# Developer Guide Exasol pytest-plugins + +## Dependencies + +Before you can start developing in this workspace, please ensure you have the following tools installed either globally or at a workspace level. + +* [Python](https://www.python.org) +* [Just](https://github.com/casey/just) + +## Run Tests + +### Slow Tests + +Some of the test cases verify connecting to a SaaS database instance and +execution will take about 20 minutes. + +These test cases are only executed by the following GitHub workflows +* `ci-main.yml` +* `ci-slow.yml` + +Both of these workflows can be run manually, workflow `ci-main.yml` is executed automatically at the 7th day of each month. + +For merging a pull request to branch `main` workflow `ci-slow.yml` needs to be run and terminate successfully. diff --git a/justfile b/justfile index 162ca92..f61c231 100644 --- a/justfile +++ b/justfile @@ -6,11 +6,18 @@ default: # Run tests for one or multiple projects within this respository test +projects=PROJECTS: - #!/usr/bin/env bash - for p in {{projects}}; do - poetry -C ${p}/ install - poetry -C ${p}/ run nox -f ${p}/noxfile.py -s coverage - done + #!/usr/bin/env python3 + import subprocess, sys + rc = 0 + def run(command): + global rc + result = subprocess.run(command.split()) + rc = result.returncode or rc + + for p in "{{projects}}".split(): + run(f"poetry -C {p}/ install") + run(f"poetry -C {p}/ run nox -f {p}/noxfile.py -s coverage") + sys.exit(rc) # Create a release release project version: diff --git a/pytest-itde/pyproject.toml b/pytest-itde/pyproject.toml index 49a105a..38cec2d 100644 --- a/pytest-itde/pyproject.toml +++ b/pytest-itde/pyproject.toml @@ -15,6 +15,10 @@ pyexasol = "^0.25" [tool.poetry.plugins.pytest11] itde = "exasol.pytest_itde" +[tool.pytest.ini_options] +markers = [ + "slow: marks tests as slow (deselect with '-m \"not slow\"')", +] [tool.poetry.group.dev.dependencies] exasol-toolbox = "0.8.0" diff --git a/pytest-saas/README.md b/pytest-saas/README.md index 687dda9..0239c53 100644 --- a/pytest-saas/README.md +++ b/pytest-saas/README.md @@ -15,3 +15,43 @@ To install the pytest-exasol-saas plugin, you can use pip: ```shell pip install pytest-exasol-saas ``` + +## Database Instances + +### Using an Existing Database Instance + +By default the fixtures in pytest-exasol-saas Plugin will create instances of Exasol SaaS database with scope `session`. + +If you want to use an existing instance instead, then you can provide the instance's ID with the command line option `--saas-database-id ` to pytest. + +### Keeping Database Instances After the Test Session + +By default the fixtures in pytest-exasol-saas Plugin will remove the created database instances after the session or in case of errors. + +However, if you provide the command line option `--keep-saas-database` then the pytest-exasol-saas plugin will _keep_ these instances for subsequent inspection or reuse. + +Please note hat long-running instances will cause significant costs. + +### Naming Database Instances + +pytest-exasol-saas Plugin will name the database instances using 3 components + +* **Project Short Tag**: Abbreviation of the current project, see below for different [options for providing the project short tag](#options-for-providing-the-project-short-tag). +* **Timestamp**: Number of seconds since epoc, see [Unix Time](https://en.wikipedia.org/wiki/Unix_time). +* A dash character `-` +* **User Name**: Login name of the current user. + +A database instances might for example have the name `1715155224SAPIPY-run` indicating it was +* created on Wednesday, May 8, 2024 +* in the context of a project with short tag `SAPIPY` +* by a user with login name starting with `run` + +Please note that Exasol SaaS limits the length of database names to 10 characters only. So pytest-exasol-saas plugin will shorten the constructed name to 10 characters max. + +If running your tests on a server for Continuous Integration (CI) then the name of the user might be not very expressive. + +### Options for providing the Project Short Tag + +* In yaml file `error_code_config.yml` in the project's root directory +* CLI option `--project-short-tag ` to pytest +* Environment variable `PROJECT_SHORT_TAG` diff --git a/pytest-saas/doc/changes/unreleased.md b/pytest-saas/doc/changes/unreleased.md index 6078a6c..46ce431 100644 --- a/pytest-saas/doc/changes/unreleased.md +++ b/pytest-saas/doc/changes/unreleased.md @@ -9,3 +9,4 @@ This version introduces the `pytest-exasol-saas` plugin providing pytest functio ## Features * #9: Added sub-project for exasol-saas-api +* #10: Added pytest fixtures for usage in integration tests of external projects diff --git a/pytest-saas/exasol/pytest_saas/__init__.py b/pytest-saas/exasol/pytest_saas/__init__.py new file mode 100644 index 0000000..5527d27 --- /dev/null +++ b/pytest-saas/exasol/pytest_saas/__init__.py @@ -0,0 +1,101 @@ +import os +from pathlib import Path + +import pytest +from exasol.saas.client import openapi +from exasol.saas.client.api_access import ( + OpenApiAccess, + create_saas_client, + timestamp_name, +) + +import exasol.pytest_saas.project_short_tag as pst + + +def pytest_addoption(parser): + parser.addoption( + f"--saas-database-id", + help="""ID of the instance of an existing SaaS database to be + used during the current pytest session instead of creating a + dedicated instance temporarily.""", + ) + parser.addoption( + f"--keep-saas-database", + action="store_true", + default=False, + help="""Keep the SaaS database instance created for the current + pytest session for subsequent inspection or reuse.""", + ) + parser.addoption( + f"--project-short-tag", + help="""Short tag aka. "abbreviation" for your current project. + See docstring in project_short_tag.py for more details. + pytest plugin for exasol-saas-api will include this short tag into + the names of created database instances.""", + ) + + +def _env(var: str) -> str: + result = os.environ.get(var) + if result: + return result + raise RuntimeError(f"Environment variable {var} is empty.") + + +@pytest.fixture(scope="session") +def saas_host() -> str: + return _env("SAAS_HOST") + + +@pytest.fixture(scope="session") +def saas_pat() -> str: + return _env("SAAS_PAT") + + +@pytest.fixture(scope="session") +def saas_account_id() -> str: + return _env("SAAS_ACCOUNT_ID") + + +@pytest.fixture(scope="session") +def project_short_tag(request): + return ( + request.config.getoption("--project-short-tag") + or os.environ.get("PROJECT_SHORT_TAG") + or pst.read_from_yaml(request.config.rootpath) + ) + + +@pytest.fixture(scope="session") +def database_name(project_short_tag): + return timestamp_name(project_short_tag) + + +@pytest.fixture(scope="session") +def api_access(saas_host, saas_pat, saas_account_id) -> OpenApiAccess: + with create_saas_client(saas_host, saas_pat) as client: + yield OpenApiAccess(client, saas_account_id) + + +@pytest.fixture(scope="session") +def saas_database( + request, api_access, database_name +) -> openapi.models.database.Database: + """ + Note: The SaaS instance database returned by this fixture initially + will not be operational. The startup takes about 20 minutes. + """ + db_id = request.config.getoption("--saas-database-id") + if db_id: + yield api_access.get_database(db_id) + return + with api_access.database(database_name) as db: + yield db + + +@pytest.fixture(scope="session") +def operational_saas_database_id(api_access, saas_database) -> str: + db = saas_database + api_access.add_allowed_ip() + api_access.wait_until_running(db.id) + return db.id diff --git a/pytest-saas/exasol/pytest_saas/project_short_tag.py b/pytest-saas/exasol/pytest_saas/project_short_tag.py new file mode 100644 index 0000000..31488fa --- /dev/null +++ b/pytest-saas/exasol/pytest_saas/project_short_tag.py @@ -0,0 +1,28 @@ +""" +A "Project Short Tag" is a short abbreviation for a project. + +The pytest plugin for exasol-saas-api will include this short tag into the +names of created database instances to enable identifying the origin of +potentially long-running database instances in order to avoid unwanted costs. +""" + +from pathlib import Path +import yaml + +FILE = "error_code_config.yml" + + +def read_from_yaml(dir: Path) -> str: + """ + Read project-short-tag from yaml file ``FILE`` in the specified + directory ``dir``. + """ + config_file = dir / FILE + if not config_file.exists(): + return None + with open(config_file, 'r') as file: + ecc = yaml.safe_load(file) + try: + return next(t for t in ecc["error-tags"]) + except Exception as ex: + raise RuntimeError(f"Could not read project short tag from file {config_file}") diff --git a/pytest-saas/poetry.lock b/pytest-saas/poetry.lock index 212cc22..9719dba 100644 --- a/pytest-saas/poetry.lock +++ b/pytest-saas/poetry.lock @@ -1643,4 +1643,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more [metadata] lock-version = "2.0" python-versions = ">=3.8,<4" -content-hash = "592d92bad05a2dda896784a642733f663a92a49895d0a44b7cc16125f2a8cebe" +content-hash = "3942d8c795fbada653e6c77425813beab1392d0c2dce927b73a944091d0a909c" diff --git a/pytest-saas/pyproject.toml b/pytest-saas/pyproject.toml index e0b80c4..3434cc6 100644 --- a/pytest-saas/pyproject.toml +++ b/pytest-saas/pyproject.toml @@ -9,12 +9,16 @@ packages = [{include = "exasol"}] [tool.poetry.dependencies] python = ">=3.8,<4" pytest = ">=7,<9" -# pyexasol = "^0.25" exasol-saas-api = "^0.5.0" +pyyaml = "^6.0.1" [tool.poetry.plugins.pytest11] saas = "exasol.pytest_saas" +[tool.pytest.ini_options] +markers = [ + "slow: marks tests as slow (deselect with '-m \"not slow\"')", +] [tool.poetry.group.dev.dependencies] exasol-toolbox = "0.9.0" diff --git a/pytest-saas/test/integration/pytest_saas_test.py b/pytest-saas/test/integration/pytest_saas_test.py new file mode 100644 index 0000000..ab17aab --- /dev/null +++ b/pytest-saas/test/integration/pytest_saas_test.py @@ -0,0 +1,132 @@ +import os +import re +from inspect import cleandoc +from unittest import mock + +import pytest + +pytest_plugins = "pytester" + + +@pytest.fixture +def make_test_files(): + def make(pytester, files): + pytester.makepyfile(**files) + + return make + + +def _testfile(body): + testname = re.sub(r"^.*def ([^(]+).*", "\\1", body, flags=re.S) + return { testname: cleandoc(body) } + + +def _cli_args(*args): + return args + + +def _env(**kwargs): + return kwargs + + +@pytest.mark.parametrize( + "files,cli_args", + [ + ( _testfile(""" + def test_no_cli_args(request): + assert not request.config.getoption("--keep-saas-database") + assert request.config.getoption("--saas-database-id") is None + """), + _cli_args(), + ), + ( _testfile(""" + import os + def test_cli_args(request): + assert request.config.getoption("--keep-saas-database") + assert "123" == request.config.getoption("--saas-database-id") + assert "PST" == request.config.getoption("--project-short-tag") + """), + _cli_args( + "--keep-saas-database", + "--project-short-tag", "PST", + "--saas-database-id", "123", + ), + ), + ]) +def test_pass_options_via_cli(pytester, make_test_files, files, cli_args): + """ + This test could also be called a unit test and verifies that the CLI + arguments are registered correctly, can be passed to pytest, and are + accessible within external test cases. + """ + make_test_files(pytester, files) + result = pytester.runpytest(*cli_args) + assert result.ret == pytest.ExitCode.OK + + +@pytest.mark.parametrize( + "pst_file, pst_env, pst_cli, expected", + [ + ("F", None, None, "F"), + ("F", "E", None, "E"), + ("F", "E", "C", "C"), + ]) +def test_project_short_tag( + request, + pytester, + pst_file, + pst_env, + pst_cli, + expected, +): + """ + This test sets different values for project short tag in file + error_code_config.yml, cli option --project-short-tag, and environment + variable PROJECT_SHORT_TAG and verifies the precedence. + """ + if pst_file: + pytester.makefile(".yml", **{ + "error_code_config": + cleandoc(f""" + error-tags: + {pst_file}: + highest-index: 0 + """) + }) + pytester.makepyfile(** _testfile( + f""" + def test_project_short_tag(project_short_tag): + assert "{expected}" == project_short_tag + """)) + env = { "PROJECT_SHORT_TAG": pst_env } if pst_env else {} + cli_args = [ "--project-short-tag", pst_cli ] if pst_cli else [] + with mock.patch.dict(os.environ, env): + result = pytester.runpytest(*cli_args) + assert result.ret == pytest.ExitCode.OK + + +def test_id_of_existing_database(request, pytester, capsys): + """ + Use an invalid ID and verify that exasol-saas-api signals an error + because that there is no database with the specified ID. + """ + testname = request.node.name + pytester.makepyfile(** _testfile( f""" + def {testname}(saas_database): + pass + """)) + result = pytester.runpytest("--saas-database-id", "123") + captured = capsys.readouterr() + assert result.ret != pytest.ExitCode.OK + assert "Database not found" in captured.out + + +@pytest.mark.slow +def test_operational_database(request, pytester): + testname = request.node.name + pytester.makepyfile(** _testfile( f""" + def {testname}(operational_saas_database_id): + assert operational_saas_database_id is not None + """)) + result = pytester.runpytest() + assert result.ret == pytest.ExitCode.OK diff --git a/pytest-saas/test/integration/smoke_test.py b/pytest-saas/test/integration/smoke_test.py deleted file mode 100644 index a048e4f..0000000 --- a/pytest-saas/test/integration/smoke_test.py +++ /dev/null @@ -1,2 +0,0 @@ -def test_smoke_integration(): - pass