From cebd3a3ff813df578869e76558f82fd8d7c54a39 Mon Sep 17 00:00:00 2001 From: ckunki Date: Fri, 17 May 2024 09:27:34 +0200 Subject: [PATCH 01/57] #10: Added pytest fixtures for usage in integration tests of external projects --- .gitignore | 3 + pytest-saas/README.md | 57 +++++++++ pytest-saas/doc/changes/changelog.md | 12 ++ pytest-saas/doc/changes/unreleased.md | 12 ++ pytest-saas/exasol/pytest_saas/__init__.py | 100 +++++++++++++++ .../exasol/pytest_saas/project_short_tag.py | 30 +++++ .../test/integration/pytest_saas_test.py | 116 ++++++++++++++++++ 7 files changed, 330 insertions(+) create mode 100644 pytest-saas/README.md create mode 100644 pytest-saas/doc/changes/changelog.md create mode 100644 pytest-saas/doc/changes/unreleased.md create mode 100644 pytest-saas/exasol/pytest_saas/__init__.py create mode 100644 pytest-saas/exasol/pytest_saas/project_short_tag.py create mode 100644 pytest-saas/test/integration/pytest_saas_test.py diff --git a/.gitignore b/.gitignore index 84914bd..c0f0782 100644 --- a/.gitignore +++ b/.gitignore @@ -161,3 +161,6 @@ cython_debug/ .lint.* + +# Emacs +TAGS \ No newline at end of file diff --git a/pytest-saas/README.md b/pytest-saas/README.md new file mode 100644 index 0000000..4e4895b --- /dev/null +++ b/pytest-saas/README.md @@ -0,0 +1,57 @@ +# pytest-saas Plugin + +The `pytest-exasol-saas` plugin is a pytest plugin designed to facilitate the integration +testing of projects using the [exasol-saas-api](https://github.com/exasol/saas-api-python). + +## Features + +* **Integration with exasol-saas-api**: Designed to work closely with the exasol-saas-api. +* **Ease of Use:** Simplifies the configuration and execution of tests by leveraging the pytest framework, making it accessible to developers familiar with pytest conventions. + +## Installation + +To install the pytest-exasol-saas plugin, you can use pip: + +```shell +pip install pytest-exasol-saas +``` + +## Database Instances + +### Using an Existing Database Instance + +By default the fixtures in pytest-exasol-saas Plugin will create instances of Exasol SaaS database with scope `session`. + +If you want to use an existing instance instead, then you can provide the instance's ID with command line option `--saas-database-id ` to pytest. + +### Keeping Database Instances After the Test Session + +By default the fixtures in pytest-exasol-saas Plugin will remove the created database instances after the session or in case of errors. + +However, if you provide command line option `--keep-saas-database` then pytest-exasol-saas Plugin will _keep_ these instances for subsequent inspection or reuse. + +Please note hat long-running instances will cause significant costs. + +### Naming Database Instances + +pytest-exasol-saas Plugin will name the database instances using 3 components + +* **Project Short Tag**: Abbreviation of the current project, see below for different [options for providing the project short tag](#options-for-providing-the-project-short-tag). +* **Timestamp**: Number of seconds since epoc, see [Unix Time](https://en.wikipedia.org/wiki/Unix_time). +* A dash character `-` +* **User Name**: Login name of the current user. + +A database instances might for example have the name `1715155224SAPIPY-run` indicating it was +* created on Wednesday, May 8, 2024 +* in the context of a project with short tag `SAPYPI` +* by a user with login name starting with `run` + +Please note that Exasol SaaS limits the length of database names to 10 characters only. So pytest-exasol-saas plugin will shorten the constructed name to 10 characters max. + +If running your tests on a server for Continuous Integration (CI) then the name of the user might be not very expressive. + +### Options for providing the Project Short Tag + +* In yaml file `error_code_config.yml` in the project's root directory +* CLI option `--project-short-tag ` to pytest +* Environment variable `PROJECT_SHORT_TAG` diff --git a/pytest-saas/doc/changes/changelog.md b/pytest-saas/doc/changes/changelog.md new file mode 100644 index 0000000..c6b9900 --- /dev/null +++ b/pytest-saas/doc/changes/changelog.md @@ -0,0 +1,12 @@ +# Changes + +* [unreleased](unreleased.md) + + +```{toctree} +--- +hidden: +--- +unreleased + +``` \ No newline at end of file diff --git a/pytest-saas/doc/changes/unreleased.md b/pytest-saas/doc/changes/unreleased.md new file mode 100644 index 0000000..efecfa7 --- /dev/null +++ b/pytest-saas/doc/changes/unreleased.md @@ -0,0 +1,12 @@ +# Unreleased + +## Summary + +🚀 Initial Release of the pytest-saas Plugin + +This version introduces the `pytest-saas` plugin providing pytest functionalities for the API to Exasol SaaS instances provided by [exasol-saas-api](https://github.com/exasol/saas-api-python). + +## Features + +* #9: Added sub-project for exasol-saas-api +* #10: Added pytest fixtures for usage in integration tests of external projects diff --git a/pytest-saas/exasol/pytest_saas/__init__.py b/pytest-saas/exasol/pytest_saas/__init__.py new file mode 100644 index 0000000..7840470 --- /dev/null +++ b/pytest-saas/exasol/pytest_saas/__init__.py @@ -0,0 +1,100 @@ +import os +from pathlib import Path + +import pytest +from exasol.saas.client import openapi +from exasol.saas.client.api_access import ( + OpenApiAccess, + create_saas_client, + timestamp_name, +) + +import exasol.pytest_saas.project_short_tag as pst + + +def pytest_addoption(parser): + parser.addoption( + f"--saas-database-id", + help="""ID of the instance of an existing SaaS database to be + used during the current pytest session instead of creating a + dedicated instance temporarily.""", + ) + parser.addoption( + f"--keep-saas-database", + action="store_true", + default=False, + help="""Keep the SaaS database instance created for the current + pytest session for subsequent inspection or reuse.""", + ) + parser.addoption( + f"--project-short-tag", + help="""Short tag aka. "abbreviation" for your current project. + See docstring in project_short_tag.py for more details. + pytest plugin for exasol-saas-api will include this short tag into + the names of created database instances.""", + ) + + +def _env(var: str) -> str: + result = os.environ.get(var) + if result: + return result + raise RuntimeError(f"Environment variable {var} is empty.") + + +@pytest.fixture(scope="session") +def saas_host() -> str: + return _env("SAAS_HOST") + + +@pytest.fixture(scope="session") +def saas_pat() -> str: + return _env("SAAS_PAT") + + +@pytest.fixture(scope="session") +def saas_account_id() -> str: + return _env("SAAS_ACCOUNT_ID") + + +@pytest.fixture(scope="session") +def project_short_tag(request): + return ( + request.config.getoption("--project-short-tag") + or os.environ.get("PROJECT_SHORT_TAG") + or pst.read_from_yaml(request.config.rootpath) + ) + + +@pytest.fixture(scope="session") +def database_name(project_short_tag): + return timestamp_name(project_short_tag) + + +@pytest.fixture(scope="session") +def api_access(saas_host, saas_pat, saas_account_id) -> OpenApiAccess: + with create_saas_client(saas_host, saas_pat) as client: + yield OpenApiAccess(client, saas_account_id) + + +@pytest.fixture(scope="session") +def saas_database( + request, api_access, database_name +) -> openapi.models.database.Database: + """ + Note: The SaaS instance database returned by this fixture initially + will not be operational. The startup takes about 20 minutes. + """ + db_id = request.config.getoption("--saas-database-id") + if db_id: + yield api_access.get_database(db_id) + return + with api_access.database(database_name) as db: + yield db + + +@pytest.fixture(scope="session") +def operational_saas_database_id(api_access, saas_database) -> str: + db = saas_database + api_access.wait_until_running(db.id) + return db.id diff --git a/pytest-saas/exasol/pytest_saas/project_short_tag.py b/pytest-saas/exasol/pytest_saas/project_short_tag.py new file mode 100644 index 0000000..709ea9b --- /dev/null +++ b/pytest-saas/exasol/pytest_saas/project_short_tag.py @@ -0,0 +1,30 @@ +""" +A "Project Short Tag" is a short abbreviation for a project. + +The pytest plugin for exasol-saas-api will include this short tag into the +names of created database instances to enable identifying the origin of +potentially long-running database instances in order to avoid unwanted costs. +""" + +from pathlib import Path + +FILE = "error_code_config.yml" + + +def read_from_yaml(dir: Path) -> str: + """ + Read project-short-tag from yaml file ``FILE`` in the specified + directory ``dir``. + """ + config_file = dir / FILE + if not config_file.exists(): + return None + content = config_file.read_text() + header = False + for line in content.splitlines(): + line = line.strip() + if header: + return line.strip().replace(":", "") + if line.startswith("error-tags:"): + header = True + raise RuntimeError(f"Could not read project short tag from file {config_file}") diff --git a/pytest-saas/test/integration/pytest_saas_test.py b/pytest-saas/test/integration/pytest_saas_test.py new file mode 100644 index 0000000..ff5c7dd --- /dev/null +++ b/pytest-saas/test/integration/pytest_saas_test.py @@ -0,0 +1,116 @@ +import os +import re +from inspect import cleandoc +from unittest import mock + +import pytest + +pytest_plugins = "pytester" + + +@pytest.fixture +def make_test_files(): + def make(pytester, files): + pytester.makepyfile(**files) + + return make + + +def _testfile(body): + testname = re.sub(r"^.*def ([^(]+).*", "\\1", body, flags=re.S) + return { testname: cleandoc(body) } + + +def _cli_args(*args): + return args + + +def _env(**kwargs): + return kwargs + + +@pytest.mark.parametrize( + "files,cli_args", + [ + ( _testfile(""" + def test_no_cli_args(request): + assert not request.config.getoption("--keep-saas-database") + assert request.config.getoption("--saas-database-id") is None + """), + _cli_args(), + ), + ( _testfile(""" + import os + def test_cli_args(request): + assert request.config.getoption("--keep-saas-database") + assert "123" == request.config.getoption("--saas-database-id") + assert "PST" == request.config.getoption("--project-short-tag") + """), + _cli_args( + "--keep-saas-database", + "--project-short-tag", "PST", + "--saas-database-id", "123", + ), + ), + ]) +def test_pass_options_via_cli(pytester, make_test_files, files, cli_args): + make_test_files(pytester, files) + result = pytester.runpytest(*cli_args) + assert result.ret == pytest.ExitCode.OK + + +@pytest.mark.parametrize( + "pst_file, pst_env, pst_cli, expected", + [ + ("F", None, None, "F"), + ("F", "E", None, "E"), + ("F", "E", "C", "C"), + ]) +def test_project_short_tag( + request, + pytester, + pst_file, + pst_env, + pst_cli, + expected, +): + """ + This test sets different values for project short tag in file + error_code_config.yml, cli option --project-short-tag, and environment + variable PROJECT_SHORT_TAG and verifies the precedence. + """ + if pst_file: + pytester.makefile(".yml", **{ + "error_code_config": + cleandoc(f""" + error-tags: + {pst_file}: + highest-index: 0 + """) + }) + pytester.makepyfile(** _testfile( + f""" + def test_project_short_tag(project_short_tag): + assert "{expected}" == project_short_tag + """)) + env = { "PROJECT_SHORT_TAG": pst_env } if pst_env else {} + cli_args = [ "--project-short-tag", pst_cli ] if pst_cli else [] + with mock.patch.dict(os.environ, env): + result = pytester.runpytest(*cli_args) + assert result.ret == pytest.ExitCode.OK + + +def test_id_of_existing_database(request, pytester, capsys): + """ + Use an invalid ID and verify that exasol-saas-api signals an error + because that there is no database with the specified ID. + """ + testname = request.node.name + pytester.makepyfile(** _testfile( f""" + def {testname}(saas_database): + pass + """)) + result = pytester.runpytest("--saas-database-id", "123") + captured = capsys.readouterr() + assert result.ret != pytest.ExitCode.OK + assert "Database not found" in captured.out From 8121b537865bf1ca2043f9e12726cd1cba9aecf5 Mon Sep 17 00:00:00 2001 From: Christoph Kuhnke Date: Fri, 17 May 2024 16:52:44 +0200 Subject: [PATCH 02/57] Apply suggestions from code review Co-authored-by: Torsten Kilias --- pytest-saas/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest-saas/README.md b/pytest-saas/README.md index b1d5528..e576e13 100644 --- a/pytest-saas/README.md +++ b/pytest-saas/README.md @@ -43,7 +43,7 @@ pytest-exasol-saas Plugin will name the database instances using 3 components A database instances might for example have the name `1715155224SAPIPY-run` indicating it was * created on Wednesday, May 8, 2024 -* in the context of a project with short tag `SAPYPI` +* in the context of a project with short tag `SAPIPY` * by a user with login name starting with `run` Please note that Exasol SaaS limits the length of database names to 10 characters only. So pytest-exasol-saas plugin will shorten the constructed name to 10 characters max. From 6190da3fb5ade8010e5cdeb9e31693163b0e35f6 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 11:46:01 +0200 Subject: [PATCH 03/57] Use pyyaml for reading project short tag from error_code_config.yml --- .../exasol/pytest_saas/project_short_tag.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/pytest-saas/exasol/pytest_saas/project_short_tag.py b/pytest-saas/exasol/pytest_saas/project_short_tag.py index 709ea9b..31488fa 100644 --- a/pytest-saas/exasol/pytest_saas/project_short_tag.py +++ b/pytest-saas/exasol/pytest_saas/project_short_tag.py @@ -7,6 +7,7 @@ """ from pathlib import Path +import yaml FILE = "error_code_config.yml" @@ -19,12 +20,9 @@ def read_from_yaml(dir: Path) -> str: config_file = dir / FILE if not config_file.exists(): return None - content = config_file.read_text() - header = False - for line in content.splitlines(): - line = line.strip() - if header: - return line.strip().replace(":", "") - if line.startswith("error-tags:"): - header = True - raise RuntimeError(f"Could not read project short tag from file {config_file}") + with open(config_file, 'r') as file: + ecc = yaml.safe_load(file) + try: + return next(t for t in ecc["error-tags"]) + except Exception as ex: + raise RuntimeError(f"Could not read project short tag from file {config_file}") From 6c76a2c12f9983036210972209b2dc14ba65dd2c Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 11:47:38 +0200 Subject: [PATCH 04/57] Use pyyaml for reading project short tag from error_code_config.yml --- pytest-saas/poetry.lock | 2 +- pytest-saas/pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pytest-saas/poetry.lock b/pytest-saas/poetry.lock index 212cc22..9719dba 100644 --- a/pytest-saas/poetry.lock +++ b/pytest-saas/poetry.lock @@ -1643,4 +1643,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more [metadata] lock-version = "2.0" python-versions = ">=3.8,<4" -content-hash = "592d92bad05a2dda896784a642733f663a92a49895d0a44b7cc16125f2a8cebe" +content-hash = "3942d8c795fbada653e6c77425813beab1392d0c2dce927b73a944091d0a909c" diff --git a/pytest-saas/pyproject.toml b/pytest-saas/pyproject.toml index e0b80c4..cd5ee32 100644 --- a/pytest-saas/pyproject.toml +++ b/pytest-saas/pyproject.toml @@ -9,8 +9,8 @@ packages = [{include = "exasol"}] [tool.poetry.dependencies] python = ">=3.8,<4" pytest = ">=7,<9" -# pyexasol = "^0.25" exasol-saas-api = "^0.5.0" +pyyaml = "^6.0.1" [tool.poetry.plugins.pytest11] saas = "exasol.pytest_saas" From 6f48ebdbf5df8b07466d673e2b70eef501f58a3d Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 13:10:29 +0200 Subject: [PATCH 05/57] Fixed review findings * Added ip-whitelisting to fixture operational_saas_database_id * Verified fixture in test_operational_database --- pytest-saas/exasol/pytest_saas/__init__.py | 1 + pytest-saas/test/integration/pytest_saas_test.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/pytest-saas/exasol/pytest_saas/__init__.py b/pytest-saas/exasol/pytest_saas/__init__.py index 7840470..5527d27 100644 --- a/pytest-saas/exasol/pytest_saas/__init__.py +++ b/pytest-saas/exasol/pytest_saas/__init__.py @@ -96,5 +96,6 @@ def saas_database( @pytest.fixture(scope="session") def operational_saas_database_id(api_access, saas_database) -> str: db = saas_database + api_access.add_allowed_ip() api_access.wait_until_running(db.id) return db.id diff --git a/pytest-saas/test/integration/pytest_saas_test.py b/pytest-saas/test/integration/pytest_saas_test.py index ff5c7dd..0d69718 100644 --- a/pytest-saas/test/integration/pytest_saas_test.py +++ b/pytest-saas/test/integration/pytest_saas_test.py @@ -54,6 +54,11 @@ def test_cli_args(request): ), ]) def test_pass_options_via_cli(pytester, make_test_files, files, cli_args): + """ + This test could also be called a unit test and verifies that the CLI + arguments are registered correctly, can be passed to pytest, and are + accessible within external test cases. + """ make_test_files(pytester, files) result = pytester.runpytest(*cli_args) assert result.ret == pytest.ExitCode.OK @@ -114,3 +119,13 @@ def {testname}(saas_database): captured = capsys.readouterr() assert result.ret != pytest.ExitCode.OK assert "Database not found" in captured.out + + +def test_operational_database(request, pytester): + testname = request.node.name + pytester.makepyfile(** _testfile( f""" + def {testname}(operational_saas_database_id): + assert operational_saas_database_id is not None + """)) + result = pytester.runpytest() + assert result.ret == pytest.ExitCode.OK From bc72a63afc3b2a691ae50c3c7adaa8281271ffe6 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 15:01:29 +0200 Subject: [PATCH 06/57] Abort shellscript in just file on errors --- justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/justfile b/justfile index 162ca92..623475d 100644 --- a/justfile +++ b/justfile @@ -6,7 +6,7 @@ default: # Run tests for one or multiple projects within this respository test +projects=PROJECTS: - #!/usr/bin/env bash + #!/usr/bin/env -S bash -eoE pipefail for p in {{projects}}; do poetry -C ${p}/ install poetry -C ${p}/ run nox -f ${p}/noxfile.py -s coverage From 37198fe7bfef467a494610c638196fb6acfa8fbc Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 15:10:02 +0200 Subject: [PATCH 07/57] Added secrets for integration tests --- .github/workflows/ci-main.yml | 1 + .github/workflows/ci-pr.yml | 1 + .github/workflows/ci.yml | 6 +++++- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-main.yml b/.github/workflows/ci-main.yml index 427dfee..1ac7801 100644 --- a/.github/workflows/ci-main.yml +++ b/.github/workflows/ci-main.yml @@ -13,3 +13,4 @@ jobs: CI: uses: ./.github/workflows/ci.yml + secrets: inherit diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml index e00b9df..34464cf 100644 --- a/.github/workflows/ci-pr.yml +++ b/.github/workflows/ci-pr.yml @@ -7,3 +7,4 @@ jobs: CI: uses: ./.github/workflows/ci.yml + secrets: inherit diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f4eb3db..15c6ffb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,6 +21,10 @@ jobs: - name: Setup Development Environment uses: ./.github/actions/pytest-plugins-environment - - name: Run Tests of all plugins + - name: Run Tests of All Plugins run: just test + env: + SAAS_HOST: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_HOST }} + SAAS_ACCOUNT_ID: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_ACCOUNT_ID }} + SAAS_PAT: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_PAT }} From 1e766467b36d9b122c4e8449e3640ecb98a809ff Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 15:32:32 +0200 Subject: [PATCH 08/57] Replaced bash script in justfile by python --- justfile | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/justfile b/justfile index 623475d..2716b17 100644 --- a/justfile +++ b/justfile @@ -6,11 +6,18 @@ default: # Run tests for one or multiple projects within this respository test +projects=PROJECTS: - #!/usr/bin/env -S bash -eoE pipefail - for p in {{projects}}; do - poetry -C ${p}/ install - poetry -C ${p}/ run nox -f ${p}/noxfile.py -s coverage - done + #!/usr/bin/env python + import subprocess, sys + rc = 0 + def run(command): + global rc + result = subprocess.run(command.split()) + rc = result.returncode or rc + + for p in "{{projects}}".split(): + run(f"poetry -C {p}/ install") + run(f"poetry -C {p}/ run nox -f {p}/noxfile.py -s coverage") + sys.exit(rc) # Create a release release project version: From b68d9f2b59ad2a3fc52bf61183bf1cfb9770f462 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 15:53:06 +0200 Subject: [PATCH 09/57] Separated long running test cases [run-slow-tests] --- .github/workflows/ci.yml | 5 +++++ pytest-saas/pyproject.toml | 4 ++++ pytest-saas/test/integration/pytest_saas_test.py | 1 + 3 files changed, 10 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 15c6ffb..425e9b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,10 +21,15 @@ jobs: - name: Setup Development Environment uses: ./.github/actions/pytest-plugins-environment + - name: Evaluate Whether to Run Slow Tests + if: "! contains(github.event.head_commit.message, '[run-slow-tests]')" + run: echo 'RUN_SLOW_TESTS=-m "not slow"' >> $GITHUB_ENV + - name: Run Tests of All Plugins run: just test env: SAAS_HOST: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_HOST }} SAAS_ACCOUNT_ID: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_ACCOUNT_ID }} SAAS_PAT: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_PAT }} + PYTEST_ADDOPTS: '-o log_cli=true -o log_cli_level=INFO ${{ env.RUN_SLOW_TESTS }}' diff --git a/pytest-saas/pyproject.toml b/pytest-saas/pyproject.toml index cd5ee32..3434cc6 100644 --- a/pytest-saas/pyproject.toml +++ b/pytest-saas/pyproject.toml @@ -15,6 +15,10 @@ pyyaml = "^6.0.1" [tool.poetry.plugins.pytest11] saas = "exasol.pytest_saas" +[tool.pytest.ini_options] +markers = [ + "slow: marks tests as slow (deselect with '-m \"not slow\"')", +] [tool.poetry.group.dev.dependencies] exasol-toolbox = "0.9.0" diff --git a/pytest-saas/test/integration/pytest_saas_test.py b/pytest-saas/test/integration/pytest_saas_test.py index 0d69718..ab17aab 100644 --- a/pytest-saas/test/integration/pytest_saas_test.py +++ b/pytest-saas/test/integration/pytest_saas_test.py @@ -121,6 +121,7 @@ def {testname}(saas_database): assert "Database not found" in captured.out +@pytest.mark.slow def test_operational_database(request, pytester): testname = request.node.name pytester.makepyfile(** _testfile( f""" From cc9ecbb8a6278d14d5a30590403efa80c7757c16 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 16:01:16 +0200 Subject: [PATCH 10/57] Separated long running test cases [run-slow-tests] --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 425e9b5..979ea55 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,10 +26,11 @@ jobs: run: echo 'RUN_SLOW_TESTS=-m "not slow"' >> $GITHUB_ENV - name: Run Tests of All Plugins - run: just test + run: | + echo "PYTEST_ADDOPTS = $PYTEST_ADDOPTS" + just test env: SAAS_HOST: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_HOST }} SAAS_ACCOUNT_ID: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_ACCOUNT_ID }} SAAS_PAT: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_PAT }} PYTEST_ADDOPTS: '-o log_cli=true -o log_cli_level=INFO ${{ env.RUN_SLOW_TESTS }}' - From 549f2017a1ef0d793422255b46e7988ff1da6b23 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 16:02:36 +0200 Subject: [PATCH 11/57] Separated long running test cases --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 979ea55..236c43c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,7 @@ jobs: - name: Run Tests of All Plugins run: | - echo "PYTEST_ADDOPTS = $PYTEST_ADDOPTS" + echo "PYTEST_ADDOPTS = $PYTEST_ADDOPTS" just test env: SAAS_HOST: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_HOST }} From 24a82d92ab12362146d0e11f73f24e86b5fdc8f0 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 16:06:39 +0200 Subject: [PATCH 12/57] Fixed if statement in github worflow [run-slow-tests] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 236c43c..bb72615 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: uses: ./.github/actions/pytest-plugins-environment - name: Evaluate Whether to Run Slow Tests - if: "! contains(github.event.head_commit.message, '[run-slow-tests]')" + if: ${{ ! contains(github.event.head_commit.message, '[run-slow-tests]') }} run: echo 'RUN_SLOW_TESTS=-m "not slow"' >> $GITHUB_ENV - name: Run Tests of All Plugins From e83d749c49b880261deaefbadfc0fb6f71284a0e Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 16:08:52 +0200 Subject: [PATCH 13/57] Fixed if statement in github worflow [run-slow-tests] --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb72615..c45afff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,9 @@ jobs: - name: Evaluate Whether to Run Slow Tests if: ${{ ! contains(github.event.head_commit.message, '[run-slow-tests]') }} - run: echo 'RUN_SLOW_TESTS=-m "not slow"' >> $GITHUB_ENV + run: | + echo ${{ github.event.head_commit.message }} + echo 'RUN_SLOW_TESTS=-m "not slow"' >> $GITHUB_ENV - name: Run Tests of All Plugins run: | From 5f89ffe5800c619844576cd89f5cb85438dd707d Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 16:13:44 +0200 Subject: [PATCH 14/57] Experiment event --- .github/workflows/ci.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c45afff..d3e8194 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,15 +21,18 @@ jobs: - name: Setup Development Environment uses: ./.github/actions/pytest-plugins-environment + - name: Show event + run: echo ${{ github.event.workflow_call.head_commit.message }} + - name: Evaluate Whether to Run Slow Tests if: ${{ ! contains(github.event.head_commit.message, '[run-slow-tests]') }} run: | - echo ${{ github.event.head_commit.message }} + echo ${{ github.event.workflow_call.head_commit.message }} echo 'RUN_SLOW_TESTS=-m "not slow"' >> $GITHUB_ENV - name: Run Tests of All Plugins run: | - echo "PYTEST_ADDOPTS = $PYTEST_ADDOPTS" + echo "PYTEST_ADDOPTS = $PYTEST_ADDOPTS" just test env: SAAS_HOST: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_HOST }} From bdd3eb54eaeddc53440f02231da0efc24226b416 Mon Sep 17 00:00:00 2001 From: Christoph Kuhnke Date: Wed, 22 May 2024 16:17:25 +0200 Subject: [PATCH 15/57] Update ci.yml --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d3e8194..3bdded7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,6 +13,9 @@ jobs: runs-on: ubuntu-20.04 steps: + - name: Show event + run: echo ${{ github.event }} + - name: SCM Checkout uses: actions/checkout@v4 with: @@ -21,9 +24,6 @@ jobs: - name: Setup Development Environment uses: ./.github/actions/pytest-plugins-environment - - name: Show event - run: echo ${{ github.event.workflow_call.head_commit.message }} - - name: Evaluate Whether to Run Slow Tests if: ${{ ! contains(github.event.head_commit.message, '[run-slow-tests]') }} run: | From f20e225b35036902522172f7b801ff36f346cd73 Mon Sep 17 00:00:00 2001 From: Christoph Kuhnke Date: Wed, 22 May 2024 16:18:27 +0200 Subject: [PATCH 16/57] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3bdded7..34e96f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: steps: - name: Show event - run: echo ${{ github.event }} + run: echo ${{ github.event.workflow_call }} - name: SCM Checkout uses: actions/checkout@v4 From 70eb7430c637eccb90de1439575661eb495cd0a1 Mon Sep 17 00:00:00 2001 From: Christoph Kuhnke Date: Wed, 22 May 2024 16:18:56 +0200 Subject: [PATCH 17/57] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 34e96f8..66e7ed8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: steps: - name: Show event - run: echo ${{ github.event.workflow_call }} + run: echo ${{ github.event.workflow_run }} - name: SCM Checkout uses: actions/checkout@v4 From 7d81626d8093f69e3c4e4132b62403321ed67af4 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 16:26:39 +0200 Subject: [PATCH 18/57] move commit message to input --- .github/workflows/ci-main.yml | 2 ++ .github/workflows/ci.yml | 8 +++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-main.yml b/.github/workflows/ci-main.yml index 1ac7801..1beaf97 100644 --- a/.github/workflows/ci-main.yml +++ b/.github/workflows/ci-main.yml @@ -14,3 +14,5 @@ jobs: CI: uses: ./.github/workflows/ci.yml secrets: inherit + with: + commit-message: ${{ github.event.head_commit.message }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 66e7ed8..9cf76fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,6 +2,9 @@ name: CI on: workflow_call: + inputs: + commit-message: + type: string secrets: ALTERNATIVE_GITHUB_TOKEN: required: false @@ -24,10 +27,13 @@ jobs: - name: Setup Development Environment uses: ./.github/actions/pytest-plugins-environment + - name: Show Commit Message of calling workflow + run: echo ${{ inputs.commit-message }} + - name: Evaluate Whether to Run Slow Tests if: ${{ ! contains(github.event.head_commit.message, '[run-slow-tests]') }} run: | - echo ${{ github.event.workflow_call.head_commit.message }} + echo ${{ github.event.head_commit.message }} echo 'RUN_SLOW_TESTS=-m "not slow"' >> $GITHUB_ENV - name: Run Tests of All Plugins From 184ea402c4e6de6b02d3967545f7fec0dfe3a8cd Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 16:28:13 +0200 Subject: [PATCH 19/57] move commit message to input --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9cf76fe..8cadbc5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,8 +16,8 @@ jobs: runs-on: ubuntu-20.04 steps: - - name: Show event - run: echo ${{ github.event.workflow_run }} + - name: Show Commit Message + run: echo ${{ inputs.commit-message }} - name: SCM Checkout uses: actions/checkout@v4 From 64db9093610f738f1f7c2acbe644416426ed2366 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 16:30:55 +0200 Subject: [PATCH 20/57] experiment 1 --- .github/workflows/ci-main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci-main.yml b/.github/workflows/ci-main.yml index 1beaf97..5e73036 100644 --- a/.github/workflows/ci-main.yml +++ b/.github/workflows/ci-main.yml @@ -10,6 +10,8 @@ on: - cron: "0 0 1/7 * *" jobs: + show-commit-message: + run: echo ${{ github.event.head_commit.message }} CI: uses: ./.github/workflows/ci.yml From 9c88ed6003060cbb5a176c93c1befdb7438c1e8d Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 16:33:11 +0200 Subject: [PATCH 21/57] experiment 2 --- .github/workflows/ci-main.yml | 5 ++++- .github/workflows/ci-pr.yml | 7 +++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-main.yml b/.github/workflows/ci-main.yml index 5e73036..0085ff0 100644 --- a/.github/workflows/ci-main.yml +++ b/.github/workflows/ci-main.yml @@ -10,7 +10,10 @@ on: - cron: "0 0 1/7 * *" jobs: - show-commit-message: + + show-commit-message-1: + name: show commit message 1 + runs-on: ubuntu-20.04 run: echo ${{ github.event.head_commit.message }} CI: diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml index 34464cf..a9fbbe7 100644 --- a/.github/workflows/ci-pr.yml +++ b/.github/workflows/ci-pr.yml @@ -5,6 +5,13 @@ on: jobs: + show-commit-message-1: + name: show commit message 1 + runs-on: ubuntu-20.04 + run: echo ${{ github.event.head_commit.message }} + CI: uses: ./.github/workflows/ci.yml secrets: inherit + with: + commit-message: ${{ github.event.head_commit.message }} From ada2d32aec3ac58e6455ee0eb587dd72df90cac6 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 16:35:37 +0200 Subject: [PATCH 22/57] experiment 3 --- .github/workflows/ci-pr.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml index a9fbbe7..e6deeef 100644 --- a/.github/workflows/ci-pr.yml +++ b/.github/workflows/ci-pr.yml @@ -5,10 +5,13 @@ on: jobs: - show-commit-message-1: - name: show commit message 1 + show-commit-message-ci-pr: + name: show commit message ci-pr runs-on: ubuntu-20.04 - run: echo ${{ github.event.head_commit.message }} + + steps: + - name: Show CM 1 + run: echo ${{ github.event.head_commit.message }} CI: uses: ./.github/workflows/ci.yml From 5a847d577471cbf6512216997de3ccfadb860ff8 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 16:36:43 +0200 Subject: [PATCH 23/57] experiment 4 --- .github/workflows/ci-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml index e6deeef..9913e2f 100644 --- a/.github/workflows/ci-pr.yml +++ b/.github/workflows/ci-pr.yml @@ -11,7 +11,7 @@ jobs: steps: - name: Show CM 1 - run: echo ${{ github.event.head_commit.message }} + run: echo ${{ github.event.pull_request.head_commit.message }} CI: uses: ./.github/workflows/ci.yml From e1bf00effadb7aa442e79505bd636758670158e5 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 16:47:00 +0200 Subject: [PATCH 24/57] experiment 5 --- .github/workflows/ci-pr.yml | 12 +++++++++--- .github/workflows/ci.yml | 10 ++++++++-- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml index 9913e2f..619fca3 100644 --- a/.github/workflows/ci-pr.yml +++ b/.github/workflows/ci-pr.yml @@ -5,13 +5,19 @@ on: jobs: - show-commit-message-ci-pr: + examine-commit-message-ci-pr: name: show commit message ci-pr runs-on: ubuntu-20.04 steps: - - name: Show CM 1 - run: echo ${{ github.event.pull_request.head_commit.message }} + - name: Examine Commit Message in ci-pr + run: | + if (git log -1 --pretty=format:"%s" | grep "[run-slow-tests]"); then + echo 'RUN_SLOW_TESTS=-m "not slow"' >> $GITHUB_ENV + fi + + - name: Verify setting of env var in ci-pr + run: echo ${{ env.RUN_SLOW_TESTS }} CI: uses: ./.github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8cadbc5..4f9b3d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,8 +16,14 @@ jobs: runs-on: ubuntu-20.04 steps: - - name: Show Commit Message - run: echo ${{ inputs.commit-message }} + - name: Examine Commit Message + run: | + if (git log -1 --pretty=format:"%s" | grep "[run-slow-tests]"); then + echo 'RUN_SLOW_TESTS=-m "not slow"' >> $GITHUB_ENV + fi + + - name: Verify setting of env var + run: echo ${{ env.RUN_SLOW_TESTS }} - name: SCM Checkout uses: actions/checkout@v4 From e3519aa44dfa9ff243e831c1322f942afcbc491a Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 16:48:26 +0200 Subject: [PATCH 25/57] experiment 6 [run-slow-tests] --- .github/workflows/ci-pr.yml | 2 +- .github/workflows/ci.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml index 619fca3..fd28555 100644 --- a/.github/workflows/ci-pr.yml +++ b/.github/workflows/ci-pr.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Examine Commit Message in ci-pr run: | - if (git log -1 --pretty=format:"%s" | grep "[run-slow-tests]"); then + if (git log -1 --pretty=format:"%s" | grep "\[run-slow-tests\]"); then echo 'RUN_SLOW_TESTS=-m "not slow"' >> $GITHUB_ENV fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4f9b3d0..a1676b4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: steps: - name: Examine Commit Message run: | - if (git log -1 --pretty=format:"%s" | grep "[run-slow-tests]"); then + if (git log -1 --pretty=format:"%s" | grep "\[run-slow-tests\]"); then echo 'RUN_SLOW_TESTS=-m "not slow"' >> $GITHUB_ENV fi From 18cdc244d58ce898ad476a63ac7d27bf45439389 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 16:49:56 +0200 Subject: [PATCH 26/57] experiment 7 [run-slow-tests] --- .github/workflows/ci-pr.yml | 5 +++++ .github/workflows/ci.yml | 10 +++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml index fd28555..356f63d 100644 --- a/.github/workflows/ci-pr.yml +++ b/.github/workflows/ci-pr.yml @@ -10,6 +10,11 @@ jobs: runs-on: ubuntu-20.04 steps: + - name: SCM Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Examine Commit Message in ci-pr run: | if (git log -1 --pretty=format:"%s" | grep "\[run-slow-tests\]"); then diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a1676b4..a16948b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,6 +16,11 @@ jobs: runs-on: ubuntu-20.04 steps: + - name: SCM Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Examine Commit Message run: | if (git log -1 --pretty=format:"%s" | grep "\[run-slow-tests\]"); then @@ -25,11 +30,6 @@ jobs: - name: Verify setting of env var run: echo ${{ env.RUN_SLOW_TESTS }} - - name: SCM Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Setup Development Environment uses: ./.github/actions/pytest-plugins-environment From f38ee40d7e77c6a60b0a5b9bfea402f1393e6257 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 17:00:29 +0200 Subject: [PATCH 27/57] experiment 8 [run-slow-tests] --- .github/workflows/ci-pr.yml | 2 +- .github/workflows/ci.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml index 356f63d..ed960f6 100644 --- a/.github/workflows/ci-pr.yml +++ b/.github/workflows/ci-pr.yml @@ -17,7 +17,7 @@ jobs: - name: Examine Commit Message in ci-pr run: | - if (git log -1 --pretty=format:"%s" | grep "\[run-slow-tests\]"); then + if (git log -1 --pretty=format:"%b" | grep "\[run-slow-tests\]"); then echo 'RUN_SLOW_TESTS=-m "not slow"' >> $GITHUB_ENV fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a16948b..dec0cfd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,7 @@ jobs: - name: Examine Commit Message run: | - if (git log -1 --pretty=format:"%s" | grep "\[run-slow-tests\]"); then + if (git log -1 --pretty=format:"%b" | grep "\[run-slow-tests\]"); then echo 'RUN_SLOW_TESTS=-m "not slow"' >> $GITHUB_ENV fi From 71e8b4cb9961a1eeded12c43df9ea1670e50c76b Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 17:01:51 +0200 Subject: [PATCH 28/57] experiment 9 [run-slow-tests] --- .github/workflows/ci-pr.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml index ed960f6..28d73c2 100644 --- a/.github/workflows/ci-pr.yml +++ b/.github/workflows/ci-pr.yml @@ -17,6 +17,7 @@ jobs: - name: Examine Commit Message in ci-pr run: | + git log -1 --pretty=format:"%b" if (git log -1 --pretty=format:"%b" | grep "\[run-slow-tests\]"); then echo 'RUN_SLOW_TESTS=-m "not slow"' >> $GITHUB_ENV fi From 615b4a43c2adb9f767537680507f34365a85339b Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 17:05:21 +0200 Subject: [PATCH 29/57] experiment 10 [run-slow-tests] --- .github/workflows/ci-pr.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml index 28d73c2..5314bb8 100644 --- a/.github/workflows/ci-pr.yml +++ b/.github/workflows/ci-pr.yml @@ -19,11 +19,11 @@ jobs: run: | git log -1 --pretty=format:"%b" if (git log -1 --pretty=format:"%b" | grep "\[run-slow-tests\]"); then - echo 'RUN_SLOW_TESTS=-m "not slow"' >> $GITHUB_ENV + export RUN_SLOW_TESTS='-m "not slow"' fi - name: Verify setting of env var in ci-pr - run: echo ${{ env.RUN_SLOW_TESTS }} + run: echo "RUN_SLOW_TESTS = $RUN_SLOW_TESTS" CI: uses: ./.github/workflows/ci.yml From 08c0a65b992bdcd36c71590e1dc14d4c1cd53ff1 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 17:16:15 +0200 Subject: [PATCH 30/57] experiment 11 [run-slow-tests] --- .github/workflows/ci-pr.yml | 7 +++++-- .github/workflows/ci.yml | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml index 5314bb8..77c3b3c 100644 --- a/.github/workflows/ci-pr.yml +++ b/.github/workflows/ci-pr.yml @@ -17,13 +17,16 @@ jobs: - name: Examine Commit Message in ci-pr run: | + export A=5 git log -1 --pretty=format:"%b" - if (git log -1 --pretty=format:"%b" | grep "\[run-slow-tests\]"); then + if ! (git log -1 --pretty=format:"%b" | grep "\[run-slow-tests\]"); then export RUN_SLOW_TESTS='-m "not slow"' fi - name: Verify setting of env var in ci-pr - run: echo "RUN_SLOW_TESTS = $RUN_SLOW_TESTS" + run: | + echo "A = $A + echo "RUN_SLOW_TESTS = $RUN_SLOW_TESTS" CI: uses: ./.github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dec0cfd..ab773fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,7 @@ jobs: - name: Examine Commit Message run: | - if (git log -1 --pretty=format:"%b" | grep "\[run-slow-tests\]"); then + if ! (git log -1 --pretty=format:"%b" | grep "\[run-slow-tests\]"); then echo 'RUN_SLOW_TESTS=-m "not slow"' >> $GITHUB_ENV fi From e9d1e7d14da7514b008383111a986f991a3fbb32 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 17:17:03 +0200 Subject: [PATCH 31/57] experiment 12 [run-slow-tests] --- .github/workflows/ci-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml index 77c3b3c..26b2c60 100644 --- a/.github/workflows/ci-pr.yml +++ b/.github/workflows/ci-pr.yml @@ -25,7 +25,7 @@ jobs: - name: Verify setting of env var in ci-pr run: | - echo "A = $A + echo "A = $A" echo "RUN_SLOW_TESTS = $RUN_SLOW_TESTS" CI: From 68b4c9865fd54891853119adf1e2bab73800ee32 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 17:21:11 +0200 Subject: [PATCH 32/57] experiment 13 [run-slow-tests] --- .github/workflows/ci-pr.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml index 26b2c60..b4f388d 100644 --- a/.github/workflows/ci-pr.yml +++ b/.github/workflows/ci-pr.yml @@ -17,16 +17,17 @@ jobs: - name: Examine Commit Message in ci-pr run: | - export A=5 + echo 5 >> "$GITHUB_OUTPUT" + git status git log -1 --pretty=format:"%b" if ! (git log -1 --pretty=format:"%b" | grep "\[run-slow-tests\]"); then - export RUN_SLOW_TESTS='-m "not slow"' + export RUN_SLOW_TESTS='-m "not slow"' >> "$GITHUB_OUTPUT" fi - name: Verify setting of env var in ci-pr run: | - echo "A = $A" - echo "RUN_SLOW_TESTS = $RUN_SLOW_TESTS" + echo "GITHUB_OUTPUT 1 = ${{ neegs.examine-commit-message-ci-pr.outputs.output1}} + echo "GITHUB_OUTPUT 2 = ${{ neegs.examine-commit-message-ci-pr.outputs.output2}} CI: uses: ./.github/workflows/ci.yml From 7edd50e685d840cbcf851e6e54d3ffd62089115c Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 17:28:41 +0200 Subject: [PATCH 33/57] experiment 14 [run-slow-tests] --- .github/workflows/ci-pr.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml index b4f388d..cacf9d3 100644 --- a/.github/workflows/ci-pr.yml +++ b/.github/workflows/ci-pr.yml @@ -15,9 +15,10 @@ jobs: with: fetch-depth: 0 - - name: Examine Commit Message in ci-pr + - id: examine + name: Examine Commit Message in ci-pr run: | - echo 5 >> "$GITHUB_OUTPUT" + echo A=5 >> "$GITHUB_OUTPUT" git status git log -1 --pretty=format:"%b" if ! (git log -1 --pretty=format:"%b" | grep "\[run-slow-tests\]"); then @@ -26,8 +27,8 @@ jobs: - name: Verify setting of env var in ci-pr run: | - echo "GITHUB_OUTPUT 1 = ${{ neegs.examine-commit-message-ci-pr.outputs.output1}} - echo "GITHUB_OUTPUT 2 = ${{ neegs.examine-commit-message-ci-pr.outputs.output2}} + echo "OUTPUT 1 = ${{ steps.examine.outputs.A }}" + echo "OUTPUT 2 = ${{ steps.examine.outputs.RUN_SLOW_TESTS }}" CI: uses: ./.github/workflows/ci.yml From 9c5964daa10c3a8558f863b5467f53c21b6170f0 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 17:33:08 +0200 Subject: [PATCH 34/57] experiment 15 [run-slow-tests] --- .github/workflows/ci-main.yml | 5 ----- .github/workflows/ci-pr.yml | 6 +++--- .github/workflows/ci.yml | 3 --- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci-main.yml b/.github/workflows/ci-main.yml index 0085ff0..1beaf97 100644 --- a/.github/workflows/ci-main.yml +++ b/.github/workflows/ci-main.yml @@ -11,11 +11,6 @@ on: jobs: - show-commit-message-1: - name: show commit message 1 - runs-on: ubuntu-20.04 - run: echo ${{ github.event.head_commit.message }} - CI: uses: ./.github/workflows/ci.yml secrets: inherit diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml index cacf9d3..bd567f6 100644 --- a/.github/workflows/ci-pr.yml +++ b/.github/workflows/ci-pr.yml @@ -19,9 +19,9 @@ jobs: name: Examine Commit Message in ci-pr run: | echo A=5 >> "$GITHUB_OUTPUT" - git status - git log -1 --pretty=format:"%b" - if ! (git log -1 --pretty=format:"%b" | grep "\[run-slow-tests\]"); then + /usr/bin/git status + /usr/bin/git log -1 --pretty=format:"%b" + if ! (/usr/bin/git log -1 --pretty=format:"%b" | grep "\[run-slow-tests\]"); then export RUN_SLOW_TESTS='-m "not slow"' >> "$GITHUB_OUTPUT" fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ab773fe..499e683 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,9 +2,6 @@ name: CI on: workflow_call: - inputs: - commit-message: - type: string secrets: ALTERNATIVE_GITHUB_TOKEN: required: false From 292b7f6188f4246c6ef8744c140b7f5ea8775cac Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 17:33:54 +0200 Subject: [PATCH 35/57] experiment 16 [run-slow-tests] --- .github/workflows/ci-pr.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml index bd567f6..eeae031 100644 --- a/.github/workflows/ci-pr.yml +++ b/.github/workflows/ci-pr.yml @@ -33,5 +33,3 @@ jobs: CI: uses: ./.github/workflows/ci.yml secrets: inherit - with: - commit-message: ${{ github.event.head_commit.message }} From 4df961a775ad312cd23540a9103c7342c1ea0aaa Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 17:37:11 +0200 Subject: [PATCH 36/57] experiment 17 [run-slow-tests] --- .github/workflows/ci-pr.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml index eeae031..362baaf 100644 --- a/.github/workflows/ci-pr.yml +++ b/.github/workflows/ci-pr.yml @@ -20,7 +20,8 @@ jobs: run: | echo A=5 >> "$GITHUB_OUTPUT" /usr/bin/git status - /usr/bin/git log -1 --pretty=format:"%b" + /usr/bin/git log | cat + # /usr/bin/git log -1 --pretty=format:"%b" if ! (/usr/bin/git log -1 --pretty=format:"%b" | grep "\[run-slow-tests\]"); then export RUN_SLOW_TESTS='-m "not slow"' >> "$GITHUB_OUTPUT" fi From 00dcb7904be95c19b49241d9374d6f3ca58061a6 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 17:47:39 +0200 Subject: [PATCH 37/57] experiment 18 [run-slow-tests] --- .github/workflows/ci-pr.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml index 362baaf..c73916a 100644 --- a/.github/workflows/ci-pr.yml +++ b/.github/workflows/ci-pr.yml @@ -19,10 +19,9 @@ jobs: name: Examine Commit Message in ci-pr run: | echo A=5 >> "$GITHUB_OUTPUT" - /usr/bin/git status - /usr/bin/git log | cat - # /usr/bin/git log -1 --pretty=format:"%b" - if ! (/usr/bin/git log -1 --pretty=format:"%b" | grep "\[run-slow-tests\]"); then + git status + git log -1 HEAD~1 --format="%b" + if ! (git log -1 HEAD~1 --format="%b" | grep "\[run-slow-tests\]"); then export RUN_SLOW_TESTS='-m "not slow"' >> "$GITHUB_OUTPUT" fi From e1b57b452afd0eb79dd58b5ee46071b19251466f Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 17:50:03 +0200 Subject: [PATCH 38/57] experiment 19 [run-slow-tests] --- .github/workflows/ci-pr.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml index c73916a..c96acb3 100644 --- a/.github/workflows/ci-pr.yml +++ b/.github/workflows/ci-pr.yml @@ -21,6 +21,7 @@ jobs: echo A=5 >> "$GITHUB_OUTPUT" git status git log -1 HEAD~1 --format="%b" + git log -2 if ! (git log -1 HEAD~1 --format="%b" | grep "\[run-slow-tests\]"); then export RUN_SLOW_TESTS='-m "not slow"' >> "$GITHUB_OUTPUT" fi From ec5240fb45e040396e1d9853b87576e6097c8975 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 17:51:29 +0200 Subject: [PATCH 39/57] experiment 20 [run-slow-tests] --- .github/workflows/ci-pr.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml index c96acb3..d5bed0b 100644 --- a/.github/workflows/ci-pr.yml +++ b/.github/workflows/ci-pr.yml @@ -19,11 +19,8 @@ jobs: name: Examine Commit Message in ci-pr run: | echo A=5 >> "$GITHUB_OUTPUT" - git status - git log -1 HEAD~1 --format="%b" - git log -2 - if ! (git log -1 HEAD~1 --format="%b" | grep "\[run-slow-tests\]"); then - export RUN_SLOW_TESTS='-m "not slow"' >> "$GITHUB_OUTPUT" + if ! (git log -2 --format="%b" | grep "\[run-slow-tests\]"); then + echo 'RUN_SLOW_TESTS='-m "not slow"' >> "$GITHUB_OUTPUT" fi - name: Verify setting of env var in ci-pr From e676eef259f06ac15876adb7b3e62cc52a1593ce Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 17:52:43 +0200 Subject: [PATCH 40/57] experiment 21 [run-slow-tests] --- .github/workflows/ci-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml index d5bed0b..527546b 100644 --- a/.github/workflows/ci-pr.yml +++ b/.github/workflows/ci-pr.yml @@ -20,7 +20,7 @@ jobs: run: | echo A=5 >> "$GITHUB_OUTPUT" if ! (git log -2 --format="%b" | grep "\[run-slow-tests\]"); then - echo 'RUN_SLOW_TESTS='-m "not slow"' >> "$GITHUB_OUTPUT" + echo 'RUN_SLOW_TESTS=\'-m "not slow"\'' >> "$GITHUB_OUTPUT" fi - name: Verify setting of env var in ci-pr From 191d7ee96cb376daa90cf121998d0fad41dc1d1e Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 17:53:26 +0200 Subject: [PATCH 41/57] experiment 22 [run-slow-tests] --- .github/workflows/ci-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml index 527546b..e362cdb 100644 --- a/.github/workflows/ci-pr.yml +++ b/.github/workflows/ci-pr.yml @@ -20,7 +20,7 @@ jobs: run: | echo A=5 >> "$GITHUB_OUTPUT" if ! (git log -2 --format="%b" | grep "\[run-slow-tests\]"); then - echo 'RUN_SLOW_TESTS=\'-m "not slow"\'' >> "$GITHUB_OUTPUT" + echo RUN_SLOW_TESTS='-m "not slow"' >> "$GITHUB_OUTPUT" fi - name: Verify setting of env var in ci-pr From 164ba41516d4fd0673ed76233b1501f1f80ed976 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 17:58:02 +0200 Subject: [PATCH 42/57] experiment 23 [run-slow-tests] --- .github/workflows/ci.yml | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 499e683..2157b87 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,27 +18,20 @@ jobs: with: fetch-depth: 0 - - name: Examine Commit Message + - id: git-log + name: Examine Commit Message run: | - if ! (git log -1 --pretty=format:"%b" | grep "\[run-slow-tests\]"); then - echo 'RUN_SLOW_TESTS=-m "not slow"' >> $GITHUB_ENV + git log -2 + if ! (git log -2 --format="%b" | grep "\[run-slow-tests\]"); then + echo RUN_SLOW_TESTS='-m "not slow"' >> "$GITHUB_OUTPUT" fi - - name: Verify setting of env var - run: echo ${{ env.RUN_SLOW_TESTS }} + - name: Verify output of step git-log + run: echo "RUN_SLOW_TESTS = ${{ steps.git-log.outputs.RUN_SLOW_TESTS }}" - name: Setup Development Environment uses: ./.github/actions/pytest-plugins-environment - - name: Show Commit Message of calling workflow - run: echo ${{ inputs.commit-message }} - - - name: Evaluate Whether to Run Slow Tests - if: ${{ ! contains(github.event.head_commit.message, '[run-slow-tests]') }} - run: | - echo ${{ github.event.head_commit.message }} - echo 'RUN_SLOW_TESTS=-m "not slow"' >> $GITHUB_ENV - - name: Run Tests of All Plugins run: | echo "PYTEST_ADDOPTS = $PYTEST_ADDOPTS" @@ -47,4 +40,4 @@ jobs: SAAS_HOST: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_HOST }} SAAS_ACCOUNT_ID: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_ACCOUNT_ID }} SAAS_PAT: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_PAT }} - PYTEST_ADDOPTS: '-o log_cli=true -o log_cli_level=INFO ${{ env.RUN_SLOW_TESTS }}' + PYTEST_ADDOPTS: '-o log_cli=true -o log_cli_level=INFO ${{ steps.git-log.outputs.RUN_SLOW_TESTS }}' From 997f4d5a5d4d89935fe01f65d5d3227b06fcef50 Mon Sep 17 00:00:00 2001 From: Christoph Kuhnke Date: Wed, 22 May 2024 18:03:15 +0200 Subject: [PATCH 43/57] Apply suggestions from code review Co-authored-by: Torsten Kilias --- justfile | 2 +- pytest-saas/README.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/justfile b/justfile index 2716b17..f61c231 100644 --- a/justfile +++ b/justfile @@ -6,7 +6,7 @@ default: # Run tests for one or multiple projects within this respository test +projects=PROJECTS: - #!/usr/bin/env python + #!/usr/bin/env python3 import subprocess, sys rc = 0 def run(command): diff --git a/pytest-saas/README.md b/pytest-saas/README.md index e576e13..0239c53 100644 --- a/pytest-saas/README.md +++ b/pytest-saas/README.md @@ -22,13 +22,13 @@ pip install pytest-exasol-saas By default the fixtures in pytest-exasol-saas Plugin will create instances of Exasol SaaS database with scope `session`. -If you want to use an existing instance instead, then you can provide the instance's ID with command line option `--saas-database-id ` to pytest. +If you want to use an existing instance instead, then you can provide the instance's ID with the command line option `--saas-database-id ` to pytest. ### Keeping Database Instances After the Test Session By default the fixtures in pytest-exasol-saas Plugin will remove the created database instances after the session or in case of errors. -However, if you provide command line option `--keep-saas-database` then pytest-exasol-saas Plugin will _keep_ these instances for subsequent inspection or reuse. +However, if you provide the command line option `--keep-saas-database` then the pytest-exasol-saas plugin will _keep_ these instances for subsequent inspection or reuse. Please note hat long-running instances will cause significant costs. From 9df4d150741678fcf40a3d28bf6890022779f3f7 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 18:03:34 +0200 Subject: [PATCH 44/57] clean up --- .github/workflows/ci-pr.yml | 23 ----------------------- .github/workflows/ci.yml | 11 ++++------- 2 files changed, 4 insertions(+), 30 deletions(-) diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml index e362cdb..34464cf 100644 --- a/.github/workflows/ci-pr.yml +++ b/.github/workflows/ci-pr.yml @@ -5,29 +5,6 @@ on: jobs: - examine-commit-message-ci-pr: - name: show commit message ci-pr - runs-on: ubuntu-20.04 - - steps: - - name: SCM Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - id: examine - name: Examine Commit Message in ci-pr - run: | - echo A=5 >> "$GITHUB_OUTPUT" - if ! (git log -2 --format="%b" | grep "\[run-slow-tests\]"); then - echo RUN_SLOW_TESTS='-m "not slow"' >> "$GITHUB_OUTPUT" - fi - - - name: Verify setting of env var in ci-pr - run: | - echo "OUTPUT 1 = ${{ steps.examine.outputs.A }}" - echo "OUTPUT 2 = ${{ steps.examine.outputs.RUN_SLOW_TESTS }}" - CI: uses: ./.github/workflows/ci.yml secrets: inherit diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2157b87..2d73407 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,17 +18,14 @@ jobs: with: fetch-depth: 0 - - id: git-log - name: Examine Commit Message + - name: Examine Git Log + id: git-log run: | git log -2 if ! (git log -2 --format="%b" | grep "\[run-slow-tests\]"); then - echo RUN_SLOW_TESTS='-m "not slow"' >> "$GITHUB_OUTPUT" + echo SLOW_TESTS='-m "not slow"' >> "$GITHUB_OUTPUT" fi - - name: Verify output of step git-log - run: echo "RUN_SLOW_TESTS = ${{ steps.git-log.outputs.RUN_SLOW_TESTS }}" - - name: Setup Development Environment uses: ./.github/actions/pytest-plugins-environment @@ -40,4 +37,4 @@ jobs: SAAS_HOST: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_HOST }} SAAS_ACCOUNT_ID: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_ACCOUNT_ID }} SAAS_PAT: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_PAT }} - PYTEST_ADDOPTS: '-o log_cli=true -o log_cli_level=INFO ${{ steps.git-log.outputs.RUN_SLOW_TESTS }}' + PYTEST_ADDOPTS: '-o log_cli=true -o log_cli_level=INFO ${{ steps.git-log.outputs.SLOW_TESTS }}' From 6125dcdebcd2f8088d6ed0d1e75ef3fe79813893 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 18:13:23 +0200 Subject: [PATCH 45/57] Added developer instructions --- README.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.rst b/README.rst index 7a331a1..1f205f9 100644 --- a/README.rst +++ b/README.rst @@ -53,3 +53,12 @@ Before you can start developing in this workspace, please ensure you have the fo - `Python `_ - `Just `_ +Run Tests +--------- + +Slow Tests +^^^^^^^^^^ + +Some of the test cases verify connecting to a SaaS database instance and execution will take about 20 minutes. + +These test cases are disabled by default and will only be executed when the commit message contains the string `[run-slow-tests]`. From ff1d2b1dff57852e4c56502efde4b622a20c8679 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 22 May 2024 18:15:20 +0200 Subject: [PATCH 46/57] Added pytest marker "slow" to pytest-itde, too --- pytest-itde/pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pytest-itde/pyproject.toml b/pytest-itde/pyproject.toml index 49a105a..38cec2d 100644 --- a/pytest-itde/pyproject.toml +++ b/pytest-itde/pyproject.toml @@ -15,6 +15,10 @@ pyexasol = "^0.25" [tool.poetry.plugins.pytest11] itde = "exasol.pytest_itde" +[tool.pytest.ini_options] +markers = [ + "slow: marks tests as slow (deselect with '-m \"not slow\"')", +] [tool.poetry.group.dev.dependencies] exasol-toolbox = "0.8.0" From 58e8c6b33136fb0b8a853b8abdda0ec4cbd86ad4 Mon Sep 17 00:00:00 2001 From: ckunki Date: Thu, 23 May 2024 09:47:33 +0200 Subject: [PATCH 47/57] May 23rd workflow experiment 1 --- .github/workflows/ci-main.yml | 4 +-- .github/workflows/ci-pr.yml | 2 +- .github/workflows/ci-slow.yml | 12 ++++++++ .github/workflows/ci.yml | 57 +++++++++++++++++++++-------------- 4 files changed, 50 insertions(+), 25 deletions(-) create mode 100644 .github/workflows/ci-slow.yml diff --git a/.github/workflows/ci-main.yml b/.github/workflows/ci-main.yml index 1beaf97..01efa10 100644 --- a/.github/workflows/ci-main.yml +++ b/.github/workflows/ci-main.yml @@ -1,4 +1,4 @@ -name: Continues Integration (Master) +name: Continuous Integration (Main) on: workflow_dispatch: @@ -15,4 +15,4 @@ jobs: uses: ./.github/workflows/ci.yml secrets: inherit with: - commit-message: ${{ github.event.head_commit.message }} + slow-tests: true diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml index 34464cf..021bf64 100644 --- a/.github/workflows/ci-pr.yml +++ b/.github/workflows/ci-pr.yml @@ -1,4 +1,4 @@ -name: Continues Integration (PR) +name: Continuous Integration (PR) on: pull_request: diff --git a/.github/workflows/ci-slow.yml b/.github/workflows/ci-slow.yml new file mode 100644 index 0000000..faa4d87 --- /dev/null +++ b/.github/workflows/ci-slow.yml @@ -0,0 +1,12 @@ +name: Continuous Integration (incl. Slow Tests) + +on: + workflow_dispatch: + +jobs: + + ci-slow: + uses: ./.github/workflows/ci.yml + secrets: inherit + with: + slow-tests: true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2d73407..390d8e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,6 +2,9 @@ name: CI on: workflow_call: + inputs: + slow-tests: + type: string secrets: ALTERNATIVE_GITHUB_TOKEN: required: false @@ -13,28 +16,38 @@ jobs: runs-on: ubuntu-20.04 steps: - - name: SCM Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 + - name: Set pytest markers + id: pytest-markers + if: ${{ inputs.slow-tests }} + run: echo slow-tests='-m "not slow"' >> "$GITHUB_OUTPUT" - - name: Examine Git Log - id: git-log + - name: Examine Inputs run: | - git log -2 - if ! (git log -2 --format="%b" | grep "\[run-slow-tests\]"); then - echo SLOW_TESTS='-m "not slow"' >> "$GITHUB_OUTPUT" - fi + echo "input slow tests = ${{ inputs.slow-tests }}" + echo "pytest-markers = ${{ pytest-markers.outputs.slow-tests }}" - - name: Setup Development Environment - uses: ./.github/actions/pytest-plugins-environment - - - name: Run Tests of All Plugins - run: | - echo "PYTEST_ADDOPTS = $PYTEST_ADDOPTS" - just test - env: - SAAS_HOST: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_HOST }} - SAAS_ACCOUNT_ID: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_ACCOUNT_ID }} - SAAS_PAT: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_PAT }} - PYTEST_ADDOPTS: '-o log_cli=true -o log_cli_level=INFO ${{ steps.git-log.outputs.SLOW_TESTS }}' +# - name: SCM Checkout +# uses: actions/checkout@v4 +# with: +# fetch-depth: 0 +# +# - name: Examine Git Log +# id: git-log +# run: | +# git log -2 +# if ! (git log -2 --format="%b" | grep "\[run-slow-tests\]"); then +# echo SLOW_TESTS='-m "not slow"' >> "$GITHUB_OUTPUT" +# fi +# +# - name: Setup Development Environment +# uses: ./.github/actions/pytest-plugins-environment +# +# - name: Run Tests of All Plugins +# run: | +# echo "PYTEST_ADDOPTS = $PYTEST_ADDOPTS" +# just test +# env: +# SAAS_HOST: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_HOST }} +# SAAS_ACCOUNT_ID: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_ACCOUNT_ID }} +# SAAS_PAT: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_PAT }} +# PYTEST_ADDOPTS: '-o log_cli=true -o log_cli_level=INFO ${{ steps.git-log.outputs.SLOW_TESTS }}' From 533fc159100fcd823b5e91af5750c98215f489d9 Mon Sep 17 00:00:00 2001 From: ckunki Date: Thu, 23 May 2024 09:49:41 +0200 Subject: [PATCH 48/57] May 23rd workflow experiment 2 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 390d8e8..b3f3755 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: - name: Examine Inputs run: | echo "input slow tests = ${{ inputs.slow-tests }}" - echo "pytest-markers = ${{ pytest-markers.outputs.slow-tests }}" + echo "pytest-markers = ${{ steps.pytest-markers.outputs.slow-tests }}" # - name: SCM Checkout # uses: actions/checkout@v4 From 449033226f28600f6363a9681a4539c03f47b8e7 Mon Sep 17 00:00:00 2001 From: ckunki Date: Thu, 23 May 2024 09:54:06 +0200 Subject: [PATCH 49/57] May 23rd workflow experiment 3 --- .github/workflows/ci.yml | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b3f3755..856181a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,23 +25,18 @@ jobs: run: | echo "input slow tests = ${{ inputs.slow-tests }}" echo "pytest-markers = ${{ steps.pytest-markers.outputs.slow-tests }}" + echo "env PYTEST_ADDOPTS = $PYTEST_ADDOPTS" + env: + PYTEST_ADDOPTS: '-o log_cli=true -o log_cli_level=INFO ${{ steps.pytest-markers.outputs.slow-tests }}' # - name: SCM Checkout # uses: actions/checkout@v4 # with: # fetch-depth: 0 -# -# - name: Examine Git Log -# id: git-log -# run: | -# git log -2 -# if ! (git log -2 --format="%b" | grep "\[run-slow-tests\]"); then -# echo SLOW_TESTS='-m "not slow"' >> "$GITHUB_OUTPUT" -# fi -# +# # - name: Setup Development Environment # uses: ./.github/actions/pytest-plugins-environment -# +# # - name: Run Tests of All Plugins # run: | # echo "PYTEST_ADDOPTS = $PYTEST_ADDOPTS" @@ -50,4 +45,4 @@ jobs: # SAAS_HOST: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_HOST }} # SAAS_ACCOUNT_ID: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_ACCOUNT_ID }} # SAAS_PAT: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_PAT }} -# PYTEST_ADDOPTS: '-o log_cli=true -o log_cli_level=INFO ${{ steps.git-log.outputs.SLOW_TESTS }}' +# PYTEST_ADDOPTS: '-o log_cli=true -o log_cli_level=INFO ${{ steps.pytest-markers.outputs.slow-tests }}' From 013645bef777e467568edc28539f412325150118 Mon Sep 17 00:00:00 2001 From: ckunki Date: Thu, 23 May 2024 09:55:10 +0200 Subject: [PATCH 50/57] May 23rd workflow experiment 4 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 856181a..35f445a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: steps: - name: Set pytest markers id: pytest-markers - if: ${{ inputs.slow-tests }} + if: ${{ ! inputs.slow-tests }} run: echo slow-tests='-m "not slow"' >> "$GITHUB_OUTPUT" - name: Examine Inputs From f677941f0b64dee677c4579725a911bebf27af72 Mon Sep 17 00:00:00 2001 From: ckunki Date: Thu, 23 May 2024 09:58:25 +0200 Subject: [PATCH 51/57] May 23rd workflow experiment 5 --- .github/workflows/ci.yml | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 35f445a..c16fc4b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,8 @@ on: workflow_call: inputs: slow-tests: - type: string + required: false + default: 'false' secrets: ALTERNATIVE_GITHUB_TOKEN: required: false @@ -18,7 +19,7 @@ jobs: steps: - name: Set pytest markers id: pytest-markers - if: ${{ ! inputs.slow-tests }} + if: ${{ inputs.slow-tests == 'false' }} run: echo slow-tests='-m "not slow"' >> "$GITHUB_OUTPUT" - name: Examine Inputs @@ -29,20 +30,20 @@ jobs: env: PYTEST_ADDOPTS: '-o log_cli=true -o log_cli_level=INFO ${{ steps.pytest-markers.outputs.slow-tests }}' -# - name: SCM Checkout -# uses: actions/checkout@v4 -# with: -# fetch-depth: 0 -# -# - name: Setup Development Environment -# uses: ./.github/actions/pytest-plugins-environment -# -# - name: Run Tests of All Plugins -# run: | -# echo "PYTEST_ADDOPTS = $PYTEST_ADDOPTS" -# just test -# env: -# SAAS_HOST: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_HOST }} -# SAAS_ACCOUNT_ID: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_ACCOUNT_ID }} -# SAAS_PAT: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_PAT }} -# PYTEST_ADDOPTS: '-o log_cli=true -o log_cli_level=INFO ${{ steps.pytest-markers.outputs.slow-tests }}' + - name: SCM Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Development Environment + uses: ./.github/actions/pytest-plugins-environment + + - name: Run Tests of All Plugins + run: | + echo "PYTEST_ADDOPTS = $PYTEST_ADDOPTS" + just test + env: + SAAS_HOST: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_HOST }} + SAAS_ACCOUNT_ID: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_ACCOUNT_ID }} + SAAS_PAT: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_PAT }} + PYTEST_ADDOPTS: '-o log_cli=true -o log_cli_level=INFO ${{ steps.pytest-markers.outputs.slow-tests }}' From a846e91c240ecb029fd1590aee7fd5cfcff7dc73 Mon Sep 17 00:00:00 2001 From: ckunki Date: Thu, 23 May 2024 09:59:47 +0200 Subject: [PATCH 52/57] May 23rd workflow experiment 6 --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c16fc4b..bf10223 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,8 +4,9 @@ on: workflow_call: inputs: slow-tests: + type: bool required: false - default: 'false' + default: false secrets: ALTERNATIVE_GITHUB_TOKEN: required: false @@ -19,7 +20,7 @@ jobs: steps: - name: Set pytest markers id: pytest-markers - if: ${{ inputs.slow-tests == 'false' }} + if: ${{ ! inputs.slow-tests }} run: echo slow-tests='-m "not slow"' >> "$GITHUB_OUTPUT" - name: Examine Inputs From 3a770205a844ed84b7a0e0a81f795cb147de80f1 Mon Sep 17 00:00:00 2001 From: ckunki Date: Thu, 23 May 2024 10:01:28 +0200 Subject: [PATCH 53/57] May 23rd workflow experiment 7 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bf10223..0f1b40f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,7 @@ on: workflow_call: inputs: slow-tests: - type: bool + type: boolean required: false default: false secrets: From 28efea4bbfa6ebe06b14f769ab6d66403c7276ae Mon Sep 17 00:00:00 2001 From: ckunki Date: Thu, 23 May 2024 10:14:40 +0200 Subject: [PATCH 54/57] Changed workflows to use input instead of commit messages --- .github/workflows/ci.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f1b40f..22fe0db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,14 +23,6 @@ jobs: if: ${{ ! inputs.slow-tests }} run: echo slow-tests='-m "not slow"' >> "$GITHUB_OUTPUT" - - name: Examine Inputs - run: | - echo "input slow tests = ${{ inputs.slow-tests }}" - echo "pytest-markers = ${{ steps.pytest-markers.outputs.slow-tests }}" - echo "env PYTEST_ADDOPTS = $PYTEST_ADDOPTS" - env: - PYTEST_ADDOPTS: '-o log_cli=true -o log_cli_level=INFO ${{ steps.pytest-markers.outputs.slow-tests }}' - - name: SCM Checkout uses: actions/checkout@v4 with: From b8d654b42573b9bb9c384500db684b5cca3001d8 Mon Sep 17 00:00:00 2001 From: ckunki Date: Thu, 23 May 2024 10:27:35 +0200 Subject: [PATCH 55/57] Updated README file --- README.md | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++ README.rst | 64 ------------------------------------------------------ 2 files changed, 62 insertions(+), 64 deletions(-) create mode 100644 README.md delete mode 100644 README.rst diff --git a/README.md b/README.md new file mode 100644 index 0000000..22a4e26 --- /dev/null +++ b/README.md @@ -0,0 +1,62 @@ +# Pytest-Plugins for Exasol + +Welcome to the official repository for Exasol pytest-plugins! + +This collection of plugins is designed to enhance and simplify the testing experience for projects related to Exasol. + +By providing a centralized location for pytest plugins, we aim to foster collaboration, ensure consistency, and improve the quality of testing practices within the organization. + +## Introduction + +[pytest](https://pytest.org) is a powerful testing framework for [python](https://www.python.org), and with the help of these plugins, developers can extend its functionality to better suit the testing requirements of Exasol-related projects. + +Whether you're looking to use database interactions, enhance test reporting, or streamline your testing pipeline, our plugins are here to help. + +## Plugins + +| Plugin | Description | PYPI | +|---------------|----------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------| +| `pytest-itde` | Fixture to enable simple usage with Exasol's project [ITDE](https://github.com/exasol/integration-test-docker-environment) | [pytest-exasol-itde](https://pypi.org/project/pytest-exasol-itde/) | +| `pytest-saas` | Fixture to enable simple usage with Exasol's project [saas-api-python](https://github.com/exasol/saas-api-python/) | [pytest-exasol-saas](https://pypi.org/project/pytest-exasol-saas/) | + + +## Installation + +To ensure you're using the latest features and bug fixes, we recommend installing the plugins directly from PyPI using your preferred package manager. This approach simplifies the process of keeping your testing environment up-to-date. + +For example, to install the `pytest-itde` plugin, you could use the following command: + + +```shell +pip install pytest-itde +``` + +To install a specific version of a plugin, simply specify the version number: + +```shell +pip install "pytest-itde==x.y.z" +``` + +Replace x.y.z with the desired version number. + +## Development + +Before you can start developing in this workspace, please ensure you have the following tools installed either globally or at a workspace level. + +* [Python](https://www.python.org) +* [Just](https://github.com/casey/just) + +## Run Tests + +### Slow Tests + +Some of the test cases verify connecting to a SaaS database instance and +execution will take about 20 minutes. + +These test cases are only executed by the following GitHub workflows +* `ci-main.yml` +* `ci-slow.yml` + +Both of these workflows can be run manually, workflow `ci-main.yml` is executed automatically at the 7th day of each month. + +For merging a pull request to branch `main` workflow `ci-slow.yml` needs to be run and terminate successfully. diff --git a/README.rst b/README.rst deleted file mode 100644 index 1f205f9..0000000 --- a/README.rst +++ /dev/null @@ -1,64 +0,0 @@ -.. _pytest-plugins-exasol: - -Pytest-Plugins for Exasol -========================== - -Welcome to the official repository for Exasol pytest-plugins! -This collection of plugins is designed to enhance and simplify the testing experience for projects related to Exasol. -By providing a centralized location for pytest plugins, we aim to foster collaboration, ensure consistency, and improve the quality of testing practices within the organization. - -Introduction ------------- - -`pytest `_ is a powerful testing framework for `python `_, and with the help of these plugins, developers can extend its functionality to better suit the testing requirements of Exasol-related projects. -Whether you're looking to use database interactions, enhance test reporting, or streamline your testing pipeline, our plugins are here to help. - -Plugins -------- - -.. list-table:: - :header-rows: 1 - - * - Plugin - - Description - - PYPI - * - pytest-itde - - fixture to enable simple usage with exasols itde proejct - - TBD - -Installation ------------- - -To ensure you're using the latest features and bug fixes, we recommend installing the plugins directly from PyPI using your preferred package manager. This approach simplifies the process of keeping your testing environment up-to-date. - -For example, to install the ``pytest-itde`` plugin, you could use the following command: - -.. code-block:: bash - - pip install pytest-itde - -To install a specific version of a plugin, simply specify the version number: - -.. code-block:: bash - - pip install "pytest-itde==x.y.z" - -Replace x.y.z with the desired version number. - -Development ------------ - -Before you can start developing in this workspace, please ensure you have the following tools installed either globally or at a workspace level. - -- `Python `_ -- `Just `_ - -Run Tests ---------- - -Slow Tests -^^^^^^^^^^ - -Some of the test cases verify connecting to a SaaS database instance and execution will take about 20 minutes. - -These test cases are disabled by default and will only be executed when the commit message contains the string `[run-slow-tests]`. From 64566088dd8b8936e50e00e6392366cbcebcf709 Mon Sep 17 00:00:00 2001 From: ckunki Date: Mon, 27 May 2024 13:21:31 +0200 Subject: [PATCH 56/57] Reverted separation of slow tests slow tests are now always executed --- .github/workflows/ci-pr.yml | 2 ++ .github/workflows/ci-slow.yml | 12 ------------ 2 files changed, 2 insertions(+), 12 deletions(-) delete mode 100644 .github/workflows/ci-slow.yml diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml index 021bf64..d22b714 100644 --- a/.github/workflows/ci-pr.yml +++ b/.github/workflows/ci-pr.yml @@ -8,3 +8,5 @@ jobs: CI: uses: ./.github/workflows/ci.yml secrets: inherit + with: + slow-tests: true diff --git a/.github/workflows/ci-slow.yml b/.github/workflows/ci-slow.yml deleted file mode 100644 index faa4d87..0000000 --- a/.github/workflows/ci-slow.yml +++ /dev/null @@ -1,12 +0,0 @@ -name: Continuous Integration (incl. Slow Tests) - -on: - workflow_dispatch: - -jobs: - - ci-slow: - uses: ./.github/workflows/ci.yml - secrets: inherit - with: - slow-tests: true From 99bca1559129a5f67c07ff4f49b3aeabbab59aba Mon Sep 17 00:00:00 2001 From: ckunki Date: Mon, 27 May 2024 13:45:53 +0200 Subject: [PATCH 57/57] Updated README move developer instructions to Developer guide. --- README.md | 34 ++++++++-------------------------- doc/developer-guide.md | 23 +++++++++++++++++++++++ 2 files changed, 31 insertions(+), 26 deletions(-) create mode 100644 doc/developer-guide.md diff --git a/README.md b/README.md index 22a4e26..18e9070 100644 --- a/README.md +++ b/README.md @@ -14,49 +14,31 @@ Whether you're looking to use database interactions, enhance test reporting, or ## Plugins -| Plugin | Description | PYPI | -|---------------|----------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------| -| `pytest-itde` | Fixture to enable simple usage with Exasol's project [ITDE](https://github.com/exasol/integration-test-docker-environment) | [pytest-exasol-itde](https://pypi.org/project/pytest-exasol-itde/) | -| `pytest-saas` | Fixture to enable simple usage with Exasol's project [saas-api-python](https://github.com/exasol/saas-api-python/) | [pytest-exasol-saas](https://pypi.org/project/pytest-exasol-saas/) | +| Plugin | Description | PYPI | +|----------------------|----------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------| +| `pytest-exasol-itde` | Fixture to enable simple usage with Exasol's project [ITDE](https://github.com/exasol/integration-test-docker-environment) | [pytest-exasol-itde](https://pypi.org/project/pytest-exasol-itde/) | +| `pytest-exasol-saas` | Fixture to enable simple usage with Exasol's project [saas-api-python](https://github.com/exasol/saas-api-python/) | [pytest-exasol-saas](https://pypi.org/project/pytest-exasol-saas/) | ## Installation To ensure you're using the latest features and bug fixes, we recommend installing the plugins directly from PyPI using your preferred package manager. This approach simplifies the process of keeping your testing environment up-to-date. -For example, to install the `pytest-itde` plugin, you could use the following command: +For example, to install the `pytest-exasol-itde` plugin, you could use the following command: ```shell -pip install pytest-itde +pip install pytest-exasol-itde ``` To install a specific version of a plugin, simply specify the version number: ```shell -pip install "pytest-itde==x.y.z" +pip install "pytest-exasol-itde==x.y.z" ``` Replace x.y.z with the desired version number. ## Development -Before you can start developing in this workspace, please ensure you have the following tools installed either globally or at a workspace level. - -* [Python](https://www.python.org) -* [Just](https://github.com/casey/just) - -## Run Tests - -### Slow Tests - -Some of the test cases verify connecting to a SaaS database instance and -execution will take about 20 minutes. - -These test cases are only executed by the following GitHub workflows -* `ci-main.yml` -* `ci-slow.yml` - -Both of these workflows can be run manually, workflow `ci-main.yml` is executed automatically at the 7th day of each month. - -For merging a pull request to branch `main` workflow `ci-slow.yml` needs to be run and terminate successfully. +See [Developer Guide](doc/developer-guide.md). diff --git a/doc/developer-guide.md b/doc/developer-guide.md new file mode 100644 index 0000000..cbc0d0a --- /dev/null +++ b/doc/developer-guide.md @@ -0,0 +1,23 @@ +# Developer Guide Exasol pytest-plugins + +## Dependencies + +Before you can start developing in this workspace, please ensure you have the following tools installed either globally or at a workspace level. + +* [Python](https://www.python.org) +* [Just](https://github.com/casey/just) + +## Run Tests + +### Slow Tests + +Some of the test cases verify connecting to a SaaS database instance and +execution will take about 20 minutes. + +These test cases are only executed by the following GitHub workflows +* `ci-main.yml` +* `ci-slow.yml` + +Both of these workflows can be run manually, workflow `ci-main.yml` is executed automatically at the 7th day of each month. + +For merging a pull request to branch `main` workflow `ci-slow.yml` needs to be run and terminate successfully.