From 9377c5f856fb163823214c48b6a98b639d9747c4 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 11 Dec 2024 15:50:49 -0500 Subject: [PATCH] add test groups to troubleshoot where the memory issue on ubuntu --- hatch.toml | 36 ++++------------------ pyproject.toml | 8 +++++ tests/functional/conftest.py | 60 ++++++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 30 deletions(-) create mode 100644 tests/functional/conftest.py diff --git a/hatch.toml b/hatch.toml index dc218aed6..5ad6c4075 100644 --- a/hatch.toml +++ b/hatch.toml @@ -28,36 +28,12 @@ setup = "pre-commit install" code-quality = "pre-commit run --all-files" unit-tests = "python -m pytest {args:tests/unit}" integration-tests = [ - "- python -m pytest tests/functional/adapter/catalog_tests {args}", - "- python -m pytest tests/functional/adapter/column_types {args}", - "- python -m pytest tests/functional/adapter/custom_schema_tests {args}", - "- python -m pytest tests/functional/adapter/dbt_clone {args}", - "- python -m pytest tests/functional/adapter/dbt_show {args}", - "- python -m pytest tests/functional/adapter/empty {args}", - "- python -m pytest tests/functional/adapter/incremental {args}", - "- python -m pytest tests/functional/adapter/list_relations_tests {args}", - "- python -m pytest tests/functional/adapter/python_model_tests {args}", - "- python -m pytest tests/functional/adapter/query_comment_tests {args}", - "- python -m pytest tests/functional/adapter/simple_copy {args}", - "- python -m pytest tests/functional/adapter/simple_seed {args}", - "- python -m pytest tests/functional/adapter/statement_test {args}", - "- python -m pytest tests/functional/adapter/unit_testing {args}", - "- python -m pytest tests/functional/adapter/utils {args}", - "- python -m pytest tests/functional/adapter/test_aliases.py {args}", - "- python -m pytest tests/functional/adapter/test_anonymous_usage_stats.py {args}", - "- python -m pytest tests/functional/adapter/test_basic.py {args}", - "- python -m pytest tests/functional/adapter/test_caching.py {args}", - "- python -m pytest tests/functional/adapter/test_changing_relation_type.py {args}", - "- python -m pytest tests/functional/adapter/test_concurrency.py {args}", - "- python -m pytest tests/functional/adapter/test_constraints.py {args}", - "- python -m pytest tests/functional/adapter/test_emphemeral.py {args}", - "- python -m pytest tests/functional/adapter/test_get_last_relation_modified.py {args}", - "- python -m pytest tests/functional/adapter/test_grants.py {args}", - "- python -m pytest tests/functional/adapter/test_incremental_microbatch.py {args}", - "- python -m pytest tests/functional/adapter/test_persist_docs.py {args}", - "- python -m pytest tests/functional/adapter/test_python_model.py {args}", - "- python -m pytest tests/functional/adapter/test_simple_snapshot.py {args}", - "- python -m pytest tests/functional/adapter/test_timestamps.py {args}", + '- python -m pytest -m "group_1" {args:tests/functional}', + '- python -m pytest -m "group_2" {args:tests/functional}', + '- python -m pytest -m "group_3" {args:tests/functional}', + '- python -m pytest -m "group_4" {args:tests/functional}', + '- python -m pytest -m "group_5" {args:tests/functional}', + '- python -m pytest -m "not group_1 and not group_2 and not group_3 and not group_4 and not group_5" {args:tests/functional}', ] docker-dev = [ "docker build -f docker/dev.Dockerfile -t dbt-snowflake-dev .", diff --git a/pyproject.toml b/pyproject.toml index 568aa3533..4f1e2b9c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,3 +46,11 @@ addopts = "-v --color=yes -n auto" filterwarnings = [ "ignore:datetime.datetime.utcnow:DeprecationWarning", ] +markers = [ + "group_1", + "group_2", + "group_3", + "group_4", + "group_5", + "group_6", +] diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py new file mode 100644 index 000000000..70cb2901c --- /dev/null +++ b/tests/functional/conftest.py @@ -0,0 +1,60 @@ +import pathlib + +import pytest + + +TEST_GROUPS = { + "catalog_tests": "group_1", + "column_types": "group_1", + "custom_schema_tests": "group_1", + "dbt_clone": "group_1", + "dbt_show": "group_1", + "empty": "group_1", + "incremental": "group_2", + "list_relations_tests": "group_2", + "python_model_tests": "group_2", + "query_comment_tests": "group_2", + "simple_copy": "group_2", + "simple_seed": "group_2", + "statement_test": "group_3", + "store_test_failures_tests": "group_3", + "unit_testing": "group_3", + "utils": "group_3", + "test_aliases.py": "group_3", + "test_anonymous_usage_stats.py": "group_3", + "test_basic.py": "group_3", + "test_caching.py": "group_4", + "test_changing_relation_type.py": "group_4", + "test_concurrency.py": "group_4", + "test_constraints.py": "group_4", + "test_ephemeral.py": "group_4", + "test_get_last_relation_modified.py": "group_4", + "test_grants.py": "group_5", + "test_incremental_microbatch.py": "group_5", + "test_persist_docs.py": "group_5", + "test_python_model.py": "group_5", + "test_simple_snapshot.py": "group_5", + "test_timestamps.py": "group_5", + "auth_tests": "group_6", + "generic_test_tests": "group_6", + "iceberg": "group_6", + "override_database": "group_6", + "query_tag": "group_6", + "redact_log_values": "group_6", + "relation_tests": "group_6", + "snowflake_view_dependency": "group_6", + "warehouse_test": "group_6", + "test_isolated_begin_commit.py": "group_6", +} + + +def pytest_collection_modifyitems(config, items): + test_root = pathlib.Path(config.rootdir) / "tests" / "functional" + for item in items: + test_path = pathlib.Path(item.fspath).relative_to(test_root) + test_module = test_path.parts[0] + if test_module == "adapter": + test_module = test_path.parts[1] + if mark_name := TEST_GROUPS.get(test_module): + mark = getattr(pytest.mark, mark_name) + item.add_marker(mark)