From fd1ab4fab5676f21a74f6234a860714cadd514d0 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 16 May 2024 14:55:06 -0400 Subject: [PATCH] create and document performance tests --- .../list_relations_tests/list_relations.py} | 75 +++++++------------ .../list_relations_tests/test_show_objects.py | 48 ++++++++++++ 2 files changed, 77 insertions(+), 46 deletions(-) rename tests/{functional/test_show_objects.py => performance/list_relations_tests/list_relations.py} (57%) create mode 100644 tests/performance/list_relations_tests/test_show_objects.py diff --git a/tests/functional/test_show_objects.py b/tests/performance/list_relations_tests/list_relations.py similarity index 57% rename from tests/functional/test_show_objects.py rename to tests/performance/list_relations_tests/list_relations.py index 04cdfbc6a..a55305ebb 100644 --- a/tests/functional/test_show_objects.py +++ b/tests/performance/list_relations_tests/list_relations.py @@ -1,3 +1,4 @@ +from dataclasses import dataclass from datetime import datetime, timedelta import os from statistics import mean @@ -9,6 +10,7 @@ from dbt.adapters.snowflake import SnowflakeRelation from dbt.tests.util import run_dbt, get_connection +from tests.performance.conftest import performance_test MY_SEED = """ @@ -34,7 +36,7 @@ """ {{ config( materialized='dynamic_table', - target_lag='1 minute', + target_lag='1 day', snowflake_warehouse='""" + os.getenv("SNOWFLAKE_TEST_WAREHOUSE") + """', @@ -44,26 +46,17 @@ ) -SHOW_OBJECTS = """ -{% macro snowflake__get_show_objects_sql(schema, results_per_iteration) %} - show objects in {{ schema.database }}.{{ schema.schema }} limit {{ results_per_iteration }} -{% endmacro %} -""" - +@dataclass +class Scenario: + views: int + tables: int + dynamic_tables: int -SHOW_TERSE_OBJECTS = """ -{% macro snowflake__get_show_objects_sql(schema, results_per_iteration) %} - show terse objects in {{ schema.database }}.{{ schema.schema }} limit {{ results_per_iteration }} -{% endmacro %} -""" - -class ListRelations: - views: int = 100 - tables: int = 100 - dynamic_tables: int = 100 - iterations: int = 10 +class BaseConfig: + scenario: Scenario expected_duration: float + iterations: int = 10 @pytest.fixture(scope="class") def seeds(self): @@ -72,10 +65,13 @@ def seeds(self): @pytest.fixture(scope="class") def models(self): models = {} - models.update({f"my_view_{i}.sql": VIEW for i in range(self.views)}) - models.update({f"my_table_{i}.sql": TABLE for i in range(self.tables)}) + models.update({f"my_view_{i}.sql": VIEW for i in range(self.scenario.views)}) + models.update({f"my_table_{i}.sql": TABLE for i in range(self.scenario.tables)}) models.update( - {f"my_dynamic_table_{i}.sql": DYNAMIC_TABLE for i in range(self.dynamic_tables)} + { + f"my_dynamic_table_{i}.sql": DYNAMIC_TABLE + for i in range(self.scenario.dynamic_tables) + } ) yield models @@ -91,41 +87,28 @@ def list_relations(self, project) -> Tuple[List[SnowflakeRelation], timedelta]: ) start = datetime.utcnow() - with get_connection(my_adapter) as conn: + with get_connection(my_adapter): relations = my_adapter.list_relations_without_caching(schema) end = datetime.utcnow() duration = end - start return relations, duration - def test_show_terse_objects(self, project): + @performance_test + def test_list_relations(self, project): durations = [] for i in range(self.iterations): relations, duration = self.list_relations(project) durations.append(duration.total_seconds()) - assert len([relation for relation in relations if relation.is_view]) == self.views assert ( - len([relation for relation in relations if relation.is_table]) == self.tables + 1 - ) # add the seed + len([relation for relation in relations if relation.is_view]) + == self.scenario.views + ) + assert ( + len([relation for relation in relations if relation.is_table]) + == self.scenario.tables + 1 # add the seed + ) assert ( len([relation for relation in relations if relation.is_dynamic_table]) - == self.dynamic_tables + == self.scenario.dynamic_tables ) - assert mean(durations) < self.expected_duration - - -class TestShowObjects(ListRelations): - expected_duration = timedelta( - seconds=1, microseconds=100_000 - ).total_seconds() # allows 10% error - - @pytest.fixture(scope="class") - def macros(self): - yield {"snowflake__get_show_objects_sql.sql": SHOW_OBJECTS} - - -class TestShowTerseObjects(ListRelations): - expected_duration = timedelta(seconds=1, microseconds=0).total_seconds() # allows 10% error - - @pytest.fixture(scope="class") - def macros(self): - yield {"snowflake__get_show_objects_sql.sql": SHOW_TERSE_OBJECTS} + assert mean(durations) < self.expected_duration * 1.10 # allow for 10% error diff --git a/tests/performance/list_relations_tests/test_show_objects.py b/tests/performance/list_relations_tests/test_show_objects.py new file mode 100644 index 000000000..1d395aff9 --- /dev/null +++ b/tests/performance/list_relations_tests/test_show_objects.py @@ -0,0 +1,48 @@ +from datetime import timedelta + +import pytest + +from tests.performance.list_relations_tests.list_relations import BaseConfig, Scenario + + +SHOW_OBJECTS_MACRO = """ +{% macro snowflake__get_show_objects_sql(schema, results_per_iteration) %} + show objects in {{ schema.database }}.{{ schema.schema }} limit {{ results_per_iteration }} +{% endmacro %} +""" + + +class ShowObjects(BaseConfig): + @pytest.fixture(scope="class") + def macros(self): + yield {"snowflake__get_show_objects_sql.sql": SHOW_OBJECTS_MACRO} + + +class TestShowObjects10View10Table10Dynamic(ShowObjects): + scenario = Scenario(10, 10, 10) + expected_duration = timedelta(seconds=0, microseconds=920_000).total_seconds() + + +class TestShowObjects15View15Table0Dynamic(ShowObjects): + scenario = Scenario(15, 15, 0) + expected_duration = timedelta(seconds=0, microseconds=920_000).total_seconds() + + +class TestShowObjects100View100Table100Dynamic(ShowObjects): + scenario = Scenario(100, 100, 100) + expected_duration = timedelta(seconds=1, microseconds=370_000).total_seconds() + + +class TestShowObjects150View150Table0Dynamic(ShowObjects): + scenario = Scenario(150, 150, 0) + expected_duration = timedelta(seconds=1, microseconds=370_000).total_seconds() + + +class TestShowObjects1000View1000Table1000Dynamic(ShowObjects): + scenario = Scenario(1000, 1000, 1000) + expected_duration = timedelta(seconds=3, microseconds=400_000).total_seconds() + + +class TestShowObjects1500View1500Table0Dynamic(ShowObjects): + scenario = Scenario(1500, 1500, 0) + expected_duration = timedelta(seconds=3, microseconds=400_000).total_seconds()