create and document performance tests

tests/functional/test_show_objects.py → tests/performance/list_relations_tests/list_relations.py

@@ -1,3 +1,4 @@
+from dataclasses import dataclass
 from datetime import datetime, timedelta
 import os
 from statistics import mean
@@ -9,6 +10,7 @@
 from dbt.adapters.snowflake import SnowflakeRelation
 
 from dbt.tests.util import run_dbt, get_connection
+from tests.performance.conftest import performance_test
 
 
 MY_SEED = """
@@ -34,7 +36,7 @@
     """
 {{ config(
     materialized='dynamic_table',
-    target_lag='1 minute',
+    target_lag='1 day',
     snowflake_warehouse='"""
     + os.getenv("SNOWFLAKE_TEST_WAREHOUSE")
     + """',
@@ -44,26 +46,17 @@
 )
 
 
-SHOW_OBJECTS = """
-{% macro snowflake__get_show_objects_sql(schema, results_per_iteration) %}
-    show objects in {{ schema.database }}.{{ schema.schema }} limit {{ results_per_iteration }}
-{% endmacro %}
-"""
-
+@dataclass
+class Scenario:
+    views: int
+    tables: int
+    dynamic_tables: int
 
-SHOW_TERSE_OBJECTS = """
-{% macro snowflake__get_show_objects_sql(schema, results_per_iteration) %}
-    show terse objects in {{ schema.database }}.{{ schema.schema }} limit {{ results_per_iteration }}
-{% endmacro %}
-"""
 
-
-class ListRelations:
-    views: int = 100
-    tables: int = 100
-    dynamic_tables: int = 100
-    iterations: int = 10
+class BaseConfig:
+    scenario: Scenario
     expected_duration: float
+    iterations: int = 10
 
     @pytest.fixture(scope="class")
     def seeds(self):
@@ -72,10 +65,13 @@ def seeds(self):
     @pytest.fixture(scope="class")
     def models(self):
         models = {}
-        models.update({f"my_view_{i}.sql": VIEW for i in range(self.views)})
-        models.update({f"my_table_{i}.sql": TABLE for i in range(self.tables)})
+        models.update({f"my_view_{i}.sql": VIEW for i in range(self.scenario.views)})
+        models.update({f"my_table_{i}.sql": TABLE for i in range(self.scenario.tables)})
         models.update(
-            {f"my_dynamic_table_{i}.sql": DYNAMIC_TABLE for i in range(self.dynamic_tables)}
+            {
+                f"my_dynamic_table_{i}.sql": DYNAMIC_TABLE
+                for i in range(self.scenario.dynamic_tables)
+            }
         )
         yield models
 
@@ -91,41 +87,28 @@ def list_relations(self, project) -> Tuple[List[SnowflakeRelation], timedelta]:
         )
 
         start = datetime.utcnow()
-        with get_connection(my_adapter) as conn:
+        with get_connection(my_adapter):
             relations = my_adapter.list_relations_without_caching(schema)
         end = datetime.utcnow()
         duration = end - start
         return relations, duration
 
-    def test_show_terse_objects(self, project):
+    @performance_test
+    def test_list_relations(self, project):
         durations = []
         for i in range(self.iterations):
             relations, duration = self.list_relations(project)
             durations.append(duration.total_seconds())
-            assert len([relation for relation in relations if relation.is_view]) == self.views
             assert (
-                len([relation for relation in relations if relation.is_table]) == self.tables + 1
-            )  # add the seed
+                len([relation for relation in relations if relation.is_view])
+                == self.scenario.views
+            )
+            assert (
+                len([relation for relation in relations if relation.is_table])
+                == self.scenario.tables + 1  # add the seed
+            )
             assert (
                 len([relation for relation in relations if relation.is_dynamic_table])
-                == self.dynamic_tables
+                == self.scenario.dynamic_tables
             )
-        assert mean(durations) < self.expected_duration
-
-
-class TestShowObjects(ListRelations):
-    expected_duration = timedelta(
-        seconds=1, microseconds=100_000
-    ).total_seconds()  # allows 10% error
-
-    @pytest.fixture(scope="class")
-    def macros(self):
-        yield {"snowflake__get_show_objects_sql.sql": SHOW_OBJECTS}
-
-
-class TestShowTerseObjects(ListRelations):
-    expected_duration = timedelta(seconds=1, microseconds=0).total_seconds()  # allows 10% error
-
-    @pytest.fixture(scope="class")
-    def macros(self):
-        yield {"snowflake__get_show_objects_sql.sql": SHOW_TERSE_OBJECTS}
+        assert mean(durations) < self.expected_duration * 1.10  # allow for 10% error

tests/performance/list_relations_tests/test_show_objects.py

@@ -0,0 +1,48 @@
+from datetime import timedelta
+
+import pytest
+
+from tests.performance.list_relations_tests.list_relations import BaseConfig, Scenario
+
+
+SHOW_OBJECTS_MACRO = """
+{% macro snowflake__get_show_objects_sql(schema, results_per_iteration) %}
+    show objects in {{ schema.database }}.{{ schema.schema }} limit {{ results_per_iteration }}
+{% endmacro %}
+"""
+
+
+class ShowObjects(BaseConfig):
+    @pytest.fixture(scope="class")
+    def macros(self):
+        yield {"snowflake__get_show_objects_sql.sql": SHOW_OBJECTS_MACRO}
+
+
+class TestShowObjects10View10Table10Dynamic(ShowObjects):
+    scenario = Scenario(10, 10, 10)
+    expected_duration = timedelta(seconds=0, microseconds=920_000).total_seconds()
+
+
+class TestShowObjects15View15Table0Dynamic(ShowObjects):
+    scenario = Scenario(15, 15, 0)
+    expected_duration = timedelta(seconds=0, microseconds=920_000).total_seconds()
+
+
+class TestShowObjects100View100Table100Dynamic(ShowObjects):
+    scenario = Scenario(100, 100, 100)
+    expected_duration = timedelta(seconds=1, microseconds=370_000).total_seconds()
+
+
+class TestShowObjects150View150Table0Dynamic(ShowObjects):
+    scenario = Scenario(150, 150, 0)
+    expected_duration = timedelta(seconds=1, microseconds=370_000).total_seconds()
+
+
+class TestShowObjects1000View1000Table1000Dynamic(ShowObjects):
+    scenario = Scenario(1000, 1000, 1000)
+    expected_duration = timedelta(seconds=3, microseconds=400_000).total_seconds()
+
+
+class TestShowObjects1500View1500Table0Dynamic(ShowObjects):
+    scenario = Scenario(1500, 1500, 0)
+    expected_duration = timedelta(seconds=3, microseconds=400_000).total_seconds()