Add tests to precommit hooks and so close out this repo for test conv…

…ersions. (#371)

Co-authored-by: Mila Page <[email protected]>

.pre-commit-config.yaml

@@ -1,8 +1,5 @@
 # For more on configuring pre-commit hooks (see https://pre-commit.com/)
 
-# TODO: remove global exclusion of tests when testing overhaul is complete
-exclude: '^tests/.*'
-
 # Force all unspecified python hooks to run python 3.8
 default_language_version:
   python: python3

pytest.ini

@@ -6,5 +6,4 @@ env_files =
     test.env
 testpaths =
     tests/unit
-    tests/integration
     tests/functional

tests/conftest.py

@@ -11,12 +11,12 @@
 @pytest.fixture(scope="class")
 def dbt_profile_target():
     return {
-        'type': 'redshift',
-        'threads': 1,
-        'retries': 6,
-        'host': os.getenv('REDSHIFT_TEST_HOST'),
-        'port': int(os.getenv('REDSHIFT_TEST_PORT')),
-        'user': os.getenv('REDSHIFT_TEST_USER'),
-        'pass': os.getenv('REDSHIFT_TEST_PASS'),
-        'dbname': os.getenv('REDSHIFT_TEST_DBNAME'),
+        "type": "redshift",
+        "threads": 1,
+        "retries": 6,
+        "host": os.getenv("REDSHIFT_TEST_HOST"),
+        "port": int(os.getenv("REDSHIFT_TEST_PORT")),
+        "user": os.getenv("REDSHIFT_TEST_USER"),
+        "pass": os.getenv("REDSHIFT_TEST_PASS"),
+        "dbname": os.getenv("REDSHIFT_TEST_DBNAME"),
     }

tests/functional/adapter/common.py

@@ -4,7 +4,9 @@
 from dbt.tests.fixtures.project import TestProjInfo
 
 
-def get_records(project: TestProjInfo, table: str, select: str = None, where: str = None) -> List[tuple]:
+def get_records(
+    project: TestProjInfo, table: str, select: str = None, where: str = None
+) -> List[tuple]:
     """
     Gets records from a single table in a dbt project
 
@@ -39,7 +41,9 @@ def update_records(project: TestProjInfo, table: str, updates: Dict[str, str], w
         where: the where clause to apply, if any; defaults to all records
     """
     table_name = relation_from_name(project.adapter, table)
-    set_clause = ', '.join([' = '.join([field, expression]) for field, expression in updates.items()])
+    set_clause = ", ".join(
+        [" = ".join([field, expression]) for field, expression in updates.items()]
+    )
     where_clause = where or "1 = 1"
     sql = f"""
         update {table_name}
@@ -49,7 +53,9 @@ def update_records(project: TestProjInfo, table: str, updates: Dict[str, str], w
     project.run_sql(sql)
 
 
-def insert_records(project: TestProjInfo, to_table: str, from_table: str, select: str, where: str = None):
+def insert_records(
+    project: TestProjInfo, to_table: str, from_table: str, select: str, where: str = None
+):
     """
     Inserts records from one table into another table in a dbt project
 
@@ -91,7 +97,9 @@ def delete_records(project: TestProjInfo, table: str, where: str = None):
     project.run_sql(sql)
 
 
-def clone_table(project: TestProjInfo, to_table: str, from_table: str, select: str, where: str = None):
+def clone_table(
+    project: TestProjInfo, to_table: str, from_table: str, select: str, where: str = None
+):
     """
     Creates a new table based on another table in a dbt project
 

tests/functional/adapter/conftest.py

@@ -21,5 +21,5 @@ def test_setting_reflects_config_option(self, model_ddl: str, backup_expected: b
     In this example, the fixture returns the contents of the backup_is_false DDL file as a string.
     This string is then referenced in the test as model_ddl.
     """
-    with open(f"target/run/test/models/{request.param}.sql", 'r') as ddl_file:
-        yield '\n'.join(ddl_file.readlines())
+    with open(f"target/run/test/models/{request.param}.sql", "r") as ddl_file:
+        yield "\n".join(ddl_file.readlines())

tests/functional/adapter/incremental/test_incremental_on_schema_change.py

@@ -1,4 +1,7 @@
-from dbt.tests.adapter.incremental.test_incremental_on_schema_change import BaseIncrementalOnSchemaChange
+from dbt.tests.adapter.incremental.test_incremental_on_schema_change import (
+    BaseIncrementalOnSchemaChange,
+)
+
 
 class TestIncrementalOnSchemaChange(BaseIncrementalOnSchemaChange):
     pass

tests/functional/adapter/incremental/test_incremental_unique_id.py

@@ -2,4 +2,4 @@
 
 
 class TestUniqueKeyRedshift(BaseIncrementalUniqueKey):
-    pass
+    pass

tests/functional/adapter/snapshot_tests/test_snapshot.py

@@ -16,7 +16,6 @@
 
 
 class SnapshotBase:
-
     @pytest.fixture(scope="class")
     def seeds(self):
         """
@@ -80,9 +79,9 @@ def delete_snapshot_records(self):
         common.delete_records(self.project, "snapshot")
 
     def _assert_results(
-            self,
-            ids_with_current_snapshot_records: Iterable,
-            ids_with_closed_out_snapshot_records: Iterable
+        self,
+        ids_with_current_snapshot_records: Iterable,
+        ids_with_closed_out_snapshot_records: Iterable,
     ):
         """
         All test cases are checked by considering whether a source record's id has a value in `dbt_valid_to`
@@ -106,13 +105,12 @@ def _assert_results(
         records = set(self.get_snapshot_records("id, dbt_valid_to is null as is_current"))
         expected_records = set().union(
             {(i, True) for i in ids_with_current_snapshot_records},
-            {(i, False) for i in ids_with_closed_out_snapshot_records}
+            {(i, False) for i in ids_with_closed_out_snapshot_records},
         )
         assert records == expected_records
 
 
 class TestSnapshot(SnapshotBase):
-
     @pytest.fixture(scope="class")
     def snapshots(self):
         return {"snapshot.sql": snapshots.SNAPSHOT_TIMESTAMP_SQL}
@@ -121,11 +119,13 @@ def test_updates_are_captured_by_snapshot(self, project):
         """
         Update the last 5 records. Show that all ids are current, but the last 5 reflect updates.
         """
-        self.update_fact_records({"updated_at": "updated_at + interval '1 day'"}, "id between 16 and 20")
+        self.update_fact_records(
+            {"updated_at": "updated_at + interval '1 day'"}, "id between 16 and 20"
+        )
         run_dbt(["snapshot"])
         self._assert_results(
             ids_with_current_snapshot_records=range(1, 21),
-            ids_with_closed_out_snapshot_records=range(16, 21)
+            ids_with_closed_out_snapshot_records=range(16, 21),
         )
 
     def test_inserts_are_captured_by_snapshot(self, project):
@@ -135,8 +135,7 @@ def test_inserts_are_captured_by_snapshot(self, project):
         self.insert_fact_records("id between 21 and 30")
         run_dbt(["snapshot"])
         self._assert_results(
-            ids_with_current_snapshot_records=range(1, 31),
-            ids_with_closed_out_snapshot_records=[]
+            ids_with_current_snapshot_records=range(1, 31), ids_with_closed_out_snapshot_records=[]
         )
 
     def test_deletes_are_captured_by_snapshot(self, project):
@@ -147,7 +146,7 @@ def test_deletes_are_captured_by_snapshot(self, project):
         run_dbt(["snapshot"])
         self._assert_results(
             ids_with_current_snapshot_records=range(1, 16),
-            ids_with_closed_out_snapshot_records=range(16, 21)
+            ids_with_closed_out_snapshot_records=range(16, 21),
         )
 
     def test_revives_are_captured_by_snapshot(self, project):
@@ -161,7 +160,7 @@ def test_revives_are_captured_by_snapshot(self, project):
         run_dbt(["snapshot"])
         self._assert_results(
             ids_with_current_snapshot_records=range(1, 19),
-            ids_with_closed_out_snapshot_records=range(16, 21)
+            ids_with_closed_out_snapshot_records=range(16, 21),
         )
 
     def test_new_column_captured_by_snapshot(self, project):
@@ -176,17 +175,16 @@ def test_new_column_captured_by_snapshot(self, project):
                 "full_name": "first_name || ' ' || last_name",
                 "updated_at": "updated_at + interval '1 day'",
             },
-            "id between 11 and 20"
+            "id between 11 and 20",
         )
         run_dbt(["snapshot"])
         self._assert_results(
             ids_with_current_snapshot_records=range(1, 21),
-            ids_with_closed_out_snapshot_records=range(11, 21)
+            ids_with_closed_out_snapshot_records=range(11, 21),
         )
 
 
 class TestSnapshotCheck(SnapshotBase):
-
     @pytest.fixture(scope="class")
     def snapshots(self):
         return {"snapshot.sql": snapshots.SNAPSHOT_CHECK_SQL}
@@ -197,10 +195,12 @@ def test_column_selection_is_reflected_in_snapshot(self, project):
         Update the middle 10 records on a tracked column. (hence records 6-10 are updated on both)
         Show that all ids are current, and only the tracked column updates are reflected in `snapshot`.
         """
-        self.update_fact_records({"last_name": "left(last_name, 3)"}, "id between 1 and 10")  # not tracked
-        self.update_fact_records({"email": "left(email, 3)"}, "id between 6 and 15")          # tracked
+        self.update_fact_records(
+            {"last_name": "left(last_name, 3)"}, "id between 1 and 10"
+        )  # not tracked
+        self.update_fact_records({"email": "left(email, 3)"}, "id between 6 and 15")  # tracked
         run_dbt(["snapshot"])
         self._assert_results(
             ids_with_current_snapshot_records=range(1, 21),
-            ids_with_closed_out_snapshot_records=range(6, 16)
+            ids_with_closed_out_snapshot_records=range(6, 16),
         )

tests/functional/adapter/test_backup_table.py

@@ -59,14 +59,12 @@
 
 
 class BackupTableBase:
-
     @pytest.fixture(scope="class", autouse=True)
     def _run_dbt(self, project):
         run_dbt(["run"])
 
 
 class TestBackupTableOption(BackupTableBase):
-
     @pytest.fixture(scope="class")
     def models(self):
         return {
@@ -84,7 +82,7 @@ def models(self):
             ("backup_is_undefined", True),
             ("backup_is_true_view", True),
         ],
-        indirect=["model_ddl"]
+        indirect=["model_ddl"],
     )
     def test_setting_reflects_config_option(self, model_ddl: str, backup_expected: bool):
         """
@@ -102,7 +100,6 @@ def test_setting_reflects_config_option(self, model_ddl: str, backup_expected: b
 
 
 class TestBackupTableSyntax(BackupTableBase):
-
     @pytest.fixture(scope="class")
     def models(self):
         return {
@@ -116,7 +113,7 @@ def models(self):
             ("syntax_with_distkey", "diststyle key distkey"),
             ("syntax_with_sortkey", "compound sortkey"),
         ],
-        indirect=["model_ddl"]
+        indirect=["model_ddl"],
     )
     def test_backup_predicate_precedes_secondary_predicates(self, model_ddl, search_phrase):
         """
@@ -133,7 +130,6 @@ def test_backup_predicate_precedes_secondary_predicates(self, model_ddl, search_
 
 
 class TestBackupTableProjectDefault(BackupTableBase):
-
     @pytest.fixture(scope="class")
     def project_config_update(self):
         return {"models": {"backup": False}}
@@ -147,11 +143,8 @@ def models(self):
 
     @pytest.mark.parametrize(
         "model_ddl,backup_expected",
-        [
-            ("backup_is_true", True),
-            ("backup_is_undefined", False)
-        ],
-        indirect=["model_ddl"]
+        [("backup_is_true", True), ("backup_is_undefined", False)],
+        indirect=["model_ddl"],
     )
     def test_setting_defaults_to_project_option(self, model_ddl: str, backup_expected: bool):
         """

tests/functional/adapter/test_basic.py

@@ -12,10 +12,17 @@
 from dbt.tests.adapter.basic.test_snapshot_timestamp import BaseSnapshotTimestamp
 from dbt.tests.adapter.basic.test_adapter_methods import BaseAdapterMethod
 from dbt.tests.adapter.basic.test_docs_generate import BaseDocsGenerate, BaseDocsGenReferences
-from dbt.tests.adapter.basic.expected_catalog import base_expected_catalog, no_stats, expected_references_catalog
+from dbt.tests.adapter.basic.expected_catalog import (
+    base_expected_catalog,
+    no_stats,
+    expected_references_catalog,
+)
 from dbt.tests.adapter.basic.files import seeds_base_csv, seeds_added_csv, seeds_newcolumns_csv
 
-from tests.functional.adapter.expected_stats import redshift_stats, redshift_ephemeral_summary_stats
+from tests.functional.adapter.expected_stats import (
+    redshift_stats,
+    redshift_ephemeral_summary_stats,
+)
 
 
 # set the datatype of the name column in the 'added' seed so that it can hold the '_update' that's added
@@ -86,19 +93,19 @@ class TestBaseAdapterMethod(BaseAdapterMethod):
 
 
 class TestDocsGenerateRedshift(BaseDocsGenerate):
-    @pytest.fixture(scope="class")                               
+    @pytest.fixture(scope="class")
     def expected_catalog(self, project, profile_user):
         return base_expected_catalog(
-            project,                               
-            role=profile_user,                    
-            id_type="integer",                      
+            project,
+            role=profile_user,
+            id_type="integer",
             text_type=AnyStringWith("character varying"),
             time_type="timestamp without time zone",
-            view_type="VIEW",                
-            table_type="BASE TABLE",                 
+            view_type="VIEW",
+            table_type="BASE TABLE",
             model_stats=no_stats(),
             seed_stats=redshift_stats(),
-        )             
+        )
 
 
 # TODO: update this or delete it

tests/functional/adapter/test_changing_relation_type.py

@@ -1,4 +1,5 @@
 from dbt.tests.adapter.relations.test_changing_relation_type import BaseChangeRelationTypeValidator
 
+
 class TestRedshiftChangeRelationTypes(BaseChangeRelationTypeValidator):
-    pass
+    pass

tests/functional/adapter/test_column_types.py

@@ -1,7 +1,7 @@
 import pytest
 from dbt.tests.adapter.column_types.test_column_types import BaseColumnTypes
 
-_MODEL_SQL =  """
+_MODEL_SQL = """
 select
     1::smallint as smallint_col,
     2::int as int_col,
@@ -46,14 +46,11 @@
             text_col: ['string', 'not number']
 """
 
-class TestRedshiftColumnTypes(BaseColumnTypes):
 
+class TestRedshiftColumnTypes(BaseColumnTypes):
     @pytest.fixture(scope="class")
     def models(self):
-        return {
-            "model.sql": _MODEL_SQL,
-            "schema.yml": _SCHEMA_YML
-        }
+        return {"model.sql": _MODEL_SQL, "schema.yml": _SCHEMA_YML}
 
     def test_run_and_test(self, project):
-        self.run_and_test()
+        self.run_and_test()