diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 1e3e76b4..99bff152 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -14,9 +14,16 @@ jobs: name: Integration tests on Azure strategy: matrix: - python_version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python_version: ["3.8", "3.9", "3.10", "3.11"] msodbc_version: ["17", "18"] runs-on: ubuntu-latest + permissions: + actions: read + contents: read + deployments: read + packages: none + pull-requests: write + security-events: write container: image: ghcr.io/dbt-msft/dbt-sqlserver:CI-${{ matrix.python_version }}-msodbc${{ matrix.msodbc_version }} steps: diff --git a/CHANGELOG.md b/CHANGELOG.md index c1db77db..6e1bffc9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,13 +14,44 @@ - store_test_failures - dbt_clone (same target and state) - seed +## v1.6.0rc1 + +* Support for [dbt-core 1.6](https://github.com/dbt-labs/dbt-core/releases/tag/v1.6.0) + +#### Breaking Changes +* Dropped support for Python 3.7 ([#7082](https://github.com/dbt-labs/dbt-core/issues/7082https://github.com/dbt-labs/dbt-core/issues/7082)) + +## Features +* Add support for materialized views ([#6911](https://github.com/dbt-labs/dbt-core/issues/6911)) + * important note! unlike [dbt's materialized view](https://docs.getdbt.com/docs/build/materializations), [Synapse's materialized view](https://learn.microsoft.com/en-us/sql/t-sql/statements/create-materialized-view-as-select-transact-sql?view=azure-sqldw-latest&context=%2Fazure%2Fsynapse-analytics%2Fcontext%2Fcontext) must be created using aggregation and/or "GROUP BY"! +* ~~dbt clone ([#7258](https://github.com/dbt-labs/dbt-core/issues/7258)~~ Synapse does not support CLONE) +* Revamp dbt debug ([#7104](https://github.com/dbt-labs/dbt-core/issues/7104)) +* Added new adapter zone tests + - constraints + - null_compare + - validate_sql + - equals + - dbt_clone + +## v.1.5.0rc1 + +* Support for [dbt-core 1.5](https://github.com/dbt-labs/dbt-core/releases/tag/v1.5.0) + * Add support for model contracts by adapting `create_table_as` and `create_view_as` macros + * Define supported constraints in `CONSTRAINT_SUPPORT` Adapter class. + * Persist docs via [extended properties](https://github.com/dbt-msft/dbt-sqlserver/issues/134) is [not supported](https://learn.microsoft.com/en-us/sql/relational-databases/system-stored-procedures/sp-addextendedproperty-transact-sql?view=sql-server-ver16) in Synapse + * Add adapter tests zones + - caching + - column_types + - constraints + - hooks + - simple_copy ## v1.4.1rc1 #### Under the hood -* Switch dependency from dbt-sqlserver to dbt-fabric (per https://github.com/dbt-msft/dbt-sqlserver/issues/441) +* Switch dependency from dbt-sqlserver to dbt-fabric [#441](https://github.com/dbt-msft/dbt-sqlserver/issues/441) * for Mac users, before running `make dev`, add `pyodbc==4.0.39 --no-binary :all:` in dev_requirements.txt - * about pyodbc "Symbol not found: _SQLAllocHandle" error https://stackoverflow.com/questions/66731036/unable-to-import-pyodbc-on-apple-silicon-symbol-not-found-sqlallochandle + * [Stackoverflow](https://stackoverflow.com/questions/66731036/unable-to-import-pyodbc-on-apple-silicon-symbol-not-found-sqlallochandle) about pyodbc "Symbol not found: _SQLAllocHandle" error ## v1.4.0 diff --git a/dbt/adapters/synapse/__init__.py b/dbt/adapters/synapse/__init__.py index 6a0d96fd..61f6d4bb 100644 --- a/dbt/adapters/synapse/__init__.py +++ b/dbt/adapters/synapse/__init__.py @@ -1,6 +1,7 @@ from dbt.adapters.base import AdapterPlugin from dbt.adapters.synapse.synapse_adapter import SynapseAdapter +from dbt.adapters.synapse.synapse_column import SynapseColumn from dbt.adapters.synapse.synapse_connection_manager import SynapseConnectionManager from dbt.adapters.synapse.synapse_credentials import SynapseCredentials from dbt.include import synapse @@ -12,4 +13,10 @@ dependencies=["fabric"], ) -__all__ = ["Plugin", "SynapseConnectionManager", "SynapseAdapter", "SynapseCredentials"] +__all__ = [ + "Plugin", + "SynapseConnectionManager", + "SynapseColumn", + "SynapseAdapter", + "SynapseCredentials", +] diff --git a/dbt/adapters/synapse/synapse_adapter.py b/dbt/adapters/synapse/synapse_adapter.py index 5c02b579..664ea133 100644 --- a/dbt/adapters/synapse/synapse_adapter.py +++ b/dbt/adapters/synapse/synapse_adapter.py @@ -1,15 +1,21 @@ +from enum import Enum +from typing import Any, Dict, List, Optional + from dbt.adapters.base.relation import BaseRelation from dbt.adapters.cache import _make_ref_key_dict from dbt.adapters.fabric import FabricAdapter from dbt.adapters.sql.impl import CREATE_SCHEMA_MACRO_NAME +from dbt.contracts.graph.nodes import ColumnLevelConstraint, ConstraintType from dbt.events.functions import fire_event from dbt.events.types import SchemaCreation +from dbt.adapters.synapse.synapse_column import SynapseColumn from dbt.adapters.synapse.synapse_connection_manager import SynapseConnectionManager class SynapseAdapter(FabricAdapter): ConnectionManager = SynapseConnectionManager + Column = SynapseColumn def create_schema(self, relation: BaseRelation) -> None: relation = relation.without_identifier() @@ -25,3 +31,51 @@ def create_schema(self, relation: BaseRelation) -> None: self.execute_macro(macro_name, kwargs=kwargs) self.commit_if_has_connection() + + class ConstraintSupport(str, Enum): + ENFORCED = "enforced" + NOT_ENFORCED = "not_enforced" + NOT_SUPPORTED = "not_supported" + + # https://learn.microsoft.com/en-us/azure/synapse-analytics/sql-data-warehouse/sql-data-warehouse-table-constraints#table-constraints + CONSTRAINT_SUPPORT = { + ConstraintType.check: ConstraintSupport.NOT_SUPPORTED, # no CHECK support for Synapse + ConstraintType.not_null: ConstraintSupport.ENFORCED, + ConstraintType.unique: ConstraintSupport.NOT_ENFORCED, + ConstraintType.primary_key: ConstraintSupport.NOT_ENFORCED, + ConstraintType.foreign_key: ConstraintSupport.NOT_SUPPORTED, # no FK support for Synapse + } + + @classmethod + def render_column_constraint(cls, constraint: ColumnLevelConstraint) -> Optional[str]: + """Render the given constraint as DDL text. + Should be overriden by adapters which need custom constraint rendering.""" + if constraint.type == ConstraintType.check and constraint.expression: + return f"check {constraint.expression}" + elif constraint.type == ConstraintType.not_null: + return "not null" + elif constraint.type == ConstraintType.unique: + return "unique NOT ENFORCED" + elif constraint.type == ConstraintType.primary_key: + return "primary key NONCLUSTERED NOT ENFORCED" + elif constraint.type == ConstraintType.foreign_key: + return "foreign key" + elif constraint.type == ConstraintType.custom and constraint.expression: + return constraint.expression + else: + return None + + @classmethod + def render_raw_columns_constraints(cls, raw_columns: Dict[str, Dict[str, Any]]) -> List: + rendered_column_constraints = [] + + for v in raw_columns.values(): + rendered_column_constraint = [f"[{v['name']}] {v['data_type']}"] + for con in v.get("constraints", None): + constraint = cls._parse_column_constraint(con) + c = cls.process_parsed_constraint(constraint, cls.render_column_constraint) + if c is not None: + rendered_column_constraint.append(c) + rendered_column_constraints.append(" ".join(rendered_column_constraint)) + + return rendered_column_constraints diff --git a/dbt/adapters/synapse/synapse_column.py b/dbt/adapters/synapse/synapse_column.py new file mode 100644 index 00000000..37e336cf --- /dev/null +++ b/dbt/adapters/synapse/synapse_column.py @@ -0,0 +1,16 @@ +from dbt.adapters.fabric import FabricColumn + + +class SynapseColumn(FabricColumn): + # extending list of integer types for synapse + def is_integer(self) -> bool: + return self.dtype.lower() in [ + # real types + "smallint", + "bigint", + "tinyint", + "serial", + "bigserial", + "int", + "bit", + ] diff --git a/dbt/include/synapse/macros/adapters/persist_docs.sql b/dbt/include/synapse/macros/adapters/persist_docs.sql new file mode 100644 index 00000000..091f954d --- /dev/null +++ b/dbt/include/synapse/macros/adapters/persist_docs.sql @@ -0,0 +1,4 @@ +{# Unfortunately adding docs via extended properties is not supported in Synapse only in SQLServer + https://github.com/dbt-msft/dbt-sqlserver/issues/134 + https://learn.microsoft.com/en-us/sql/relational-databases/system-stored-procedures/sp-addextendedproperty-transact-sql?view=sql-server-ver16 + #} diff --git a/dbt/include/synapse/macros/adapters/relation.sql b/dbt/include/synapse/macros/adapters/relation.sql index 8c1d3582..e5136e56 100644 --- a/dbt/include/synapse/macros/adapters/relation.sql +++ b/dbt/include/synapse/macros/adapters/relation.sql @@ -5,21 +5,35 @@ {% endmacro %} {% macro synapse__drop_relation_script(relation) -%} - {% if relation.type == 'view' -%} + {% if relation.type == 'view' or relation.type == 'materialized_view' -%} {% set object_id_type = 'V' %} {% elif relation.type == 'table'%} {% set object_id_type = 'U' %} {%- else -%} invalid target name {% endif %} + if object_id ('{{ relation.include(database=False) }}','{{ object_id_type }}') is not null + {% if relation.type == 'view' or relation.type == 'materialized_view' -%} + begin + drop view {{ relation.include(database=False) }} + end + {% elif relation.type == 'table' %} begin drop {{ relation.type }} {{ relation.include(database=False) }} end + {% endif %} {% endmacro %} - {% macro synapse__rename_relation(from_relation, to_relation) -%} - {% call statement('rename_relation') -%} + {# dbt needs this 'call' macro, but it overwrites other SQL when reused in other macros #} + {# so '_script' macro is reuseable script, for other macros to combine with more SQL #} + + {% call statement('rename_relation') %} + {{ synapse__rename_relation_script(from_relation, to_relation) }} + {%- endcall %} +{% endmacro %} + +{% macro synapse__rename_relation_script(from_relation, to_relation) -%} -- drop all object types with to_relation.identifier name, to avoid error "new name already in use...duplicate...not permitted" if object_id ('{{ to_relation.include(database=False) }}','V') is not null begin @@ -32,11 +46,10 @@ end rename object {{ from_relation.include(database=False) }} to {{ to_relation.identifier }} - {%- endcall %} {% endmacro %} {% macro synapse__truncate_relation(relation) %} {% call statement('truncate_relation') -%} - truncate table {{ relation }} + truncate table {{ relation }} {%- endcall %} {% endmacro %} diff --git a/dbt/include/synapse/macros/adapters/show.sql b/dbt/include/synapse/macros/adapters/show.sql new file mode 100644 index 00000000..4caa6eea --- /dev/null +++ b/dbt/include/synapse/macros/adapters/show.sql @@ -0,0 +1,22 @@ +{% macro get_show_sql(compiled_code, sql_header, limit) -%} + {%- if sql_header -%} + {{ sql_header }} + {%- endif -%} + {%- if limit is not none -%} + {{ get_limit_subquery_sql(compiled_code, limit) }} + {%- else -%} + {{ compiled_code }} + {%- endif -%} +{% endmacro %} + +{% macro get_limit_subquery_sql(sql, limit) %} + {{ adapter.dispatch('get_limit_subquery_sql', 'dbt')(sql, limit) }} +{% endmacro %} + +{# Synapse doesnt support ANSI LIMIT clause #} +{% macro synapse__get_limit_subquery_sql(sql, limit) %} + select top {{ limit }} * + from ( + {{ sql }} + ) as model_limit_subq +{% endmacro %} diff --git a/dbt/include/synapse/macros/materializations/models/materialized_view/materialized_view.sql b/dbt/include/synapse/macros/materializations/models/materialized_view/materialized_view.sql new file mode 100644 index 00000000..2dff1b4b --- /dev/null +++ b/dbt/include/synapse/macros/materializations/models/materialized_view/materialized_view.sql @@ -0,0 +1,29 @@ +{% macro ref(model_name) %} + + {% do return(builtins.ref(model_name).include(database=false)) %} + +{% endmacro %} + +{% macro synapse__get_replace_materialized_view_as_sql(relation, sql, existing_relation, backup_relation, intermediate_relation) %} + {# Synapse does not have ALTER...RENAME function, so use synapse__rename_relation_script #} + + {%- set dist = config.get('dist', default="ROUND_ROBIN") -%} + EXEC(' + CREATE materialized view {{ intermediate_relation.include(database=False) }} + WITH ( DISTRIBUTION = {{dist}} ) + AS {{ sql }} + '); + + {{ synapse__rename_relation_script(existing_relation, backup_relation) }} + {{ synapse__rename_relation_script(intermediate_relation, relation) }} + +{% endmacro %} + +{% macro synapse__get_create_materialized_view_as_sql(relation, sql) %} + {%- set dist = config.get('dist', default="ROUND_ROBIN") -%} + + CREATE materialized view {{ relation.include(database=False) }} + WITH ( DISTRIBUTION = {{dist}} ) + AS {{ sql }} + +{% endmacro %} diff --git a/dbt/include/synapse/macros/materializations/models/table/create_table_as.sql b/dbt/include/synapse/macros/materializations/models/table/create_table_as.sql index 60331e50..744e7872 100644 --- a/dbt/include/synapse/macros/materializations/models/table/create_table_as.sql +++ b/dbt/include/synapse/macros/materializations/models/table/create_table_as.sql @@ -10,16 +10,35 @@ {{ synapse__drop_relation_script(relation) }} - EXEC('create view [{{ tmp_relation.schema }}].[{{ tmp_relation.identifier }}] as - {{ temp_view_sql }} - '); - - CREATE TABLE {{ relation.include(database=False) }} - WITH( - DISTRIBUTION = {{dist}}, - {{index}} - ) - AS (SELECT * FROM [{{ tmp_relation.schema }}].[{{ tmp_relation.identifier }}]) + {{ synapse__create_view_as(tmp_relation, sql) }} + + {% set contract_config = config.get('contract') %} + + {% if contract_config.enforced %} + + {{exceptions.warn("Model contracts cannot be enforced by !")}} + + CREATE TABLE [{{relation.schema}}].[{{relation.identifier}}] + {{ synapse__build_columns_constraints(tmp_relation) }} + WITH( + DISTRIBUTION = {{dist}}, + {{index}} + ) + {{ get_assert_columns_equivalent(sql) }} + + {% set listColumns %} + {% for column in model['columns'] %} + {{ "["~column~"]" }}{{ ", " if not loop.last }} + {% endfor %} + {%endset%} + {{ synapse__build_model_constraints(relation) }} + + INSERT INTO [{{relation.schema}}].[{{relation.identifier}}] + ({{listColumns}}) SELECT {{listColumns}} FROM [{{tmp_relation.schema}}].[{{tmp_relation.identifier}}] + + {%- else %} + EXEC('CREATE TABLE [{{relation.database}}].[{{relation.schema}}].[{{relation.identifier}}]WITH(DISTRIBUTION = {{dist}},{{index}}) AS (SELECT * FROM [{{tmp_relation.database}}].[{{tmp_relation.schema}}].[{{tmp_relation.identifier}}]);'); + {% endif %} {{ synapse__drop_relation_script(tmp_relation) }} diff --git a/dbt/include/synapse/macros/materializations/models/table/create_table_constraints.sql b/dbt/include/synapse/macros/materializations/models/table/create_table_constraints.sql new file mode 100644 index 00000000..7ae3650e --- /dev/null +++ b/dbt/include/synapse/macros/materializations/models/table/create_table_constraints.sql @@ -0,0 +1,17 @@ +{% macro synapse__build_columns_constraints(relation) %} + {# loop through user_provided_columns to create DDL with data types and constraints #} + {%- set raw_column_constraints = adapter.render_raw_columns_constraints(raw_columns=model['columns']) -%} + ( + {% for c in raw_column_constraints -%} + {{ c }}{{ "," if not loop.last }} + {% endfor %} + ) +{% endmacro %} + +{% macro synapse__build_model_constraints(relation) %} + {# loop through user_provided_columns to create DDL with data types and constraints #} + {%- set raw_model_constraints = adapter.render_raw_model_constraints(raw_constraints=model['constraints']) -%} + {% for c in raw_model_constraints -%} + alter table {{ relation.include(database=False) }} {{c}}; + {% endfor -%} +{% endmacro %} diff --git a/dbt/include/synapse/macros/materializations/models/view/create_view_as.sql b/dbt/include/synapse/macros/materializations/models/view/create_view_as.sql index b4a93ec5..7deee62e 100644 --- a/dbt/include/synapse/macros/materializations/models/view/create_view_as.sql +++ b/dbt/include/synapse/macros/materializations/models/view/create_view_as.sql @@ -1,4 +1,15 @@ {% macro synapse__create_view_as(relation, sql) -%} - create view {{ relation.include(database=False) }} as - {{ sql }} + + {%- set temp_view_sql = sql.replace("'", "''") -%} + + {% set contract_config = config.get('contract') %} + + {{exceptions.warn("Model contracts cannot be enforced by !")}} + + {% if contract_config.enforced %} + {{ get_assert_columns_equivalent(sql) }} + {%- endif %} + + EXEC('create view {{ relation.include(database=False) }} as {{ temp_view_sql }};'); + {% endmacro %} diff --git a/dbt/include/synapse/macros/utils/split_part.sql b/dbt/include/synapse/macros/utils/split_part.sql index ee01de1d..0e7055d1 100644 --- a/dbt/include/synapse/macros/utils/split_part.sql +++ b/dbt/include/synapse/macros/utils/split_part.sql @@ -2,8 +2,18 @@ We have to replace the macro from dbt-sqlserver since that one uses XML which is an unsupported data type in Synapse. The function below is not supported in Synapse Dedicated SQL according to the documentation, but it seems to work. #} + {% macro synapse__split_part(string_text, delimiter_text, part_number) %} - (select value from string_split({{ string_text }}, {{ delimiter_text }}, 1) where ordinal = {{ part_number }}) + {% if part_number >= 0 %} + + (select value from string_split({{ string_text }}, {{ delimiter_text }}, 1) where ordinal = {{ part_number }}) + + {% else %} + + (select value from string_split({{ string_text }}, {{ delimiter_text }}, 1) + where ordinal = len(replace({{ string_text }}, {{delimiter_text}}, '')) + 1 + {{ part_number }}) + + {% endif %} {% endmacro %} diff --git a/setup.py b/setup.py index ea2a8d65..9990d69c 100644 --- a/setup.py +++ b/setup.py @@ -83,7 +83,6 @@ def run(self): "Operating System :: Microsoft :: Windows", "Operating System :: MacOS :: MacOS X", "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", diff --git a/tests/functional/adapter/data/seed_model.sql b/tests/functional/adapter/data/seed_model.sql new file mode 100644 index 00000000..52c2ce4e --- /dev/null +++ b/tests/functional/adapter/data/seed_model.sql @@ -0,0 +1,19 @@ +create table {schema}.on_model_hook +( + test_state VARCHAR(100), -- start|end + target_dbname VARCHAR(100), + target_host VARCHAR(100), + target_name VARCHAR(100), + target_schema VARCHAR(100), + target_type VARCHAR(100), + target_user VARCHAR(100), + target_pass VARCHAR(100), + target_threads INTEGER, + run_started_at VARCHAR(100), + invocation_id VARCHAR(100), + thread_id VARCHAR(100) +) +WITH( + DISTRIBUTION = ROUND_ROBIN, + HEAP +) diff --git a/tests/functional/adapter/data/seed_run.sql b/tests/functional/adapter/data/seed_run.sql new file mode 100644 index 00000000..2c0d23b9 --- /dev/null +++ b/tests/functional/adapter/data/seed_run.sql @@ -0,0 +1,22 @@ +if object_id ('{schema}.on_run_hook') is not null + drop table {schema}.on_run_hook; + +create table {schema}.on_run_hook +( + test_state VARCHAR(100), -- start|end + target_dbname VARCHAR(100), + target_host VARCHAR(100), + target_name VARCHAR(100), + target_schema VARCHAR(100), + target_type VARCHAR(100), + target_user VARCHAR(100), + target_pass VARCHAR(100), + target_threads INTEGER, + run_started_at VARCHAR(100), + invocation_id VARCHAR(100), + thread_id VARCHAR(100) +) +WITH( + DISTRIBUTION = ROUND_ROBIN, + HEAP +); diff --git a/tests/functional/adapter/test_caching.py b/tests/functional/adapter/test_caching.py new file mode 100644 index 00000000..a517c58f --- /dev/null +++ b/tests/functional/adapter/test_caching.py @@ -0,0 +1,117 @@ +import pytest +from dbt.tests.util import run_dbt + +model_sql = """ +{{ + config( + materialized='table' + ) +}} +select 1 as id +""" + +another_schema_model_sql = """ +{{ + config( + materialized='table', + schema='another_schema' + ) +}} +select 1 as id +""" + + +class BaseCachingTest: + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "config-version": 2, + "quoting": { + "identifier": False, + "schema": False, + }, + } + + def run_and_inspect_cache(self, project, run_args=None): + run_dbt(run_args) + + # the cache was empty at the start of the run. + # the model materialization returned an unquoted relation and added to the cache. + adapter = project.adapter + assert len(adapter.cache.relations) == 1 + relation = list(adapter.cache.relations).pop() + assert relation.schema == project.test_schema + assert relation.schema == project.test_schema.lower() + + # on the second run, dbt will find a relation in the database during cache population. + # this relation will be quoted, because list_relations_without_caching (by default) uses + # quote_policy = {"database": True, "schema": True, "identifier": True} + # when adding relations to the cache. + run_dbt(run_args) + adapter = project.adapter + assert len(adapter.cache.relations) == 1 + second_relation = list(adapter.cache.relations).pop() + + # perform a case-insensitive + quote-insensitive comparison + for key in ["database", "schema", "identifier"]: + assert getattr(relation, key).lower() == getattr(second_relation, key).lower() + + def test_cache(self, project): + self.run_and_inspect_cache(project, run_args=["run"]) + + +class BaseCachingLowercaseModel(BaseCachingTest): + @pytest.fixture(scope="class") + def models(self): + return { + "model.sql": model_sql, + } + + +class BaseCachingUppercaseModel(BaseCachingTest): + @pytest.fixture(scope="class") + def models(self): + return { + "MODEL.sql": model_sql, + } + + +class BaseCachingSelectedSchemaOnly(BaseCachingTest): + @pytest.fixture(scope="class") + def models(self): + return { + "model.sql": model_sql, + "another_schema_model.sql": another_schema_model_sql, + } + + def test_cache(self, project): + # this should only cache the schema containing the selected model + run_args = ["--cache-selected-only", "run", "--select", "model"] + self.run_and_inspect_cache(project, run_args) + + +class TestNoPopulateCache(BaseCachingTest): + @pytest.fixture(scope="class") + def models(self): + return { + "model.sql": model_sql, + } + + def test_cache(self, project): + # --no-populate-cache still allows the cache to populate all relations + # under a schema, so the behavior here remains the same as other tests + run_args = ["--no-populate-cache", "run"] + self.run_and_inspect_cache(project, run_args) + + +class TestCachingLowerCaseModelSynapse(BaseCachingLowercaseModel): + pass + + +@pytest.mark.skip(reason="Synapse DW does not support Case Insensivity.") +class TestCachingUppercaseModelSynapse(BaseCachingUppercaseModel): + pass + + +class TestCachingSelectedSchemaOnlySynapse(BaseCachingSelectedSchemaOnly): + pass diff --git a/tests/functional/adapter/test_column_types.py b/tests/functional/adapter/test_column_types.py new file mode 100644 index 00000000..455ec252 --- /dev/null +++ b/tests/functional/adapter/test_column_types.py @@ -0,0 +1,126 @@ +import pytest +from dbt.tests.util import run_dbt + +model_sql = """ +select + cast(1 as smallint) as smallint_col, + cast(2 as integer) as int_col, + cast(3 as bigint) as bigint_col, + cast(4.0 as real) as real_col, + cast(5.0 as double precision) as double_col, + cast(6.0 as numeric) as numeric_col, + cast('7' as varchar(20)) as varchar_col +""" + +schema_yml = """ +version: 2 +models: + - name: model + tests: + - is_type: + column_map: + smallint_col: ['integer', 'number'] + int_col: ['integer', 'number'] + bigint_col: ['integer', 'number'] + real_col: ['float', 'number'] + double_col: ['float', 'number'] + numeric_col: ['numeric', 'number'] + varchar_col: ['string', 'not number'] +""" + +macro_test_is_type_sql = """ +{% macro simple_type_check_column(column, check) %} + {% if check == 'string' %} + {{ return(column.is_string()) }} + {% elif check == 'float' %} + {{ return(column.is_float()) }} + {% elif check == 'number' %} + {{ return(column.is_number()) }} + {% elif check == 'numeric' %} + {{ return(column.is_numeric()) }} + {% elif check == 'integer' %} + {{ return(column.is_integer()) }} + {% else %} + {% do exceptions.raise_compiler_error('invalid type check value: ' ~ check) %} + {% endif %} +{% endmacro %} + +{% macro type_check_column(column, type_checks) %} + {% set failures = [] %} + {% for type_check in type_checks %} + {% if type_check.startswith('not ') %} + {% if simple_type_check_column(column, type_check[4:]) %} + {% do log('simple_type_check_column got ', True) %} + {% do failures.append(type_check) %} + {% endif %} + {% else %} + {% if not simple_type_check_column(column, type_check) %} + {% do failures.append(type_check) %} + {% endif %} + {% endif %} + {% endfor %} + {% if (failures | length) > 0 %} + {% do log('column ' ~ column.name ~ ' had failures: ' ~ failures, info=True) %} + {% endif %} + {% do return((failures | length) == 0) %} +{% endmacro %} + +{% test is_type(model, column_map) %} + {% if not execute %} + {{ return(None) }} + {% endif %} + {% if not column_map %} + {% do exceptions.raise_compiler_error('test_is_type must have a column name') %} + {% endif %} + {% set columns = adapter.get_columns_in_relation(model) %} + {% if (column_map | length) != (columns | length) %} + {% set column_map_keys = (column_map | list | string) %} + {% set column_names = (columns | map(attribute='name') | list | string) %} + {% do exceptions.raise_compiler_error('did not get all the columns/all columns not + specified:\n' ~ column_map_keys ~ '\nvs\n' ~ column_names) %} + {% endif %} + {% set bad_columns = [] %} + {% for column in columns %} + {% set column_key = (column.name | lower) %} + {% if column_key in column_map %} + {% set type_checks = column_map[column_key] %} + {% if not type_checks %} + {% do exceptions.raise_compiler_error('no type checks?') %} + {% endif %} + {% if not type_check_column(column, type_checks) %} + {% do bad_columns.append(column.name) %} + {% endif %} + {% else %} + {% do exceptions.raise_compiler_error('column key ' ~ column_key ~ ' + not found in ' ~ (column_map | list | string)) %} + {% endif %} + {% endfor %} + {% do log('bad columns: ' ~ bad_columns, info=True) %} + {% for bad_column in bad_columns %} + select '{{ bad_column }}' as bad_column + {{ 'union all' if not loop.last }} + {% endfor %} + select top 0 1 as nothing +{% endtest %} +""" + + +class BaseColumnTypes: + @pytest.fixture(scope="class") + def macros(self): + return {"test_is_type.sql": macro_test_is_type_sql} + + def run_and_test(self): + results = run_dbt(["run"]) + assert len(results) == 1 + results = run_dbt(["test"]) + assert len(results) == 1 + + +class TestBaseColumnTypesSynapse(BaseColumnTypes): + @pytest.fixture(scope="class") + def models(self): + return {"model.sql": model_sql, "schema.yml": schema_yml} + + def test_run_and_test(self, project): + self.run_and_test() diff --git a/tests/functional/adapter/test_constraints.py b/tests/functional/adapter/test_constraints.py new file mode 100644 index 00000000..fe93747b --- /dev/null +++ b/tests/functional/adapter/test_constraints.py @@ -0,0 +1,707 @@ +import pytest +from dbt.tests.adapter.constraints.fixtures import ( + foreign_key_model_sql, + model_data_type_schema_yml, + my_incremental_model_sql, + my_model_data_type_sql, + my_model_incremental_with_nulls_sql, + my_model_incremental_wrong_name_sql, + my_model_incremental_wrong_order_depends_on_fk_sql, + my_model_incremental_wrong_order_sql, + my_model_sql, + my_model_view_wrong_name_sql, + my_model_view_wrong_order_sql, + my_model_with_nulls_sql, + my_model_with_quoted_column_name_sql, + my_model_wrong_name_sql, + my_model_wrong_order_depends_on_fk_sql, + my_model_wrong_order_sql, +) +from dbt.tests.adapter.constraints.test_constraints import ( + BaseConstraintsRuntimeDdlEnforcement, + BaseContractSqlHeader, + BaseModelConstraintsRuntimeEnforcement, +) +from dbt.tests.util import ( + get_manifest, + relation_from_name, + run_dbt, + run_dbt_and_capture, + write_file, +) + +model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + contract: + enforced: true + constraints: + - type: primary_key + columns: [id] + name: pk_my_model_pk + columns: + - name: id + data_type: int + description: hello + constraints: + - type: not_null + - type: primary_key + - type: unique + tests: + - unique + - name: color + data_type: varchar(100) + - name: date_day + data_type: varchar(100) + - name: my_model_error + config: + contract: + enforced: true + columns: + - name: id + data_type: int + description: hello + constraints: + - type: not_null + - type: primary_key + tests: + - unique + - name: color + data_type: varchar(100) + - name: date_day + data_type: varchar(100) + - name: my_model_wrong_order + config: + contract: + enforced: true + constraints: + - type: unique + columns: [id] + name: uk_my_model_pk + columns: + - name: id + data_type: int + description: hello + constraints: + - type: not_null + - type: primary_key + tests: + - unique + - name: color + data_type: varchar(100) + - name: date_day + data_type: varchar(100) + - name: my_model_wrong_name + config: + contract: + enforced: true + columns: + - name: id + data_type: int + description: hello + constraints: + - type: not_null + - type: primary_key + tests: + - unique + - name: color + data_type: varchar(100) + - name: date_day + data_type: varchar(100) +""" + +model_fk_constraint_schema_yml = """ +version: 2 +models: + - name: my_model + config: + contract: + enforced: true + columns: + - name: id + data_type: int + description: hello + constraints: + - type: not_null + tests: + - unique + - name: color + data_type: varchar(100) + - name: date_day + data_type: varchar(100) + - name: my_model_error + config: + contract: + enforced: true + columns: + - name: id + data_type: int + description: hello + constraints: + - type: not_null + - type: primary_key + tests: + - unique + - name: color + data_type: varchar(100) + - name: date_day + data_type: varchar(100) + - name: my_model_wrong_order + config: + contract: + enforced: true + columns: + - name: id + data_type: int + description: hello + constraints: + - type: not_null + - type: primary_key + tests: + - unique + - name: color + data_type: varchar(100) + - name: date_day + data_type: varchar(100) + - name: my_model_wrong_name + config: + contract: + enforced: true + columns: + - name: id + data_type: int + description: hello + constraints: + - type: not_null + - type: primary_key + tests: + - unique + - name: color + data_type: varchar(100) + - name: date_day + data_type: varchar(100) + - name: foreign_key_model + config: + contract: + enforced: true + constraints: + - type: primary_key + columns: [id] + name: pk_my_ref_model_id + - type: unique + name: uk_my_ref_model_id + columns: [id] + columns: + - name: id + data_type: int + constraints: + - type: not_null +""" + +constrained_model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + contract: + enforced: true + constraints: + - type: primary_key + columns: [ id ] + name: strange_pk_requirement_my_model + - type: unique + columns: [ color, date_day ] + name: strange_uniqueness_requirement_my_model + - type: foreign_key + columns: [ id ] + expression: {schema}.foreign_key_model (id) + name: strange_pk_fk_requirement_my_model + columns: + - name: id + data_type: int + description: hello + constraints: + - type: not_null + tests: + - unique + - name: color + data_type: varchar(100) + - name: date_day + data_type: varchar(100) + - name: foreign_key_model + config: + contract: + enforced: true + constraints: + - type: primary_key + columns: [ id ] + name: strange_pk_requirement_fk_my_model + - type: unique + columns: [ id ] + name: fk_id_uniqueness_requirement + columns: + - name: id + data_type: int + constraints: + - type: not_null +""" + +model_contract_header_schema_yml = """ +version: 2 +models: + - name: my_model_contract_sql_header + config: + contract: + enforced: true + columns: + - name: column_name + data_type: int +""" + + +# no current_timezone() in Synapse +my_model_contract_sql_header_sql = """ +{{ + config( + materialized = "table" + ) +}} + +{% call set_sql_header(config) %} +set session time zone 'Asia/Kolkata'; +{%- endcall %} +select datepart(tzoffset, sysdatetimeoffset()) as column_name +""" + +my_model_incremental_contract_sql_header_sql = """ +{{ + config( + materialized = "incremental", + on_schema_change="append_new_columns" + ) +}} + +{% call set_sql_header(config) %} +set session time zone 'Asia/Kolkata'; +{%- endcall %} +select datepart(tzoffset, sysdatetimeoffset()) as column_name +""" + +model_quoted_column_schema_yml = """ +version: 2 +models: + - name: my_model + config: + contract: + enforced: true + materialized: table + constraints: + - type: check + # this one is the on the user + expression: ("from" = 'blue') + columns: [ '"from"' ] + columns: + - name: id + data_type: integer + description: hello + constraints: + - type: not_null + tests: + - unique + - name: from # reserved word + quote: true + data_type: varchar(100) + constraints: + - type: not_null + - name: date_day + data_type: varchar(100) +""" + + +class BaseConstraintsColumnsEqual: + """ + dbt should catch these mismatches during its "preflight" checks. + """ + + @pytest.fixture + def string_type(self): + return "varchar" + + @pytest.fixture + def int_type(self): + return "int" + + @pytest.fixture + def schema_string_type(self, string_type): + return string_type + + @pytest.fixture + def schema_int_type(self, int_type): + return int_type + + @pytest.fixture + def data_types(self, schema_int_type, int_type, string_type): + # sql_column_value, schema_data_type, error_data_type + return [ + ["1", schema_int_type, int_type], + ["'1'", string_type, string_type], + ["cast('2019-01-01' as date)", "date", "date"], + ["cast(1 as bit)", "bit", "bit"], + ["cast('2013-11-03 00:00:00.000000' as datetime2(6))", "datetime2(6)", "datetime2(6)"], + ["cast(1 as decimal(5,2))", "decimal", "decimal"], + ] + + def test__constraints_wrong_column_order(self, project): + # This no longer causes an error, since we enforce yaml column order + run_dbt(["run", "-s", "my_model_wrong_order"], expect_pass=True) + manifest = get_manifest(project.project_root) + model_id = "model.test.my_model_wrong_order" + my_model_config = manifest.nodes[model_id].config + contract_actual_config = my_model_config.contract + + assert contract_actual_config.enforced is True + + def test__constraints_wrong_column_names(self, project, string_type, int_type): + _, log_output = run_dbt_and_capture( + ["run", "-s", "my_model_wrong_name"], expect_pass=False + ) + manifest = get_manifest(project.project_root) + model_id = "model.test.my_model_wrong_name" + my_model_config = manifest.nodes[model_id].config + contract_actual_config = my_model_config.contract + + assert contract_actual_config.enforced is True + + expected = ["id", "error", "missing in definition", "missing in contract"] + assert all([(exp in log_output or exp.upper() in log_output) for exp in expected]) + + def test__constraints_wrong_column_data_types( + self, project, string_type, int_type, schema_string_type, schema_int_type, data_types + ): + for sql_column_value, schema_data_type, error_data_type in data_types: + # Write parametrized data_type to sql file + write_file( + my_model_data_type_sql.format(sql_value=sql_column_value), + "models", + "my_model_data_type.sql", + ) + + # Write wrong data_type to corresponding schema file + # Write integer type for all schema yaml values except when testing integer type itself + wrong_schema_data_type = ( + schema_int_type + if schema_data_type.upper() != schema_int_type.upper() + else schema_string_type + ) + wrong_schema_error_data_type = ( + int_type if schema_data_type.upper() != schema_int_type.upper() else string_type + ) + write_file( + model_data_type_schema_yml.format(data_type=wrong_schema_data_type), + "models", + "constraints_schema.yml", + ) + + results, log_output = run_dbt_and_capture( + ["run", "-s", "my_model_data_type"], expect_pass=False + ) + manifest = get_manifest(project.project_root) + model_id = "model.test.my_model_data_type" + my_model_config = manifest.nodes[model_id].config + contract_actual_config = my_model_config.contract + + assert contract_actual_config.enforced is True + expected = [ + "wrong_data_type_column_name", + error_data_type, + wrong_schema_error_data_type, + "data type mismatch", + ] + assert all([(exp in log_output or exp.upper() in log_output) for exp in expected]) + + def test__constraints_correct_column_data_types(self, project, data_types): + for sql_column_value, schema_data_type, _ in data_types: + # Write parametrized data_type to sql file + write_file( + my_model_data_type_sql.format(sql_value=sql_column_value), + "models", + "my_model_data_type.sql", + ) + # Write correct data_type to corresponding schema file + write_file( + model_data_type_schema_yml.format(data_type=schema_data_type), + "models", + "constraints_schema.yml", + ) + + run_dbt(["run", "-s", "my_model_data_type"]) + + manifest = get_manifest(project.project_root) + model_id = "model.test.my_model_data_type" + my_model_config = manifest.nodes[model_id].config + contract_actual_config = my_model_config.contract + + assert contract_actual_config.enforced is True + + +class BaseConstraintsRuntimeDdlEnforcement(BaseConstraintsRuntimeDdlEnforcement): + """ + These constraints pass muster for dbt's preflight checks. Make sure they're + passed into the DDL statement. If they don't match up with the underlying data, + the data platform should raise an error at runtime. + """ + + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_wrong_order_depends_on_fk_sql, + "foreign_key_model.sql": foreign_key_model_sql, + "constraints_schema.yml": model_fk_constraint_schema_yml, + } + + @pytest.fixture(scope="class") + def expected_sql(self): + return """ + if object_id is not null begin drop view end + if object_id is not null begin drop table end + exec('create view as -- depends_on: + select ''blue'' as color,1 as id,''2019-01-01'' as date_day;'); + create table ([id] int not null,[color] varchar(100),[date_day] varchar(100)) + with(distribution = round_robin,heap) + insert into ([id],[color],[date_day]) + select [id],[color],[date_day] from + if object_id is not null begin drop view end + """ + + +class BaseIncrementalConstraintsRuntimeDdlEnforcement(BaseConstraintsRuntimeDdlEnforcement): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_incremental_wrong_order_depends_on_fk_sql, + "foreign_key_model.sql": foreign_key_model_sql, + "constraints_schema.yml": model_fk_constraint_schema_yml, + } + + +class BaseModelConstraintsRuntimeEnforcement(BaseModelConstraintsRuntimeEnforcement): + """ + These model-level constraints pass muster for dbt's preflight checks. Make sure they're + passed into the DDL statement. If they don't match up with the underlying data, + the data platform should raise an error at runtime. + """ + + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_wrong_order_depends_on_fk_sql, + "foreign_key_model.sql": foreign_key_model_sql, + "constraints_schema.yml": constrained_model_schema_yml, + } + + @pytest.fixture(scope="class") + def expected_sql(self): + return """ + if object_id is not null begin drop view end + if object_id is not null begin drop table end + exec('create view as -- depends_on: + select ''blue'' as color,1 as id,''2019-01-01'' as date_day;'); + create table ([id] int not null,[color] varchar(100),[date_day] varchar(100)) + with(distribution = round_robin,heap) + alter table add constraint + primary key nonclustered(id)not enforced; + alter table add constraint + unique nonclustered(color,date_day)not enforced; + insert into ([id],[color],[date_day]) + select [id],[color],[date_day] from + if object_id is not null begin drop view end + """ + + +class BaseTableConstraintsColumnsEqual(BaseConstraintsColumnsEqual): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_wrong_order.sql": my_model_wrong_order_sql, + "my_model_wrong_name.sql": my_model_wrong_name_sql, + "constraints_schema.yml": model_schema_yml, + } + + +class BaseViewConstraintsColumnsEqual(BaseConstraintsColumnsEqual): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_wrong_order.sql": my_model_view_wrong_order_sql, + "my_model_wrong_name.sql": my_model_view_wrong_name_sql, + "constraints_schema.yml": model_schema_yml, + } + + +class BaseIncrementalConstraintsColumnsEqual(BaseConstraintsColumnsEqual): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_wrong_order.sql": my_model_incremental_wrong_order_sql, + "my_model_wrong_name.sql": my_model_incremental_wrong_name_sql, + "constraints_schema.yml": model_schema_yml, + } + + +class BaseConstraintsRollback: + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_sql, + "constraints_schema.yml": model_schema_yml, + } + + @pytest.fixture(scope="class") + def null_model_sql(self): + return my_model_with_nulls_sql + + @pytest.fixture(scope="class") + def expected_color(self): + return "blue" + + @pytest.fixture(scope="class") + def expected_error_messages(self): + return [ + "Cannot insert the value NULL into column", + "column does not allow nulls", + "There is already an object", + ] + + def assert_expected_error_messages(self, error_message, expected_error_messages): + assert any(msg in error_message for msg in expected_error_messages) + + def test__constraints_enforcement_rollback( + self, project, expected_color, expected_error_messages, null_model_sql + ): + results = run_dbt(["run", "-s", "my_model"]) + assert len(results) == 1 + + # Make a contract-breaking change to the model + write_file(null_model_sql, "models", "my_model.sql") + + failing_results = run_dbt(["run", "-s", "my_model"], expect_pass=False) + assert len(failing_results) == 1 + + # Verify the previous table still exists + relation = relation_from_name(project.adapter, "my_model") + old_model_exists_sql = f"select * from {relation}" + old_model_exists = project.run_sql(old_model_exists_sql, fetch="all") + assert len(old_model_exists) == 1 + assert old_model_exists[0][1] == expected_color + + # Confirm this model was contracted + # TODO: is this step really necessary? + manifest = get_manifest(project.project_root) + model_id = "model.test.my_model" + my_model_config = manifest.nodes[model_id].config + contract_actual_config = my_model_config.contract + assert contract_actual_config.enforced is True + + # Its result includes the expected error messages + self.assert_expected_error_messages(failing_results[0].message, expected_error_messages) + + +class BaseIncrementalConstraintsRollback(BaseConstraintsRollback): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_incremental_model_sql, + "constraints_schema.yml": model_schema_yml, + } + + @pytest.fixture(scope="class") + def null_model_sql(self): + return my_model_incremental_with_nulls_sql + + +class BaseTableContractSqlHeader(BaseContractSqlHeader): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_contract_sql_header.sql": my_model_contract_sql_header_sql, + "constraints_schema.yml": model_contract_header_schema_yml, + } + + +class BaseIncrementalContractSqlHeader(BaseContractSqlHeader): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_contract_sql_header.sql": my_model_incremental_contract_sql_header_sql, + "constraints_schema.yml": model_contract_header_schema_yml, + } + + +class BaseConstraintQuotedColumn(BaseConstraintsRuntimeDdlEnforcement): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_with_quoted_column_name_sql, + "constraints_schema.yml": model_quoted_column_schema_yml, + } + + @pytest.fixture(scope="class") + def expected_sql(self): + return """ + if object_id is not null begin drop view end + if object_id is not null begin drop table end + exec(\'create view as select \'\'blue\'\' as "from",1 as id,\'\'2019-01-01\'\' as date_day;\'); + create table ([id] integer not null,[from] varchar(100)not null,[date_day] varchar(100)) + with(distribution = round_robin,heap) + insert into ([id],[from],[date_day]) + select [id],[from],[date_day] from + if object_id is not null begin drop view end + """ + + +class TestTableConstraintsRuntimeDdlEnforcementSynapse(BaseConstraintsRuntimeDdlEnforcement): + pass + + +class TestIncrementalConstraintsRuntimeDdlEnforcementSynapse( + BaseIncrementalConstraintsRuntimeDdlEnforcement +): + pass + + +class TestModelConstraintsRuntimeEnforcementSynapse(BaseModelConstraintsRuntimeEnforcement): + pass + + +class TestTableConstraintsColumnsEqualSynapse(BaseTableConstraintsColumnsEqual): + pass + + +class TestViewConstraintsColumnsEquaSynapse(BaseViewConstraintsColumnsEqual): + pass + + +class TestIncrementalConstraintsColumnsEqualSynapse(BaseIncrementalConstraintsColumnsEqual): + pass + + +class TestTableConstraintsRollbackSynapse(BaseConstraintsRollback): + pass + + +class TestIncrementalConstraintsRollbackSynapse(BaseIncrementalConstraintsRollback): + pass + + +class TestTableContractSqlHeaderSynapse(BaseTableContractSqlHeader): + pass + + +class TestIncrementalContractSqlHeaderSynapse(BaseIncrementalContractSqlHeader): + pass + + +class TestConstraintQuotedColumnSynapse(BaseConstraintQuotedColumn): + pass diff --git a/tests/functional/adapter/test_copy_uppercase.py b/tests/functional/adapter/test_copy_uppercase.py new file mode 100644 index 00000000..1e425082 --- /dev/null +++ b/tests/functional/adapter/test_copy_uppercase.py @@ -0,0 +1,54 @@ +import pytest +from dbt.tests.adapter.simple_copy.fixtures import ( + _MODELS__ADVANCED_INCREMENTAL, + _MODELS__COMPOUND_SORT, + _MODELS__DISABLED, + _MODELS__EMPTY, + _MODELS__INCREMENTAL, + _MODELS__INTERLEAVED_SORT, + _MODELS__MATERIALIZED, + _MODELS__VIEW_MODEL, + _MODELS_GET_AND_REF_UPPERCASE, + _PROPERTIES__SCHEMA_YML, + _SEEDS__SEED_INITIAL, +) +from dbt.tests.util import check_relations_equal, run_dbt + + +class TestSimpleCopyUppercase: + @pytest.fixture(scope="class") + def models(self): + return { + "ADVANCED_INCREMENTAL.sql": _MODELS__ADVANCED_INCREMENTAL, + "COMPOUND_SORT.sql": _MODELS__COMPOUND_SORT, + "DISABLED.sql": _MODELS__DISABLED, + "EMPTY.sql": _MODELS__EMPTY, + "GET_AND_REF.sql": _MODELS_GET_AND_REF_UPPERCASE, + "INCREMENTAL.sql": _MODELS__INCREMENTAL, + "INTERLEAVED_SORT.sql": _MODELS__INTERLEAVED_SORT, + "MATERIALIZED.sql": _MODELS__MATERIALIZED, + "VIEW_MODEL.sql": _MODELS__VIEW_MODEL, + } + + @pytest.fixture(scope="class") + def properties(self): + return { + "schema.yml": _PROPERTIES__SCHEMA_YML, + } + + @pytest.fixture(scope="class") + def seeds(self): + return {"seed.csv": _SEEDS__SEED_INITIAL} + + def test_simple_copy_uppercase(self, project): + # Load the seed file and check that it worked + results = run_dbt(["seed"]) + assert len(results) == 1 + + # Run the project and ensure that all the models loaded + results = run_dbt() + assert len(results) == 7 + + check_relations_equal( + project.adapter, ["seed", "VIEW_MODEL", "INCREMENTAL", "MATERIALIZED", "GET_AND_REF"] + ) diff --git a/tests/functional/adapter/test_dbt_clone.py b/tests/functional/adapter/test_dbt_clone.py index 62e564f1..d2b72abb 100644 --- a/tests/functional/adapter/test_dbt_clone.py +++ b/tests/functional/adapter/test_dbt_clone.py @@ -1,5 +1,238 @@ -from dbt.tests.adapter.dbt_clone.test_dbt_clone import TestCloneSameTargetAndState +import os +import shutil +from collections import Counter +from copy import deepcopy +import pytest +from dbt.exceptions import DbtRuntimeError +from dbt.tests.adapter.dbt_clone.fixtures import ( + custom_can_clone_tables_false_macros_sql, + ephemeral_model_sql, + exposures_yml, + get_schema_name_sql, + infinite_macros_sql, + macros_sql, + schema_yml, + seed_csv, + snapshot_sql, + table_model_sql, + view_model_sql, +) +from dbt.tests.util import run_dbt + + +class BaseClone: + @pytest.fixture(scope="class") + def models(self): + return { + "table_model.sql": table_model_sql, + "view_model.sql": view_model_sql, + "ephemeral_model.sql": ephemeral_model_sql, + "schema.yml": schema_yml, + "exposures.yml": exposures_yml, + } + + @pytest.fixture(scope="class") + def macros(self): + return { + "macros.sql": macros_sql, + "infinite_macros.sql": infinite_macros_sql, + "get_schema_name.sql": get_schema_name_sql, + } + + @pytest.fixture(scope="class") + def seeds(self): + return { + "seed.csv": seed_csv, + } + + @pytest.fixture(scope="class") + def snapshots(self): + return { + "snapshot.sql": snapshot_sql, + } + + @pytest.fixture(scope="class") + def other_schema(self, unique_schema): + return unique_schema + "_other" + + @property + def project_config_update(self): + return { + "seeds": { + "test": { + "quote_columns": False, + } + } + } + + @pytest.fixture(scope="class") + def profiles_config_update(self, dbt_profile_target, unique_schema, other_schema): + outputs = {"default": dbt_profile_target, "otherschema": deepcopy(dbt_profile_target)} + outputs["default"]["schema"] = unique_schema + outputs["otherschema"]["schema"] = other_schema + return {"test": {"outputs": outputs, "target": "default"}} + + def copy_state(self, project_root): + state_path = os.path.join(project_root, "state") + if not os.path.exists(state_path): + os.makedirs(state_path) + shutil.copyfile( + f"{project_root}/target/manifest.json", f"{project_root}/state/manifest.json" + ) + + def run_and_save_state(self, project_root, with_snapshot=False): + results = run_dbt(["seed"]) + assert len(results) == 1 + assert not any(r.node.deferred for r in results) + results = run_dbt(["run"]) + assert len(results) == 2 + assert not any(r.node.deferred for r in results) + results = run_dbt(["test"]) + assert len(results) == 2 + + if with_snapshot: + results = run_dbt(["snapshot"]) + assert len(results) == 1 + assert not any(r.node.deferred for r in results) + + # copy files + self.copy_state(project_root) + + +# -- Below we define base classes for tests you import the one based on +# -- if your adapter uses dbt clone or not -- +class BaseClonePossible(BaseClone): + @pytest.mark.skip(reason="Synapse does not support cloning from different state") + def test_can_clone_true(self, project, unique_schema, other_schema): + project.create_test_schema(other_schema) + self.run_and_save_state(project.project_root, with_snapshot=True) + + clone_args = [ + "clone", + "--state", + "state", + "--target", + "otherschema", + ] + + results = run_dbt(clone_args) + assert len(results) == 4 + + schema_relations = project.adapter.list_relations( + database=project.database, schema=other_schema + ) + types = [r.type for r in schema_relations] + count_types = Counter(types) + assert count_types == Counter({"table": 3, "view": 1}) + + # objects already exist, so this is a no-op + results = run_dbt(clone_args) + assert len(results) == 4 + assert all("ok" in r.message.lower() for r in results) + + # recreate all objects + results = run_dbt([*clone_args, "--full-refresh"]) + assert len(results) == 4 + + # select only models this time + results = run_dbt([*clone_args, "--resource-type", "model"]) + assert len(results) == 2 + assert all("ok" in r.message.lower() for r in results) + + def test_clone_no_state(self, project, unique_schema, other_schema): + project.create_test_schema(other_schema) + self.run_and_save_state(project.project_root, with_snapshot=True) + + clone_args = [ + "clone", + "--target", + "otherschema", + ] + + with pytest.raises( + DbtRuntimeError, + match="--state or --defer-state are required for deferral, but neither was provided", + ): + run_dbt(clone_args) + + +class BaseCloneNotPossible(BaseClone): + @pytest.fixture(scope="class") + def macros(self): + return { + "macros.sql": macros_sql, + "my_can_clone_tables.sql": custom_can_clone_tables_false_macros_sql, + "infinite_macros.sql": infinite_macros_sql, + "get_schema_name.sql": get_schema_name_sql, + } + + def test_can_clone_false(self, project, unique_schema, other_schema): + project.create_test_schema(other_schema) + self.run_and_save_state(project.project_root, with_snapshot=True) + + clone_args = [ + "clone", + "--state", + "state", + "--target", + "otherschema", + ] + + results = run_dbt(clone_args) + assert len(results) == 4 + + schema_relations = project.adapter.list_relations( + database=project.database, schema=other_schema + ) + assert all(r.type == "view" for r in schema_relations) + + # objects already exist, so this is a no-op + results = run_dbt(clone_args) + assert len(results) == 4 + assert all("ok" in r.message.lower() for r in results) + + # recreate all objects + results = run_dbt([*clone_args, "--full-refresh"]) + assert len(results) == 4 + + # select only models this time + results = run_dbt([*clone_args, "--resource-type", "model"]) + assert len(results) == 2 + assert all("ok" in r.message.lower() for r in results) + + +class TestCloneNotPossibleSynapse(BaseCloneNotPossible): + @pytest.fixture(autouse=True) + def clean_up(self, project): + yield + with project.adapter.connection_named("__test"): + relation = project.adapter.Relation.create( + database=project.database, schema=f"{project.test_schema}_seeds" + ) + project.adapter.drop_schema(relation) + + relation = project.adapter.Relation.create( + database=project.database, schema=project.test_schema + ) + project.adapter.drop_schema(relation) + + pass + + +class TestClonePossibleSynapse(BaseClonePossible): + @pytest.fixture(autouse=True) + def clean_up(self, project): + yield + with project.adapter.connection_named("__test"): + relation = project.adapter.Relation.create( + database=project.database, schema=f"{project.test_schema}_seeds" + ) + project.adapter.drop_schema(relation) + + relation = project.adapter.Relation.create( + database=project.database, schema=project.test_schema + ) + project.adapter.drop_schema(relation) -class TestCloneSameTargetAndState(TestCloneSameTargetAndState): pass diff --git a/tests/functional/adapter/test_dbt_show.py b/tests/functional/adapter/test_dbt_show.py new file mode 100644 index 00000000..6cad1411 --- /dev/null +++ b/tests/functional/adapter/test_dbt_show.py @@ -0,0 +1,70 @@ +import pytest +from dbt.tests.adapter.dbt_show.test_dbt_show import ( + models__sample_model, + models__sql_header, + seeds__sample_seed, +) +from dbt.tests.util import run_dbt + +models__sample_model_a = """ +select + coalesce(sample_num, 0) + 10 as col_deci +from {{ ref('sample_model') }} +""" + +models__sample_model_b = """ +select + col_deci + 100 as col_hundo +from {{ ref('sample_model_a') }} +""" + + +# Synapse doesn't support ephemeral models so we need to alter the base tests +class BaseShowLimit: + @pytest.fixture(scope="class") + def models(self): + return { + "sample_model.sql": models__sample_model, + "sample_model_a.sql": models__sample_model_a, + } + + @pytest.fixture(scope="class") + def seeds(self): + return {"sample_seed.csv": seeds__sample_seed} + + @pytest.mark.parametrize( + "args,expected", + [ + ([], 5), # default limit + (["--limit", 3], 3), # fetch 3 rows + (["--limit", -1], 7), # fetch all rows + ], + ) + def test_limit(self, project, args, expected): + run_dbt(["build"]) + dbt_args = ["show", "--inline", models__sample_model_b, *args] + results = run_dbt(dbt_args) + assert len(results.results[0].agate_table) == expected + # ensure limit was injected in compiled_code when limit specified in command args + limit = results.args.get("limit") + if limit > 0: + assert f"top {limit}" in results.results[0].node.compiled_code + + +class BaseShowSqlHeader: + @pytest.fixture(scope="class") + def models(self): + return { + "sql_header.sql": models__sql_header, + } + + def test_sql_header(self, project): + run_dbt(["show", "--select", "sql_header", "--vars", "timezone: Asia/Kolkata"]) + + +class TestShowSqlHeaderSynapse(BaseShowSqlHeader): + pass + + +class TestShowLimitSynapse(BaseShowLimit): + pass diff --git a/tests/functional/adapter/test_debug.py b/tests/functional/adapter/test_debug.py new file mode 100644 index 00000000..57063e6b --- /dev/null +++ b/tests/functional/adapter/test_debug.py @@ -0,0 +1,82 @@ +import os +import re + +import pytest +import yaml +from dbt.cli.exceptions import DbtUsageException +from dbt.tests.adapter.dbt_debug.test_dbt_debug import BaseDebug, BaseDebugProfileVariable +from dbt.tests.util import run_dbt, run_dbt_and_capture + + +class TestDebugSynapse(BaseDebug): + def test_ok(self, project): + run_dbt(["debug"]) + assert "ERROR" not in self.capsys.readouterr().out + + def test_nopass(self, project): + run_dbt(["debug", "--target", "nopass"], expect_pass=False) + self.assertGotValue(re.compile(r"\s+profiles\.yml file"), "ERROR invalid") + + def test_connection_flag(self, project): + """Testing the --connection flag works as expected, including that output is not lost""" + _, out = run_dbt_and_capture(["debug", "--connection"]) + assert "Skipping steps before connection verification" in out + + _, out = run_dbt_and_capture( + ["debug", "--connection", "--target", "NONE"], expect_pass=False + ) + assert "1 check failed" in out + assert "The profile 'test' does not have a target named 'NONE'." in out + + _, out = run_dbt_and_capture( + ["debug", "--connection", "--profiles-dir", "NONE"], expect_pass=False + ) + assert "Using profiles dir at NONE" + assert "1 check failed" in out + assert "dbt looked for a profiles.yml file in NONE" in out + + def test_wronguser(self, project): + run_dbt(["debug", "--target", "wronguser"], expect_pass=False) + self.assertGotValue(re.compile(r"\s+Connection test"), "ERROR") + + def test_empty_target(self, project): + run_dbt(["debug", "--target", "none_target"], expect_pass=False) + self.assertGotValue(re.compile(r"\s+output 'none_target'"), "misconfigured") + + +class TestDebugProfileVariableSynapse(BaseDebugProfileVariable): + pass + + +class TestDebugInvalidProjectSynapse(BaseDebug): + def test_empty_project(self, project): + with open("dbt_project.yml", "w") as f: # noqa: F841 + pass + + run_dbt(["debug", "--profile", "test"], expect_pass=False) + splitout = self.capsys.readouterr().out.split("\n") + self.check_project(splitout) + + def test_badproject(self, project): + update_project = {"invalid-key": "not a valid key so this is bad project"} + + with open("dbt_project.yml", "w") as f: + yaml.safe_dump(update_project, f) + + run_dbt(["debug", "--profile", "test"], expect_pass=False) + splitout = self.capsys.readouterr().out.split("\n") + self.check_project(splitout) + + def test_not_found_project(self, project): + with pytest.raises(DbtUsageException): + run_dbt(["debug", "--project-dir", "nopass"]) + + def test_invalid_project_outside_current_dir(self, project): + # create a dbt_project.yml + project_config = {"invalid-key": "not a valid key in this project"} + os.makedirs("custom", exist_ok=True) + with open("custom/dbt_project.yml", "w") as f: + yaml.safe_dump(project_config, f, default_flow_style=True) + run_dbt(["debug", "--project-dir", "custom"], expect_pass=False) + splitout = self.capsys.readouterr().out.split("\n") + self.check_project(splitout) diff --git a/tests/functional/adapter/test_equals.py b/tests/functional/adapter/test_equals.py new file mode 100644 index 00000000..f2f496b4 --- /dev/null +++ b/tests/functional/adapter/test_equals.py @@ -0,0 +1,5 @@ +from dbt.tests.adapter.utils.test_equals import BaseEquals + + +class TestEqualsSynapse(BaseEquals): + pass diff --git a/tests/functional/adapter/test_grants.py b/tests/functional/adapter/test_grants.py index 026512d0..060087d0 100644 --- a/tests/functional/adapter/test_grants.py +++ b/tests/functional/adapter/test_grants.py @@ -13,7 +13,7 @@ class TestIncrementalGrantsSynapse(BaseIncrementalGrants): pass -class TestInvalidGrantsSQLServer(BaseInvalidGrants): +class TestInvalidGrantsSynapse(BaseInvalidGrants): def grantee_does_not_exist_error(self): return "Cannot find the user" diff --git a/tests/functional/adapter/test_materialized_views.py b/tests/functional/adapter/test_materialized_views.py new file mode 100644 index 00000000..06e86220 --- /dev/null +++ b/tests/functional/adapter/test_materialized_views.py @@ -0,0 +1,212 @@ +import pytest +from dbt.tests.adapter.materialized_view.basic import MaterializedViewBasic +from dbt.tests.util import ( + assert_message_in_logs, + check_relation_types, + get_model_file, + run_dbt, + run_dbt_and_capture, + set_model_file, +) + +MY_TABLE = """ +{{ config( + materialized='table', +) }} +select i.id, count(i.value) as counted +from {{ ref('my_seed') }} i +group by i.id +""" + + +MY_VIEW = """ +{{ config( + materialized='view', +) }} +select i.id, count(i.value) as counted +from {{ ref('my_seed') }} i +group by i.id +""" + + +MY_MATERIALIZED_VIEW = """ +{{ config( + materialized='materialized_view', +) }} +select i.id, count(*) as counted +from {{ ref('my_seed') }} i +group by i.id +""" + + +def drop_cascade(project, test_model_identifier): + # SYNAPSE HAS NO "DROP SCHEMA...CASCADE" + # so drop all test materializations, to allow drop my_seed + # "my_materialized_view" always created in setup(), so always need to be dropped before my_seed + for identifier in ["my_materialized_view", test_model_identifier]: + project.run_sql( + f""" + if object_id ('"{project.test_schema}"."{identifier}"','V') is not null + begin + drop view "{project.test_schema}"."{identifier}" + end + + if object_id ('"{project.test_schema}"."{identifier}"','U') is not null + begin + drop table "{project.test_schema}"."{identifier}" + end + """ + ) + # then drop object my_seed, to allow drop schema + project.run_sql( + f""" + if object_id ('"{project.test_schema}"."my_seed"','U') is not null + begin + drop table "{project.test_schema}"."my_seed" + end + """ + ) + # finally drop schema can proceed in setup function + + +class TestMaterializedViewsBasicSynapse(MaterializedViewBasic): + @pytest.fixture(scope="class", autouse=True) + def models(self): + yield { + "my_table.sql": MY_TABLE, + "my_view.sql": MY_VIEW, + "my_materialized_view.sql": MY_MATERIALIZED_VIEW, + } + + @pytest.fixture(scope="function", autouse=True) + def setup(self, project, my_materialized_view): + run_dbt(["seed"]) + run_dbt(["run", "--models", my_materialized_view.identifier, "--full-refresh"]) + + # the tests touch these files, store their contents in memory + initial_model = get_model_file(project, my_materialized_view) + + yield + + # and then reset them after the test runs + set_model_file(project, my_materialized_view, initial_model) + # Synapse no support "if exists" and "cascade" + project.run_sql(f"drop schema {project.test_schema}") + + def test_materialized_view_create(self, project): + # check relation types + expected = { + # sys.objects has no type "materialized view", it's type "view" + "my_materialized_view": "view", + } + check_relation_types(project.adapter, expected) + + drop_cascade(project, "my_materialized_view") + + def test_materialized_view_create_idempotent(self, project, my_materialized_view): + # setup creates it once; verify it's there and run once + expected = { + # sys.objects has no type "materialized view", it's type "view" + "my_materialized_view": "view", + } + check_relation_types(project.adapter, expected) + + run_dbt(["run", "--models", my_materialized_view.identifier]) + expected = { + # sys.objects has no type "materialized view", it's type "view" + my_materialized_view.identifier: "view", + } + check_relation_types(project.adapter, expected) + + drop_cascade(project, my_materialized_view.identifier) + + def test_materialized_view_full_refresh(self, project, my_materialized_view): + _, logs = run_dbt_and_capture( + ["--debug", "run", "--models", my_materialized_view.identifier, "--full-refresh"] + ) + expected = { + # sys.objects has no type "materialized view", it's type "view" + my_materialized_view.identifier: "view", + } + check_relation_types(project.adapter, expected) + assert_message_in_logs(f"Applying REPLACE to: {my_materialized_view}", logs) + + drop_cascade(project, my_materialized_view.identifier) + + def test_materialized_view_replaces_table(self, project, my_table): + run_dbt(["run", "--models", my_table.identifier]) + expected = { + my_table.identifier: "table", + } + check_relation_types(project.adapter, expected) + + self.swap_table_to_materialized_view(project, my_table) + + run_dbt(["run", "--models", my_table.identifier]) + expected = { + # sys.objects has no type "materialized view", it's type "view" + my_table.identifier: "view", + } + check_relation_types(project.adapter, expected) + + drop_cascade(project, my_table.identifier) + + def test_materialized_view_replaces_view(self, project, my_view): + run_dbt(["run", "--models", my_view.identifier]) + expected = { + my_view.identifier: "view", + } + check_relation_types(project.adapter, expected) + + self.swap_view_to_materialized_view(project, my_view) + + run_dbt(["run", "--models", my_view.identifier]) + expected = { + # sys.objects has no type "materialized view", it's type "view" + my_view.identifier: "view", + } + check_relation_types(project.adapter, expected) + + drop_cascade(project, my_view.identifier) + + def test_table_replaces_materialized_view(self, project, my_materialized_view): + run_dbt(["run", "--models", my_materialized_view.identifier]) + expected = { + # sys.objects has no type "materialized view", it's type "view" + my_materialized_view.identifier: "view", + } + check_relation_types(project.adapter, expected) + + self.swap_materialized_view_to_table(project, my_materialized_view) + + run_dbt(["run", "--models", my_materialized_view.identifier]) + expected = { + my_materialized_view.identifier: "table", + } + check_relation_types(project.adapter, expected) + + drop_cascade(project, my_materialized_view.identifier) + + def test_view_replaces_materialized_view(self, project, my_materialized_view): + run_dbt(["run", "--models", my_materialized_view.identifier]) + expected = { + # sys.objects has no type "materialized view", it's type "view" + my_materialized_view.identifier: "view", + } + check_relation_types(project.adapter, expected) + + self.swap_materialized_view_to_view(project, my_materialized_view) + + run_dbt(["run", "--models", my_materialized_view.identifier]) + expected = { + my_materialized_view.identifier: "view", + } + check_relation_types(project.adapter, expected) + + drop_cascade(project, my_materialized_view.identifier) + + @pytest.mark.skip(reason="Synapse materialized view is always updated") + def test_materialized_view_only_updates_after_refresh( + self, project, my_materialized_view, my_seed + ): + pass diff --git a/tests/functional/adapter/test_model_hooks.py b/tests/functional/adapter/test_model_hooks.py new file mode 100644 index 00000000..d765705c --- /dev/null +++ b/tests/functional/adapter/test_model_hooks.py @@ -0,0 +1,227 @@ +import pytest +from dbt.tests.adapter.hooks.fixtures import ( + models__hooks, + models__hooks_configured, + models__hooks_kwargs, +) +from dbt.tests.adapter.hooks.test_model_hooks import ( + MODEL_POST_HOOK, + MODEL_PRE_HOOK, + BaseTestPrePost, + TestDuplicateHooksInConfigs, + TestHooksRefsOnSeeds, + TestPrePostModelHooksOnSeeds, + TestPrePostModelHooksOnSeedsPlusPrefixed, + TestPrePostModelHooksOnSeedsPlusPrefixedWhitespace, + TestPrePostModelHooksOnSnapshots, + TestPrePostSnapshotHooksInConfigKwargs, +) +from dbt.tests.util import run_dbt + + +class BaseTestPrePostSynapse(BaseTestPrePost): + def check_hooks(self, state, project, host, count=1): + ctxs = self.get_ctx_vars(state, count=count, project=project) + for ctx in ctxs: + assert ctx["test_state"] == state + assert ctx["target_dbname"] == "" + assert ctx["target_host"] == "" + assert ctx["target_name"] == "default" + assert ctx["target_schema"] == project.test_schema + assert ctx["target_threads"] == 1 + assert ctx["target_type"] == "synapse" + assert ctx["target_user"] == "None" + assert ctx["target_pass"] == "" + + assert ( + ctx["run_started_at"] is not None and len(ctx["run_started_at"]) > 0 + ), "run_started_at was not set" + assert ( + ctx["invocation_id"] is not None and len(ctx["invocation_id"]) > 0 + ), "invocation_id was not set" + assert ctx["thread_id"].startswith("Thread-") + + +class PrePostModelHooksInConfigSetup(BaseTestPrePostSynapse): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "macro-paths": ["macros"], + } + + @pytest.fixture(scope="class") + def models(self): + return {"hooks.sql": models__hooks_configured} + + +class TestHookRefs(BaseTestPrePostSynapse): + pass + + +class TestPrePostModelHooksOnSeeds(TestPrePostModelHooksOnSeeds): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seed-paths": ["seeds"], + "models": {}, + "seeds": { + "post-hook": [ + "alter table {{ this }} add new_col int", + "update {{ this }} set new_col = 1", + # call any macro to track dependency: + # https://github.com/dbt-labs/dbt-core/issues/6806 + "select cast(null as {{ dbt.type_int() }}) as id", + ], + "quote_columns": False, + }, + } + + +class TestHooksRefsOnSeeds(TestHooksRefsOnSeeds): + pass + + +# +class TestPrePostModelHooksOnSeedsPlusPrefixed(TestPrePostModelHooksOnSeedsPlusPrefixed): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seed-paths": ["seeds"], + "models": {}, + "seeds": { + "+post-hook": [ + "alter table {{ this }} add new_col int", + "update {{ this }} set new_col = 1", + ], + "quote_columns": False, + }, + } + + +class TestPrePostModelHooksOnSeedsPlusPrefixedWhitespace( + TestPrePostModelHooksOnSeedsPlusPrefixedWhitespace +): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seed-paths": ["seeds"], + "models": {}, + "seeds": { + "+post-hook": [ + "alter table {{ this }} add new_col int", + "update {{ this }} set new_col = 1", + ], + "quote_columns": False, + }, + } + + +class TestPrePostModelHooksOnSnapshots(TestPrePostModelHooksOnSnapshots): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seed-paths": ["seeds"], + "snapshot-paths": ["test-snapshots"], + "models": {}, + "snapshots": { + "post-hook": [ + "alter table {{ this }} add new_col int", + "update {{ this }} set new_col = 1", + ] + }, + "seeds": { + "quote_columns": False, + }, + } + + +class TestPrePostModelHooksInConfig(PrePostModelHooksInConfigSetup): + def test_pre_and_post_model_hooks_model(self, project, dbt_profile_target): + run_dbt() + + self.check_hooks("start", project, dbt_profile_target.get("host", None)) + self.check_hooks("end", project, dbt_profile_target.get("host", None)) + + +class TestPrePostModelHooksInConfigKwargs(TestPrePostModelHooksInConfig): + @pytest.fixture(scope="class") + def models(self): + return {"hooks.sql": models__hooks_kwargs} + + +class TestPrePostSnapshotHooksInConfigKwargs(TestPrePostSnapshotHooksInConfigKwargs): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seed-paths": ["seeds"], + "snapshot-paths": ["test-kwargs-snapshots"], + "models": {}, + "snapshots": { + "post-hook": [ + "alter table {{ this }} add new_col int", + "update {{ this }} set new_col = 1", + ] + }, + "seeds": { + "quote_columns": False, + }, + } + + +class TestDuplicateHooksInConfigs(TestDuplicateHooksInConfigs): + pass + + +# vacuum command is removed because not supported in synapse +class TestPrePostModelHooks(BaseTestPrePostSynapse): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "test": { + "pre-hook": [MODEL_PRE_HOOK], + "post-hook": [MODEL_POST_HOOK], + } + } + } + + @pytest.fixture(scope="class") + def models(self): + return {"hooks.sql": models__hooks} + + def test_pre_and_post_run_hooks(self, project, dbt_profile_target): + run_dbt() + self.check_hooks("start", project, dbt_profile_target.get("host", None)) + self.check_hooks("end", project, dbt_profile_target.get("host", None)) + + +class TestPrePostModelHooksInConfigWithCount(PrePostModelHooksInConfigSetup): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "test": { + "pre-hook": [MODEL_PRE_HOOK], + "post-hook": [MODEL_POST_HOOK], + } + } + } + + def test_pre_and_post_model_hooks_model_and_project(self, project, dbt_profile_target): + run_dbt() + + self.check_hooks("start", project, dbt_profile_target.get("host", None), count=2) + self.check_hooks("end", project, dbt_profile_target.get("host", None), count=2) + + +@pytest.mark.skip(reason="Not supporting underscores config") +class TestPrePostModelHooksUnderscores(TestPrePostModelHooks): + def project_config_update(self): + return { + "models": { + "test": { + "pre_hook": [MODEL_PRE_HOOK], + "post_hook": [MODEL_POST_HOOK], + } + } + } diff --git a/tests/functional/adapter/test_new_project.py b/tests/functional/adapter/test_new_project.py index b5eef440..7a9fadf1 100644 --- a/tests/functional/adapter/test_new_project.py +++ b/tests/functional/adapter/test_new_project.py @@ -62,7 +62,7 @@ """ -class TestNewProjectSQLServer: +class TestNewProjectSynapse: @pytest.fixture(scope="class") def project_config_update(self): return {"name": "my_new_project"} diff --git a/tests/functional/adapter/test_null_compare.py b/tests/functional/adapter/test_null_compare.py new file mode 100644 index 00000000..e4f51020 --- /dev/null +++ b/tests/functional/adapter/test_null_compare.py @@ -0,0 +1,9 @@ +from dbt.tests.adapter.utils.test_null_compare import BaseMixedNullCompare, BaseNullCompare + + +class TestMixedNullCompareSynapse(BaseMixedNullCompare): + pass + + +class TestNullCompareSynapse(BaseNullCompare): + pass diff --git a/tests/functional/adapter/test_persist_docs.py b/tests/functional/adapter/test_persist_docs.py new file mode 100644 index 00000000..e45f90cb --- /dev/null +++ b/tests/functional/adapter/test_persist_docs.py @@ -0,0 +1,21 @@ +import pytest +from dbt.tests.adapter.persist_docs.test_persist_docs import ( + BasePersistDocs, + BasePersistDocsColumnMissing, + BasePersistDocsCommentOnQuotedColumn, +) + + +@pytest.mark.skip(reason="Synapse does not support adding/updating extended properties") +class TestPersistDocsSynapse(BasePersistDocs): + pass + + +@pytest.mark.skip(reason="Synapse does not support adding/updating extended properties") +class TestPersistDocsColumnMissingSynapse(BasePersistDocsColumnMissing): + pass + + +@pytest.mark.skip(reason="Synapse does not support adding/updating extended properties") +class TestPersistDocsCommentOnQuotedColumnSynapse(BasePersistDocsCommentOnQuotedColumn): + pass diff --git a/tests/functional/adapter/test_run_hooks.py b/tests/functional/adapter/test_run_hooks.py new file mode 100644 index 00000000..cf2a32ee --- /dev/null +++ b/tests/functional/adapter/test_run_hooks.py @@ -0,0 +1,80 @@ +import os +from pathlib import Path + +import pytest +from dbt.tests.adapter.hooks.test_run_hooks import TestAfterRunHooks, TestPrePostRunHooks + + +class TestPrePostRunHooks(TestPrePostRunHooks): + @pytest.fixture(scope="function") + def setUp(self, project): + project.run_sql_file(project.test_data_dir / Path("seed_run.sql")) + project.run_sql( + f""" + if object_id ('{ project.test_schema }.schemas') is not null + drop table { project.test_schema }.schemas + """ + ) + project.run_sql( + f""" + if object_id ('{ project.test_schema }.db_schemas') is not null + drop table { project.test_schema }.db_schemas + """ + ) + os.environ["TERM_TEST"] = "TESTING" + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + # The create and drop table statements here validate that these hooks run + # in the same order that they are defined. Drop before create is an error. + # Also check that the table does not exist below. + "on-run-start": [ + "{{ custom_run_hook('start', target, run_started_at, invocation_id) }}", + "create table {{ target.schema }}.start_hook_order_test ( id int )", + "drop table {{ target.schema }}.start_hook_order_test", + "{{ log(env_var('TERM_TEST'), info=True) }}", + ], + "on-run-end": [ + "{{ custom_run_hook('end', target, run_started_at, invocation_id) }}", + "create table {{ target.schema }}.end_hook_order_test ( id int )", + "drop table {{ target.schema }}.end_hook_order_test", + "create table {{ target.schema }}.schemas ( sch varchar(100) )", + """insert into {{ target.schema }}.schemas (sch) values + {% for schema in schemas %}( '{{ schema }}' ) + {% if not loop.last %},{% endif %}{% endfor %}""", + """create table {{ target.schema }}.db_schemas + ( db varchar(100), sch varchar(100) )""", + """insert into {{ target.schema }}.db_schemas (db, sch) values + {% for db, schema in database_schemas %}('{{ db }}', '{{ schema }}' ) + {% if not loop.last %},{% endif %}{% endfor %}""", + ], + "seeds": { + "quote_columns": False, + }, + } + + def check_hooks(self, state, project, host): + ctx = self.get_ctx_vars(state, project) + + assert ctx["test_state"] == state + assert ctx["target_dbname"] == "" + assert ctx["target_host"] == "" + assert ctx["target_name"] == "default" + assert ctx["target_schema"] == project.test_schema + assert ctx["target_threads"] == 1 + assert ctx["target_type"] == "synapse" + assert ctx["target_user"] == "None" + assert ctx["target_pass"] == "" + + assert ( + ctx["run_started_at"] is not None and len(ctx["run_started_at"]) > 0 + ), "run_started_at was not set" + assert ( + ctx["invocation_id"] is not None and len(ctx["invocation_id"]) > 0 + ), "invocation_id was not set" + assert ctx["thread_id"].startswith("Thread-") or ctx["thread_id"] == "MainThread" + + +class TestAfterRunHooks(TestAfterRunHooks): + pass diff --git a/tests/functional/adapter/test_simple_copy.py b/tests/functional/adapter/test_simple_copy.py new file mode 100644 index 00000000..bda9fcfb --- /dev/null +++ b/tests/functional/adapter/test_simple_copy.py @@ -0,0 +1,177 @@ +from pathlib import Path + +import pytest +from dbt.adapters.factory import get_adapter_by_type +from dbt.tests.adapter.simple_copy.fixtures import _SEEDS__SEED_UPDATE +from dbt.tests.adapter.simple_copy.test_simple_copy import SimpleCopySetup +from dbt.tests.util import ( + check_relations_equal, + get_connection, + rm_file, + run_dbt, + run_sql_with_adapter, + write_file, +) + + +class TestProjInfoSynapse: + __test__ = False + + def __init__( + self, + project_root, + profiles_dir, + adapter_type, + test_dir, + shared_data_dir, + test_data_dir, + test_schema, + database, + test_config, + ): + self.project_root = project_root + self.profiles_dir = profiles_dir + self.adapter_type = adapter_type + self.test_dir = test_dir + self.shared_data_dir = shared_data_dir + self.test_data_dir = test_data_dir + self.test_schema = test_schema + self.database = database + self.test_config = test_config + self.created_schemas = [] + + @property + def adapter(self): + # This returns the last created "adapter" from the adapter factory. Each + # dbt command will create a new one. This allows us to avoid patching the + # providers 'get_adapter' function. + return get_adapter_by_type(self.adapter_type) + + # Run sql from a path + def run_sql_file(self, sql_path, fetch=None): + with open(sql_path, "r") as f: + statements = f.read().split(";") + for statement in statements: + self.run_sql(statement, fetch) + + # Run sql from a string, using adapter saved at test startup + def run_sql(self, sql, fetch=None): + return run_sql_with_adapter(self.adapter, sql, fetch=fetch) + + # Create the unique test schema. Used in test setup, so that we're + # ready for initial sql prior to a run_dbt command. + def create_test_schema(self, schema_name=None): + if schema_name is None: + schema_name = self.test_schema + with get_connection(self.adapter): + relation = self.adapter.Relation.create(database=self.database, schema=schema_name) + self.adapter.create_schema(relation) + self.created_schemas.append(schema_name) + + # Drop the unique test schema, usually called in test cleanup + def drop_test_schema(self): + with get_connection(self.adapter): + for schema_name in self.created_schemas: + relation = self.adapter.Relation.create(database=self.database, schema=schema_name) + self.adapter.drop_schema(relation) + self.created_schemas = [] + + # This return a dictionary of table names to 'view' or 'table' values. + # Override class because Synapse doesnt have 'ILIKE' + def synapse_get_tables_in_schema(self): + sql = """ + select table_name, + case when table_type = 'BASE TABLE' then 'table' + when table_type = 'VIEW' then 'view' + else table_type + end as materialization + from information_schema.tables + where {} + order by table_name + """ + sql = sql.format("{} like '{}'".format("table_schema", self.test_schema)) + result = self.run_sql(sql, fetch="all") + return {model_name: materialization for (model_name, materialization) in result} + + +# create new project fixture replacing the syntax-incompatible method +@pytest.fixture +def synapse_project(project): + # Replace the original class with the new one + project.__class__ = TestProjInfoSynapse + + return project + + +class SimpleCopyBase(SimpleCopySetup): + def test_simple_copy(self, synapse_project): + # Load the seed file and check that it worked + results = run_dbt(["seed"]) + assert len(results) == 1 + + # Run the synapse_project and ensure that all the models loaded + results = run_dbt() + assert len(results) == 7 + check_relations_equal( + synapse_project.adapter, + ["seed", "view_model", "incremental", "materialized", "get_and_ref"], + ) + + # Change the seed.csv file and see if everything is the same, + # i.e. everything has been updated + main_seed_file = synapse_project.project_root / Path("seeds") / Path("seed.csv") + rm_file(main_seed_file) + write_file(_SEEDS__SEED_UPDATE, main_seed_file) + results = run_dbt(["seed"]) + assert len(results) == 1 + results = run_dbt() + assert len(results) == 7 + check_relations_equal( + synapse_project.adapter, + ["seed", "view_model", "incremental", "materialized", "get_and_ref"], + ) + + # in Synapse materialized views must be created with aggregation and distribution option + def test_simple_copy_with_materialized_views(self, synapse_project): + synapse_project.run_sql( + f"create table {synapse_project.test_schema}.unrelated_table (id int)" + ) + sql = f""" + create materialized view {synapse_project.test_schema}.unrelated_materialized_view + with ( distribution = round_robin ) as ( + select id from {synapse_project.test_schema}.unrelated_table group by id + ) + """ + synapse_project.run_sql(sql) + sql = f""" + create view {synapse_project.test_schema}.unrelated_view as ( + select id from {synapse_project.test_schema}.unrelated_materialized_view + ) + """ + synapse_project.run_sql(sql) + results = run_dbt(["seed"]) + assert len(results) == 1 + results = run_dbt() + assert len(results) == 7 + + +class EmptyModelsArentRunBaseSynapse(SimpleCopySetup): + def test_dbt_doesnt_run_empty(self, synapse_project): + results = run_dbt(["seed"]) + assert len(results) == 1 + results = run_dbt() + assert len(results) == 7 + + # Overwriting the original method with the custom implementation + tables = synapse_project.synapse_get_tables_in_schema() + + assert "empty" not in tables.keys() + assert "disabled" not in tables.keys() + + +class TestSimpleCopyBaseSynapse(SimpleCopyBase): + pass + + +class TestEmptyModelsArentRunSynapse(EmptyModelsArentRunBaseSynapse): + pass diff --git a/tests/functional/adapter/test_validate_sql.py b/tests/functional/adapter/test_validate_sql.py new file mode 100644 index 00000000..b370b01e --- /dev/null +++ b/tests/functional/adapter/test_validate_sql.py @@ -0,0 +1,5 @@ +from dbt.tests.adapter.utils.test_validate_sql import BaseValidateSqlMethod + + +class TestValidateSqlMethodSynapse(BaseValidateSqlMethod): + pass