From 521748fb84297aa942d80a87e61446fb1fa19c65 Mon Sep 17 00:00:00 2001 From: Petro Tiurin <93913847+ptiurin@users.noreply.github.com> Date: Fri, 23 Aug 2024 13:27:14 +0100 Subject: [PATCH] feat(FIR-29284): dbt 1.7 support (#134) --- .../unreleased/Added-20240822-111700.yaml | 3 + dbt/adapters/firebolt/impl.py | 24 ++++- dbt/adapters/firebolt/relation.py | 22 +++- .../firebolt/relation_configs/__init__.py | 0 dbt/include/firebolt/macros/adapters.sql | 84 --------------- dbt/include/firebolt/macros/catalog.sql | 98 +++++++++++++++-- .../materializations/materialized_view.sql | 45 -------- .../relations/materialized_view/alter.sql | 11 ++ .../relations/materialized_view/create.sql | 3 + .../relations/materialized_view/describe.sql | 3 + .../relations/materialized_view/drop.sql | 3 + .../relations/materialized_view/refresh.sql | 3 + .../macros/relations/table/create.sql | 74 +++++++++++++ .../firebolt/macros/relations/table/drop.sql | 3 + .../macros/relations/table/rename.sql | 6 ++ .../macros/relations/table/replace.sql | 3 + .../firebolt/macros/relations/view/create.sql | 16 +++ .../firebolt/macros/relations/view/drop.sql | 3 + .../firebolt/macros/relations/view/rename.sql | 6 ++ .../macros/relations/view/replace.sql | 3 + tests/functional/adapter/test_clone.py | 23 +++- tests/functional/adapter/test_limit.py | 12 +++ tests/functional/adapter/test_override.py | 40 +++++++ tests/functional/adapter/test_seed.py | 51 +++++++++ .../adapter/test_store_test_failures.py | 34 ++++++ tests/functional/adapter/utils/test_utils.py | 101 ++++++++++++++++-- 26 files changed, 522 insertions(+), 152 deletions(-) create mode 100644 .changes/unreleased/Added-20240822-111700.yaml create mode 100644 dbt/adapters/firebolt/relation_configs/__init__.py create mode 100644 dbt/include/firebolt/macros/relations/materialized_view/alter.sql create mode 100644 dbt/include/firebolt/macros/relations/materialized_view/create.sql create mode 100644 dbt/include/firebolt/macros/relations/materialized_view/describe.sql create mode 100644 dbt/include/firebolt/macros/relations/materialized_view/drop.sql create mode 100644 dbt/include/firebolt/macros/relations/materialized_view/refresh.sql create mode 100644 dbt/include/firebolt/macros/relations/table/create.sql create mode 100644 dbt/include/firebolt/macros/relations/table/drop.sql create mode 100644 dbt/include/firebolt/macros/relations/table/rename.sql create mode 100644 dbt/include/firebolt/macros/relations/table/replace.sql create mode 100644 dbt/include/firebolt/macros/relations/view/create.sql create mode 100644 dbt/include/firebolt/macros/relations/view/drop.sql create mode 100644 dbt/include/firebolt/macros/relations/view/rename.sql create mode 100644 dbt/include/firebolt/macros/relations/view/replace.sql create mode 100644 tests/functional/adapter/test_limit.py create mode 100644 tests/functional/adapter/test_override.py create mode 100644 tests/functional/adapter/test_seed.py create mode 100644 tests/functional/adapter/test_store_test_failures.py diff --git a/.changes/unreleased/Added-20240822-111700.yaml b/.changes/unreleased/Added-20240822-111700.yaml new file mode 100644 index 000000000..468ed00c9 --- /dev/null +++ b/.changes/unreleased/Added-20240822-111700.yaml @@ -0,0 +1,3 @@ +kind: Added +body: dbt 1.7 support with tests and refactoring +time: 2024-08-22T11:17:00.770973+01:00 diff --git a/dbt/adapters/firebolt/impl.py b/dbt/adapters/firebolt/impl.py index 5ff5bcb06..37977554e 100644 --- a/dbt/adapters/firebolt/impl.py +++ b/dbt/adapters/firebolt/impl.py @@ -5,10 +5,17 @@ from typing import Any, List, Mapping, Optional, Union import agate -from dbt.adapters.base import available -from dbt.adapters.base.impl import AdapterConfig, ConstraintSupport +from dbt.adapters.base import available # type: ignore +from dbt.adapters.base.impl import AdapterConfig # type: ignore +from dbt.adapters.base.impl import ConstraintSupport from dbt.adapters.base.relation import BaseRelation -from dbt.adapters.sql import SQLAdapter +from dbt.adapters.capability import ( + Capability, + CapabilityDict, + CapabilitySupport, + Support, +) +from dbt.adapters.sql import SQLAdapter # type: ignore from dbt.contracts.graph.nodes import ConstraintType from dbt.dataclass_schema import ValidationError, dbtClassMixin from dbt.exceptions import ( @@ -114,6 +121,17 @@ class FireboltAdapter(SQLAdapter): ConstraintType.foreign_key: ConstraintSupport.NOT_SUPPORTED, } + _capabilities: CapabilityDict = CapabilityDict( + { + Capability.SchemaMetadataByRelations: CapabilitySupport( + support=Support.Full + ), + Capability.TableLastModifiedMetadata: CapabilitySupport( + support=Support.Unsupported + ), + } + ) + @classmethod def is_cancelable(cls) -> bool: return False diff --git a/dbt/adapters/firebolt/relation.py b/dbt/adapters/firebolt/relation.py index 10b9ee57e..37d43dc56 100644 --- a/dbt/adapters/firebolt/relation.py +++ b/dbt/adapters/firebolt/relation.py @@ -1,7 +1,9 @@ from dataclasses import dataclass, field -from typing import Optional +from typing import Dict, FrozenSet, Optional -from dbt.adapters.base.relation import BaseRelation, Policy +from dbt.adapters.base import RelationType # type: ignore +from dbt.adapters.base.relation import BaseRelation, Policy # type: ignore +from dbt.adapters.relation_configs import RelationConfigBase # type: ignore from dbt.exceptions import DbtRuntimeError @@ -30,6 +32,22 @@ class FireboltRelation(BaseRelation): quote_character: str = '"' is_delta: Optional[bool] = None information: Optional[str] = None + relation_configs: Dict[RelationType, RelationConfigBase] = field( + default_factory=lambda: {} + ) + # list relations that can be renamed (e.g. `RENAME my_relation TO my_new_name;`) + renameable_relations: FrozenSet[RelationType] = field( + default_factory=lambda: frozenset({}) + ) + # list relations that can be atomically replaced + # (e.g. `CREATE OR REPLACE my_relation..` versus `DROP` and `CREATE`) + replaceable_relations: FrozenSet[RelationType] = field( + default_factory=lambda: frozenset( + { + RelationType.View, + } + ) + ) def render(self) -> str: if self.include_policy.database and self.include_policy.schema: diff --git a/dbt/adapters/firebolt/relation_configs/__init__.py b/dbt/adapters/firebolt/relation_configs/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/dbt/include/firebolt/macros/adapters.sql b/dbt/include/firebolt/macros/adapters.sql index 9a3fc6cd2..139fd03b8 100644 --- a/dbt/include/firebolt/macros/adapters.sql +++ b/dbt/include/firebolt/macros/adapters.sql @@ -183,90 +183,6 @@ {{ return(info_table) }} {% endmacro %} - -{% macro firebolt__create_table_as(temporary, - relation, - select_sql, - language='sql') -%} - {# Create table using CTAS - Args: - temporary (bool): Unused, included so macro signature matches - that of dbt's default macro - relation (dbt relation/dict) - select_sql (string): The SQL query that will be used to generate - the internal query of the CTAS - language (string): sql or python models. Firebolt only supports sql. - #} - {%- if language == 'python' -%} - {{ exceptions.raise_compiler_error("Firebolt does not currently support " - "Python models.") }} - {%- elif language not in ['python', 'sql'] -%} - {{ exceptions.raise_compiler_error("Unexpected language parameter supplied: %s " - "Must be either 'sql' or 'python'." % (language)) }} - {%- endif -%} - - {%- set table_type = config.get('table_type', default='dimension') | upper -%} - {%- set primary_index = config.get('primary_index') -%} - {%- set incremental_strategy = config.get('incremental_strategy') -%} - {%- set partitions = config.get('partition_by') %} - - CREATE {{ table_type }} TABLE IF NOT EXISTS {{ relation }} - {%- set contract_config = config.get('contract') -%} - {%- if contract_config.enforced -%} - {{ get_assert_columns_equivalent(select_sql) }} - {{ get_table_columns_and_constraints() }} ; - insert into {{ relation }} ( - {{ adapter.dispatch('get_column_names', 'dbt')() }} - ) - {%- set select_sql = get_select_subquery(select_sql) %} - {% endif %} - {%- if primary_index %} - PRIMARY INDEX - {% if primary_index is iterable and primary_index is not string %} - {{ primary_index | join(', ') }} - {%- else -%} - {{ primary_index }} - {%- endif -%} - {%- endif -%} - {% if partitions %} - PARTITION BY - {% if partitions is iterable and partitions is not string %} - {{ partitions | join(', ') }} - {%- else -%} - {{ partitions }} - {%- endif -%} - {%- endif %} - {%- if not contract_config.enforced %} - AS ( - {% endif -%} - {{ select_sql }} - {% if not contract_config.enforced -%} - ) - {%- endif -%} -{% endmacro %} - - -{% macro firebolt__create_view_as(relation, select_sql) %} - {#- - Return SQL string to create view. - Args: - relation (dict): dbt relation - select_sql (string): The SQL query that will be used to generate - the internal query of the CTAS - #} - - CREATE OR REPLACE VIEW {{ relation.identifier }} - - {%- set contract_config = config.get('contract') -%} - {%- if contract_config.enforced -%} - {{ get_assert_columns_equivalent(select_sql) }} - {%- endif %} - AS ( - {{ select_sql }} - ) -{% endmacro %} - - {% macro firebolt__truncate_relation(relation) -%} {# Truncate relation. Actual macro is drop_relation in ./adapters/relation.sql. diff --git a/dbt/include/firebolt/macros/catalog.sql b/dbt/include/firebolt/macros/catalog.sql index b09f1020c..77f098b62 100644 --- a/dbt/include/firebolt/macros/catalog.sql +++ b/dbt/include/firebolt/macros/catalog.sql @@ -1,23 +1,103 @@ {# This is for building docs. Right now it's an incomplete description of the columns (for instance, `is_nullable` is missing) but more could be added later. #} -{% macro firebolt__get_catalog(information_schemas, schemas) -%} - {%- call statement('catalog', fetch_result=True) %} +{% macro firebolt__get_catalog(information_schema, schemas) -%} + + {% set query %} + with tables as ( + {{ firebolt__get_catalog_tables_sql(information_schema) }} + {{ firebolt__get_catalog_schemas_where_clause_sql(schemas) }} + ), + columns as ( + {{ firebolt__get_catalog_columns_sql(information_schema) }} + {{ firebolt__get_catalog_schemas_where_clause_sql(schemas) }} + ) + {{ firebolt__get_catalog_results_sql() }} + {%- endset -%} + + {{ return(run_query(query)) }} + +{%- endmacro %} + +{% macro firebolt__get_catalog_relations(information_schema, relations) -%} + + {% set query %} + with tables as ( + {{ firebolt__get_catalog_tables_sql(information_schema) }} + {{ firebolt__get_catalog_relations_where_clause_sql(relations) }} + ), + columns as ( + {{ firebolt__get_catalog_columns_sql(information_schema) }} + {{ firebolt__get_catalog_relations_where_clause_sql(relations) }} + ) + {{ firebolt__get_catalog_results_sql() }} + {%- endset -%} + + {{ return(run_query(query)) }} + +{%- endmacro %} + + +{% macro firebolt__get_catalog_tables_sql(information_schema) -%} SELECT tbls.table_catalog AS table_database, tbls.table_schema as table_schema, table_type, tbls.table_name as table_name, - cols.column_name as column_name, - cols.data_type AS column_type, CASE WHEN table_type = 'VIEW' THEN 'VIEW' ELSE 'TABLE' - END AS relation_type, - cols.ordinal_position as column_index + END AS relation_type FROM information_schema.tables tbls - JOIN information_schema.columns cols USING (table_name) - {% endcall -%} - {{ return(load_result('catalog').table) }} +{%- endmacro %} + + +{% macro firebolt__get_catalog_columns_sql(information_schema) -%} + select + table_catalog as "table_database", + table_schema as "table_schema", + table_name as "table_name", + column_name as "column_name", + ordinal_position as "column_index", + data_type as "column_type" + from information_schema.columns +{%- endmacro %} + +{% macro firebolt__get_catalog_results_sql() -%} + SELECT * + FROM tables + JOIN columns USING ("table_database", "table_schema", "table_name") + ORDER BY "column_index" +{%- endmacro %} + + +{% macro firebolt__get_catalog_schemas_where_clause_sql(schemas) -%} + WHERE ({%- for schema in schemas -%} + UPPER("table_schema") = UPPER('{{ schema }}'){%- if not loop.last %} OR {% endif -%} + {%- endfor -%}) +{%- endmacro %} + + +{% macro firebolt__get_catalog_relations_where_clause_sql(relations) -%} + WHERE ( + {%- for relation in relations -%} + {% if relation.schema and relation.identifier %} + ( + UPPER("table_schema") = UPPER('{{ relation.schema }}') + AND UPPER("table_name") = UPPER('{{ relation.identifier }}') + ) + {% elif relation.schema %} + ( + UPPER("table_schema") = UPPER('{{ relation.schema }}') + ) + {% else %} + {% do exceptions.raise_compiler_error( + '`get_catalog_relations` requires a list of relations, each with a schema' + ) %} + {% endif %} + + {%- if not loop.last %} OR {% endif -%} + {%- endfor -%} + ) {%- endmacro %} diff --git a/dbt/include/firebolt/macros/materializations/materialized_view.sql b/dbt/include/firebolt/macros/materializations/materialized_view.sql index d37b3f713..24c9f238c 100644 --- a/dbt/include/firebolt/macros/materializations/materialized_view.sql +++ b/dbt/include/firebolt/macros/materializations/materialized_view.sql @@ -1,48 +1,3 @@ -{# - All these macros are stubbed with compiler errors because Firebolt does not - support materialized views. -#} -{% materialization materialized_view, adapter='firebolt' %} - - {{ exceptions.raise_compiler_error("Firebolt does not support materialized views") }} - -{% endmaterialization %} - -{% macro firebolt__get_alter_materialized_view_as_sql( - relation, - configuration_changes, - sql, - existing_relation, - backup_relation, - intermediate_relation -) %} - - {{ exceptions.raise_compiler_error("Firebolt does not support materialized views") }} -{% endmacro %} - - -{% macro firebolt__get_create_materialized_view_as_sql(relation, sql) %} - - {{ exceptions.raise_compiler_error("Firebolt does not support materialized views") }} - -{% endmacro %} - - -{% macro firebolt__get_replace_materialized_view_as_sql(relation, sql, existing_relation, backup_relation, intermediate_relation) %} - {{ exceptions.raise_compiler_error("Firebolt does not support materialized views") }} -{% endmacro %} - - {% macro firebolt__get_materialized_view_configuration_changes(existing_relation, new_config) %} {{ exceptions.raise_compiler_error("Firebolt does not support materialized views") }} {% endmacro %} - - -{% macro firebolt__refresh_materialized_view(relation) -%} - {{ exceptions.raise_compiler_error("Firebolt does not support materialized views") }} -{% endmacro %} - - -{% macro firebolt__drop_materialized_view(relation) -%} - {{ exceptions.raise_compiler_error("Firebolt does not support materialized views") }} -{%- endmacro %} diff --git a/dbt/include/firebolt/macros/relations/materialized_view/alter.sql b/dbt/include/firebolt/macros/relations/materialized_view/alter.sql new file mode 100644 index 000000000..63efe1fa3 --- /dev/null +++ b/dbt/include/firebolt/macros/relations/materialized_view/alter.sql @@ -0,0 +1,11 @@ +{% macro firebolt__get_alter_materialized_view_as_sql( + relation, + configuration_changes, + sql, + existing_relation, + backup_relation, + intermediate_relation +) %} + + {{ exceptions.raise_compiler_error("Firebolt does not support materialized views") }} +{% endmacro %} diff --git a/dbt/include/firebolt/macros/relations/materialized_view/create.sql b/dbt/include/firebolt/macros/relations/materialized_view/create.sql new file mode 100644 index 000000000..d733bf2b5 --- /dev/null +++ b/dbt/include/firebolt/macros/relations/materialized_view/create.sql @@ -0,0 +1,3 @@ +{% macro firebolt__get_create_materialized_view_as_sql(relation, sql) %} + {{ exceptions.raise_compiler_error("Firebolt does not support materialized views") }} +{% endmacro %} diff --git a/dbt/include/firebolt/macros/relations/materialized_view/describe.sql b/dbt/include/firebolt/macros/relations/materialized_view/describe.sql new file mode 100644 index 000000000..91282b27b --- /dev/null +++ b/dbt/include/firebolt/macros/relations/materialized_view/describe.sql @@ -0,0 +1,3 @@ +{% macro redshift__describe_materialized_view(relation) %} + {{ exceptions.raise_compiler_error("Firebolt does not support materialized views") }} +{% endmacro %} diff --git a/dbt/include/firebolt/macros/relations/materialized_view/drop.sql b/dbt/include/firebolt/macros/relations/materialized_view/drop.sql new file mode 100644 index 000000000..239a892a3 --- /dev/null +++ b/dbt/include/firebolt/macros/relations/materialized_view/drop.sql @@ -0,0 +1,3 @@ +{% macro firebolt__drop_materialized_view(relation) %} + {{ exceptions.raise_compiler_error("Firebolt does not support materialized views") }} +{% endmacro %} diff --git a/dbt/include/firebolt/macros/relations/materialized_view/refresh.sql b/dbt/include/firebolt/macros/relations/materialized_view/refresh.sql new file mode 100644 index 000000000..74ff6c279 --- /dev/null +++ b/dbt/include/firebolt/macros/relations/materialized_view/refresh.sql @@ -0,0 +1,3 @@ +{% macro firebolt__refresh_materialized_view(relation) %} + {{ exceptions.raise_compiler_error("Firebolt does not support materialized views") }} +{% endmacro %} diff --git a/dbt/include/firebolt/macros/relations/table/create.sql b/dbt/include/firebolt/macros/relations/table/create.sql new file mode 100644 index 000000000..5b5a47c8d --- /dev/null +++ b/dbt/include/firebolt/macros/relations/table/create.sql @@ -0,0 +1,74 @@ +{% macro firebolt__create_table_as( + temporary, + relation, + select_sql, + language = 'sql' + ) -%} + {# Create table using CTAS + Args: + temporary (bool): Unused, included so macro signature matches + that of dbt's default macro + relation (dbt relation/dict) + select_sql (string): The SQL query that will be used to generate + the internal query of the CTAS + language (string): sql or python models. Firebolt only supports sql. + #} + {%- if language == 'python' -%} + {{ exceptions.raise_compiler_error( + "Firebolt does not currently support " "Python models." + ) }} + + {%- elif language not in ['python', 'sql'] -%} + {{ exceptions.raise_compiler_error( + "Unexpected language parameter supplied: %s " "Must be either 'sql' or 'python'." % (language) + ) }} + {%- endif -%} + + {%- set table_type = config.get( + 'table_type', + default = 'dimension' + ) | upper -%} + {%- set primary_index = config.get('primary_index') -%} + {%- set incremental_strategy = config.get('incremental_strategy') -%} + {%- set partitions = config.get('partition_by') %} + CREATE {{ table_type }} TABLE IF NOT EXISTS {{ relation }} + + {%- set contract_config = config.get('contract') -%} + {%- if contract_config.enforced -%} + {{ get_assert_columns_equivalent(select_sql) }} + {{ get_table_columns_and_constraints() }}; + INSERT INTO + {{ relation }} + ( + {{ adapter.dispatch( + 'get_column_names', + 'dbt' + )() }} + ) {%- set select_sql = get_select_subquery(select_sql) %} + {% endif %} + + {%- if primary_index %} + primary INDEX {% if primary_index is iterable and primary_index is not string %} + {{ primary_index | join(', ') }} + {%- else -%} + {{ primary_index }} + {%- endif -%} + {%- endif -%} + + {% if partitions %} + PARTITION BY {% if partitions is iterable and partitions is not string %} + {{ partitions | join(', ') }} + {%- else -%} + {{ partitions }} + {%- endif -%} + {%- endif %} + + {%- if not contract_config.enforced %} + AS ( + {% endif -%} + + {{ select_sql }} + + {% if not contract_config.enforced -%}) + {%- endif -%} +{% endmacro %} diff --git a/dbt/include/firebolt/macros/relations/table/drop.sql b/dbt/include/firebolt/macros/relations/table/drop.sql new file mode 100644 index 000000000..36f83f7d0 --- /dev/null +++ b/dbt/include/firebolt/macros/relations/table/drop.sql @@ -0,0 +1,3 @@ +{%- macro firebolt__drop_table(relation) -%} + DROP TABLE IF EXISTS {{ relation }} CASCADE +{%- endmacro -%} diff --git a/dbt/include/firebolt/macros/relations/table/rename.sql b/dbt/include/firebolt/macros/relations/table/rename.sql new file mode 100644 index 000000000..ba7575e3d --- /dev/null +++ b/dbt/include/firebolt/macros/relations/table/rename.sql @@ -0,0 +1,6 @@ +{% macro firebolt__get_rename_table_sql( + relation, + new_name + ) %} + {{ exceptions.raise_compiler_error("Firebolt does not support table renames") }} +{% endmacro %} diff --git a/dbt/include/firebolt/macros/relations/table/replace.sql b/dbt/include/firebolt/macros/relations/table/replace.sql new file mode 100644 index 000000000..d8b5de6fd --- /dev/null +++ b/dbt/include/firebolt/macros/relations/table/replace.sql @@ -0,0 +1,3 @@ +{% macro firebolt__get_replace_table_sql(relation, sql) %} + {{ firebolt__create_table_as(False, relation, sql) }} +{% endmacro %} diff --git a/dbt/include/firebolt/macros/relations/view/create.sql b/dbt/include/firebolt/macros/relations/view/create.sql new file mode 100644 index 000000000..b5a218cab --- /dev/null +++ b/dbt/include/firebolt/macros/relations/view/create.sql @@ -0,0 +1,16 @@ +{% macro firebolt__create_view_as( + relation, + select_sql + ) %} + CREATE + OR REPLACE VIEW {{ relation.identifier }} + + {%- set contract_config = config.get('contract') -%} + {%- if contract_config.enforced -%} + {{ get_assert_columns_equivalent(select_sql) }} + {%- endif %} + + AS ( + {{ select_sql }} + ) +{% endmacro %} diff --git a/dbt/include/firebolt/macros/relations/view/drop.sql b/dbt/include/firebolt/macros/relations/view/drop.sql new file mode 100644 index 000000000..e96ba372e --- /dev/null +++ b/dbt/include/firebolt/macros/relations/view/drop.sql @@ -0,0 +1,3 @@ +{% macro firebolt__get_drop_view_sql(relation) %} + DROP VIEW if EXISTS {{ relation }} CASCADE +{% endmacro %} diff --git a/dbt/include/firebolt/macros/relations/view/rename.sql b/dbt/include/firebolt/macros/relations/view/rename.sql new file mode 100644 index 000000000..860576c5a --- /dev/null +++ b/dbt/include/firebolt/macros/relations/view/rename.sql @@ -0,0 +1,6 @@ +{% macro firebolt__get_rename_view_sql( + relation, + new_name + ) %} + {{ exceptions.raise_compiler_error("Firebolt does not support view renames") }} +{% endmacro %} diff --git a/dbt/include/firebolt/macros/relations/view/replace.sql b/dbt/include/firebolt/macros/relations/view/replace.sql new file mode 100644 index 000000000..1d7a0444a --- /dev/null +++ b/dbt/include/firebolt/macros/relations/view/replace.sql @@ -0,0 +1,3 @@ +{% macro firebolt__get_replace_view_sql(relation, sql) %} + {{ firebolt__create_view_as(relation, sql) }} +{% endmacro %} diff --git a/tests/functional/adapter/test_clone.py b/tests/functional/adapter/test_clone.py index 959d9ac59..e0d39995e 100644 --- a/tests/functional/adapter/test_clone.py +++ b/tests/functional/adapter/test_clone.py @@ -1,4 +1,8 @@ -from dbt.tests.adapter.dbt_clone.test_dbt_clone import BaseCloneNotPossible +from dbt.tests.adapter.dbt_clone.test_dbt_clone import ( + BaseClone, + BaseCloneNotPossible, +) +from dbt.tests.util import run_dbt_and_capture from pytest import mark @@ -6,3 +10,20 @@ @mark.skip("Can't test this before schemas are implemented") class TestFireboltCloneNotPossible(BaseCloneNotPossible): pass + + +class TestCloneSameTargetAndState(BaseClone): + def test_clone_same_target_and_state(self, project, unique_schema, other_schema): + project.create_test_schema(other_schema) + self.run_and_save_state(project.project_root) + + clone_args = [ + 'clone', + '--state', + 'target', + ] + + results, output = run_dbt_and_capture(clone_args, expect_pass=False) + assert ( + "Warning: The state and target directories are the same: 'target'" in output + ) diff --git a/tests/functional/adapter/test_limit.py b/tests/functional/adapter/test_limit.py new file mode 100644 index 000000000..0dfd20b95 --- /dev/null +++ b/tests/functional/adapter/test_limit.py @@ -0,0 +1,12 @@ +from dbt.tests.adapter.dbt_show.test_dbt_show import ( + BaseShowLimit, + BaseShowSqlHeader, +) + + +class TestFireboltShowLimit(BaseShowLimit): + pass + + +class TestFireboltShowSqlHeader(BaseShowSqlHeader): + pass diff --git a/tests/functional/adapter/test_override.py b/tests/functional/adapter/test_override.py new file mode 100644 index 000000000..220dddf02 --- /dev/null +++ b/tests/functional/adapter/test_override.py @@ -0,0 +1,40 @@ +import pytest +from dbt.exceptions import CompilationError +from dbt.tests.util import run_dbt + +model_sql = """ +select 1 as id +""" + +fail_macros__failure_sql = """ +{% macro get_catalog_relations(information_schema, relations) %} + {% do exceptions.raise_compiler_error('rejected: no catalogs for you') %} +{% endmacro %} + +""" + + +class TestDocsGenerateOverride: + """ + This essentially tests that the capability to get relations + is enabled in the adapter. + """ + + @pytest.fixture(scope='class') + def models(self): + return {'model.sql': model_sql} + + @pytest.fixture(scope='class') + def macros(self): + return {'failure.sql': fail_macros__failure_sql} + + def test_override_used( + self, + project, + ): + results = run_dbt(['run']) + assert len(results) == 1 # type: ignore + # this should pick up our failure macro and raise a compilation exception + with pytest.raises(CompilationError) as excinfo: + run_dbt(['--warn-error', 'docs', 'generate']) + assert 'rejected: no catalogs for you' in str(excinfo.value) diff --git a/tests/functional/adapter/test_seed.py b/tests/functional/adapter/test_seed.py new file mode 100644 index 000000000..7f52dd414 --- /dev/null +++ b/tests/functional/adapter/test_seed.py @@ -0,0 +1,51 @@ +from dbt.tests.adapter.simple_seed.test_seed import SeedUniqueDelimiterTestBase +from dbt.tests.util import run_dbt +from pytest import fixture + +seeds__expected_sql = """ +create table {schema}.seed_expected ( +seed_id INTEGER, +first_name TEXT, +email TEXT, +ip_address TEXT, +birthday TIMESTAMPNTZ +);""" + + +class TestSeedWithWrongDelimiter(SeedUniqueDelimiterTestBase): + @fixture(scope='class', autouse=True) + def setUp(self, project): + """Create table for ensuring seeds and models used in tests build correctly""" + project.run_sql(seeds__expected_sql) + + @fixture(scope='class') + def project_config_update(self): + return { + 'seeds': {'quote_columns': False, 'delimiter': ';'}, + } + + def test_seed_with_wrong_delimiter(self, project): + """Testing failure of running dbt seed with a wrongly configured delimiter""" + seed_result = run_dbt(['seed'], expect_pass=False) + assert 'syntax error' in seed_result.results[0].message.lower() # type: ignore + + +class TestSeedWithEmptyDelimiter(SeedUniqueDelimiterTestBase): + @fixture(scope='class', autouse=True) + def setUp(self, project): + """Create table for ensuring seeds and models used in tests build correctly""" + project.run_sql(seeds__expected_sql) + + @fixture(scope='class') + def project_config_update(self): + return { + 'seeds': {'quote_columns': False, 'delimiter': ''}, + } + + def test_seed_with_empty_delimiter(self, project): + """ + Testing failure of running dbt seed with an empty configured delimiter value + """ + seed_result = run_dbt(['seed'], expect_pass=False) + message = seed_result.results[0].message.lower() # type: ignore + assert 'compilation error' in message diff --git a/tests/functional/adapter/test_store_test_failures.py b/tests/functional/adapter/test_store_test_failures.py new file mode 100644 index 000000000..f716f2f52 --- /dev/null +++ b/tests/functional/adapter/test_store_test_failures.py @@ -0,0 +1,34 @@ +from dbt.tests.adapter.store_test_failures_tests.basic import ( + StoreTestFailuresAsExceptions, + StoreTestFailuresAsGeneric, + StoreTestFailuresAsInteractions, + StoreTestFailuresAsProjectLevelEphemeral, + StoreTestFailuresAsProjectLevelOff, + StoreTestFailuresAsProjectLevelView, +) + + +class TestStoreTestFailuresAsInteractions(StoreTestFailuresAsInteractions): + pass + + +class TestStoreTestFailuresAsProjectLevelOff(StoreTestFailuresAsProjectLevelOff): + pass + + +class TestStoreTestFailuresAsProjectLevelView(StoreTestFailuresAsProjectLevelView): + pass + + +class TestStoreTestFailuresAsGeneric(StoreTestFailuresAsGeneric): + pass + + +class TestStoreTestFailuresAsProjectLevelEphemeral( + StoreTestFailuresAsProjectLevelEphemeral +): + pass + + +class TestStoreTestFailuresAsExceptions(StoreTestFailuresAsExceptions): + pass diff --git a/tests/functional/adapter/utils/test_utils.py b/tests/functional/adapter/utils/test_utils.py index 6ebbd5fe7..9bb4f7266 100644 --- a/tests/functional/adapter/utils/test_utils.py +++ b/tests/functional/adapter/utils/test_utils.py @@ -7,6 +7,9 @@ models__test_concat_sql, models__test_concat_yml, ) +from dbt.tests.adapter.utils.fixture_date_spine import ( + models__test_date_spine_yml, +) from dbt.tests.adapter.utils.fixture_date_trunc import ( models__test_date_trunc_sql, models__test_date_trunc_yml, @@ -20,6 +23,9 @@ MODELS__EQUAL_VALUES_SQL, MODELS__NOT_EQUAL_VALUES_SQL, ) +from dbt.tests.adapter.utils.fixture_get_intervals_between import ( + models__test_get_intervals_between_yml, +) from dbt.tests.adapter.utils.fixture_hash import ( models__test_hash_sql, models__test_hash_yml, @@ -66,6 +72,7 @@ from dbt.tests.adapter.utils.test_current_timestamp import ( BaseCurrentTimestampAware, ) +from dbt.tests.adapter.utils.test_date_spine import BaseDateSpine from dbt.tests.adapter.utils.test_date_trunc import BaseDateTrunc from dbt.tests.adapter.utils.test_dateadd import BaseDateAdd from dbt.tests.adapter.utils.test_datediff import BaseDateDiff @@ -74,6 +81,11 @@ BaseEscapeSingleQuotesBackslash, ) from dbt.tests.adapter.utils.test_except import BaseExcept +from dbt.tests.adapter.utils.test_generate_series import BaseGenerateSeries +from dbt.tests.adapter.utils.test_get_intervals_between import ( + BaseGetIntervalsBetween, +) +from dbt.tests.adapter.utils.test_get_powers_of_two import BaseGetPowersOfTwo from dbt.tests.adapter.utils.test_hash import BaseHash from dbt.tests.adapter.utils.test_intersect import BaseIntersect from dbt.tests.adapter.utils.test_last_day import BaseLastDay @@ -228,14 +240,14 @@ def models(self): -- Also test correct casting of literal values. -union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "second") }} as actual, 1 as expected -union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "minute") }} as actual, 1 as expected -union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "hour") }} as actual, 1 as expected -union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "day") }} as actual, 1 as expected -union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-03 00:00:00.000000'", "week") }} as actual, 1 as expected -union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "month") }} as actual, 1 as expected -union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "quarter") }} as actual, 1 as expected -union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "year") }} as actual, 1 as expected +union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", 'second') }} as actual, 1 as expected +union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", 'minute') }} as actual, 1 as expected +union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", 'hour') }} as actual, 1 as expected +union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", 'day') }} as actual, 1 as expected +union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-03 00:00:00.000000'", 'week') }} as actual, 1 as expected +union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", 'month') }} as actual, 1 as expected +union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", 'quarter') }} as actual, 1 as expected +union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", 'year') }} as actual, 1 as expected """ @@ -529,3 +541,76 @@ class TestFireboltMixedNullCompare(BaseMixedNullCompare): class TestFireboltValidateSqlMethod(BaseValidateSqlMethod): pass + + +class TestDateSpine(BaseDateSpine): + # Override to use postgres dialect + models__test_date_spine_sql = """ + with generated_dates as ( + {{ date_spine('day', "'2023-09-01'::date", "'2023-09-10'::date") }} + ), expected_dates as ( + select '2023-09-01'::date as expected + union all + select '2023-09-02'::date as expected + union all + select '2023-09-03'::date as expected + union all + select '2023-09-04'::date as expected + union all + select '2023-09-05'::date as expected + union all + select '2023-09-06'::date as expected + union all + select '2023-09-07'::date as expected + union all + select '2023-09-08'::date as expected + union all + select '2023-09-09'::date as expected + ), joined as ( + select + generated_dates.date_day, + expected_dates.expected + from generated_dates + left join expected_dates on generated_dates.date_day = expected_dates.expected + ) + + SELECT * from joined + """ + + @pytest.fixture(scope='class') + def models(self): + return { + 'test_date_spine.yml': models__test_date_spine_yml, + 'test_date_spine.sql': self.interpolate_macro_namespace( + self.models__test_date_spine_sql, 'date_spine' + ), + } + + +class TestGenerateSeries(BaseGenerateSeries): + pass + + +class TestGetIntervalsBeteween(BaseGetIntervalsBetween): + # Override to use postgres dialect + # Also, switch from MM/DD/YYYY to DD/MM/YYYY to reflect the default for + # the Firebolt database + models__test_get_intervals_between_sql = """ + SELECT + {{ get_intervals_between("'01/09/2023'::date", "'12/09/2023'::date", 'day') }} as intervals, + 11 as expected + + """ + + @pytest.fixture(scope='class') + def models(self): + return { + 'test_get_intervals_between.yml': models__test_get_intervals_between_yml, + 'test_get_intervals_between.sql': self.interpolate_macro_namespace( + self.models__test_get_intervals_between_sql, 'get_intervals_between' + ), + } + + +class TestGetPowersOfTwo(BaseGetPowersOfTwo): + pass