From d8bbc8f18ae653f09f8cfb717fcb50bc05f24950 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 6 Jul 2023 00:08:24 -0400 Subject: [PATCH 01/12] migrating to new framework --- Makefile | 1 + dbt/adapters/redshift/materialization.py | 0 .../__init__.py | 8 +-- .../base.py | 2 +- .../materialization_config/database.py | 58 +++++++++++++++++++ .../dist.py | 2 +- .../materialized_view.py | 19 +++--- .../policy.py} | 0 .../redshift/materialization_config/schema.py | 0 .../sort.py | 2 +- dbt/adapters/redshift/relation.py | 2 +- dev-requirements.txt | 6 +- 12 files changed, 80 insertions(+), 20 deletions(-) create mode 100644 dbt/adapters/redshift/materialization.py rename dbt/adapters/redshift/{relation_configs => materialization_config}/__init__.py (56%) rename dbt/adapters/redshift/{relation_configs => materialization_config}/base.py (97%) create mode 100644 dbt/adapters/redshift/materialization_config/database.py rename dbt/adapters/redshift/{relation_configs => materialization_config}/dist.py (98%) rename dbt/adapters/redshift/{relation_configs => materialization_config}/materialized_view.py (95%) rename dbt/adapters/redshift/{relation_configs/policies.py => materialization_config/policy.py} (100%) create mode 100644 dbt/adapters/redshift/materialization_config/schema.py rename dbt/adapters/redshift/{relation_configs => materialization_config}/sort.py (98%) diff --git a/Makefile b/Makefile index 0cc3a43d6..efd23b806 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,7 @@ dev: ## Installs adapter in develop mode along with development dependencies dev-uninstall: ## Uninstalls all packages while maintaining the virtual environment ## Useful when updating versions, or if you accidentally installed into the system interpreter pip freeze | grep -v "^-e" | cut -d "@" -f1 | xargs pip uninstall -y + pip uninstall -y dbt-redshift .PHONY: mypy mypy: ## Runs mypy against staged changes for static type checking. diff --git a/dbt/adapters/redshift/materialization.py b/dbt/adapters/redshift/materialization.py new file mode 100644 index 000000000..e69de29bb diff --git a/dbt/adapters/redshift/relation_configs/__init__.py b/dbt/adapters/redshift/materialization_config/__init__.py similarity index 56% rename from dbt/adapters/redshift/relation_configs/__init__.py rename to dbt/adapters/redshift/materialization_config/__init__.py index 26e36c86c..90555568e 100644 --- a/dbt/adapters/redshift/relation_configs/__init__.py +++ b/dbt/adapters/redshift/materialization_config/__init__.py @@ -1,18 +1,18 @@ -from dbt.adapters.redshift.relation_configs.sort import ( +from dbt.adapters.redshift.materialization_config.sort import ( RedshiftSortConfig, RedshiftSortConfigChange, ) -from dbt.adapters.redshift.relation_configs.dist import ( +from dbt.adapters.redshift.materialization_config.dist import ( RedshiftDistConfig, RedshiftDistConfigChange, ) -from dbt.adapters.redshift.relation_configs.materialized_view import ( +from dbt.adapters.redshift.materialization_config.materialized_view import ( RedshiftMaterializedViewConfig, RedshiftAutoRefreshConfigChange, RedshiftBackupConfigChange, RedshiftMaterializedViewConfigChangeset, ) -from dbt.adapters.redshift.relation_configs.policies import ( +from dbt.adapters.redshift.materialization_config.policies import ( RedshiftIncludePolicy, RedshiftQuotePolicy, MAX_CHARACTERS_IN_IDENTIFIER, diff --git a/dbt/adapters/redshift/relation_configs/base.py b/dbt/adapters/redshift/materialization_config/base.py similarity index 97% rename from dbt/adapters/redshift/relation_configs/base.py rename to dbt/adapters/redshift/materialization_config/base.py index ebbd46b1b..920cde6c2 100644 --- a/dbt/adapters/redshift/relation_configs/base.py +++ b/dbt/adapters/redshift/materialization_config/base.py @@ -10,7 +10,7 @@ from dbt.contracts.graph.nodes import ModelNode from dbt.contracts.relation import ComponentName -from dbt.adapters.redshift.relation_configs.policies import ( +from dbt.adapters.redshift.materialization_config.policies import ( RedshiftIncludePolicy, RedshiftQuotePolicy, ) diff --git a/dbt/adapters/redshift/materialization_config/database.py b/dbt/adapters/redshift/materialization_config/database.py new file mode 100644 index 000000000..145311ce5 --- /dev/null +++ b/dbt/adapters/redshift/materialization_config/database.py @@ -0,0 +1,58 @@ +from dbt.adapters.materialization_config import ( + DatabaseConfig, + RelationConfigValidationMixin, + RelationConfigValidationRule, +) + +from dbt.adapters.redshift.materialization_config.policy import redshift_conform_part + + +@dataclass(frozen=True, eq=True, unsafe_hash=True) +class RedshiftDatabaseConfig(DatabaseConfig): + """ + This config follow the specs found here: + https://docs.aws.amazon.com/redshift/latest/dg/r_CREATE_DATABASE.html + + The following parameters are configurable by dbt: + - name: name of the database + """ + + name: str + + @property + def validation_rules(self) -> Set[RelationConfigValidationRule]: + return { + RelationConfigValidationRule( + validation_check=len(self.name or "") > 0, + validation_error=DbtRuntimeError( + f"dbt-redshift requires a name for a database, received: {self.name}" + ), + ) + } + + @classmethod + def from_dict(cls, config_dict: dict) -> "RedshiftDatabaseConfig": + """ + Because this returns a frozen dataclass, this method should be overridden if additional parameters are supplied. + """ + kwargs_dict = {"name": redshift_conform_part(ComponentName.Database, config_dict["name"])} + database = super().from_dict(kwargs_dict) + assert isinstance(database, RedshiftDatabaseConfig) + return database + + @classmethod + def parse_model_node(cls, model_node: ModelNode) -> dict: + """ + Because this returns a `dict`, this method should be extended if additional parameters are supplied. + """ + config_dict = {"name": model_node.database} + super().parse_model_node() + return config_dict + + @classmethod + def parse_describe_relation_results(cls, describe_relation_results: agate.Row) -> dict: + """ + Because this returns a `dict`, this method should be extended if additional parameters are supplied. + """ + config_dict = {"name": describe_relation_results["database"]} + return config_dict diff --git a/dbt/adapters/redshift/relation_configs/dist.py b/dbt/adapters/redshift/materialization_config/dist.py similarity index 98% rename from dbt/adapters/redshift/relation_configs/dist.py rename to dbt/adapters/redshift/materialization_config/dist.py index 668f3f65a..0ec3f72ae 100644 --- a/dbt/adapters/redshift/relation_configs/dist.py +++ b/dbt/adapters/redshift/materialization_config/dist.py @@ -12,7 +12,7 @@ from dbt.dataclass_schema import StrEnum from dbt.exceptions import DbtRuntimeError -from dbt.adapters.redshift.relation_configs.base import RedshiftRelationConfigBase +from dbt.adapters.redshift.materialization_config.base import RedshiftRelationConfigBase class RedshiftDistStyle(StrEnum): diff --git a/dbt/adapters/redshift/relation_configs/materialized_view.py b/dbt/adapters/redshift/materialization_config/materialized_view.py similarity index 95% rename from dbt/adapters/redshift/relation_configs/materialized_view.py rename to dbt/adapters/redshift/materialization_config/materialized_view.py index 82bc0d084..9d12e6448 100644 --- a/dbt/adapters/redshift/relation_configs/materialized_view.py +++ b/dbt/adapters/redshift/materialization_config/materialized_view.py @@ -2,31 +2,32 @@ from typing import Optional, Set import agate -from dbt.adapters.relation_configs import ( +from dbt.adapters.materialization_config import ( RelationResults, RelationConfigChange, RelationConfigValidationMixin, RelationConfigValidationRule, + MaterializationConfig, ) from dbt.contracts.graph.nodes import ModelNode from dbt.contracts.relation import ComponentName from dbt.exceptions import DbtRuntimeError -from dbt.adapters.redshift.relation_configs.base import RedshiftRelationConfigBase -from dbt.adapters.redshift.relation_configs.dist import ( +from dbt.adapters.redshift.materialization_config.base import RedshiftRelationConfigBase +from dbt.adapters.redshift.materialization_config.dist import ( RedshiftDistConfig, RedshiftDistStyle, RedshiftDistConfigChange, ) -from dbt.adapters.redshift.relation_configs.policies import MAX_CHARACTERS_IN_IDENTIFIER -from dbt.adapters.redshift.relation_configs.sort import ( +from dbt.adapters.redshift.materialization_config.policies import MAX_CHARACTERS_IN_IDENTIFIER +from dbt.adapters.redshift.materialization_config.sort import ( RedshiftSortConfig, RedshiftSortConfigChange, ) @dataclass(frozen=True, eq=True, unsafe_hash=True) -class RedshiftMaterializedViewConfig(RedshiftRelationConfigBase, RelationConfigValidationMixin): +class RedshiftMaterializedViewConfig(MaterializationConfig, RelationConfigValidationMixin): """ This config follow the specs found here: https://docs.aws.amazon.com/redshift/latest/dg/materialized-view-create-sql-command.html @@ -51,9 +52,9 @@ class RedshiftMaterializedViewConfig(RedshiftRelationConfigBase, RelationConfigV There are currently no non-configurable parameters. """ - mv_name: str - schema_name: str - database_name: str + name: str + schema: str + database: str query: str backup: bool = True dist: RedshiftDistConfig = RedshiftDistConfig(diststyle=RedshiftDistStyle.even) diff --git a/dbt/adapters/redshift/relation_configs/policies.py b/dbt/adapters/redshift/materialization_config/policy.py similarity index 100% rename from dbt/adapters/redshift/relation_configs/policies.py rename to dbt/adapters/redshift/materialization_config/policy.py diff --git a/dbt/adapters/redshift/materialization_config/schema.py b/dbt/adapters/redshift/materialization_config/schema.py new file mode 100644 index 000000000..e69de29bb diff --git a/dbt/adapters/redshift/relation_configs/sort.py b/dbt/adapters/redshift/materialization_config/sort.py similarity index 98% rename from dbt/adapters/redshift/relation_configs/sort.py rename to dbt/adapters/redshift/materialization_config/sort.py index 58104b65f..1bd9b5f01 100644 --- a/dbt/adapters/redshift/relation_configs/sort.py +++ b/dbt/adapters/redshift/materialization_config/sort.py @@ -12,7 +12,7 @@ from dbt.dataclass_schema import StrEnum from dbt.exceptions import DbtRuntimeError -from dbt.adapters.redshift.relation_configs.base import RedshiftRelationConfigBase +from dbt.adapters.redshift.materialization_config.base import RedshiftRelationConfigBase class RedshiftSortStyle(StrEnum): diff --git a/dbt/adapters/redshift/relation.py b/dbt/adapters/redshift/relation.py index 0ef4fe276..27f12cacf 100644 --- a/dbt/adapters/redshift/relation.py +++ b/dbt/adapters/redshift/relation.py @@ -12,7 +12,7 @@ from dbt.contracts.relation import RelationType from dbt.exceptions import DbtRuntimeError -from dbt.adapters.redshift.relation_configs import ( +from dbt.adapters.redshift.materialization_config import ( RedshiftMaterializedViewConfig, RedshiftMaterializedViewConfigChangeset, RedshiftAutoRefreshConfigChange, diff --git a/dev-requirements.txt b/dev-requirements.txt index bed71ec05..dd054fdb0 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,8 +1,8 @@ # install latest changes in dbt-core + dbt-postgres # TODO: how to switch from HEAD to x.y.latest branches after minor releases? -git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-tests-adapter&subdirectory=tests/adapter -git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-postgres&subdirectory=plugins/postgres +git+https://github.com/dbt-labs/dbt-core.git@feature/materialized-views/adap-608#egg=dbt-core&subdirectory=core +git+https://github.com/dbt-labs/dbt-core.git@feature/materialized-views/adap-608#egg=dbt-tests-adapter&subdirectory=tests/adapter +git+https://github.com/dbt-labs/dbt-core.git@feature/materialized-views/adap-608#egg=dbt-postgres&subdirectory=plugins/postgres # if version 1.x or greater -> pin to major version # if version 0.x -> pin to minor From 9cc18b5f43c123aa1a08af469090fcf937be0d1e Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 11 Jul 2023 01:10:08 -0400 Subject: [PATCH 02/12] new relation/materialization first draft --- dbt/adapters/redshift/impl.py | 18 +- .../materialization_config/__init__.py | 19 -- .../redshift/materialization_config/base.py | 70 ------ .../materialization_config/database.py | 58 ----- .../redshift/materialization_config/policy.py | 19 -- dbt/adapters/redshift/relation.py | 105 -------- .../redshift/relation/models/__init__.py | 12 + .../redshift/relation/models/database.py | 42 ++++ .../models}/dist.py | 40 ++- .../models}/materialized_view.py | 232 +++++++++++------- .../redshift/relation/models/policy.py | 27 ++ .../redshift/relation/models/schema.py | 49 ++++ .../models}/sort.py | 63 +++-- dbt/include/redshift/macros/adapters.sql | 126 ---------- .../{relations.sql => get_relations.sql} | 0 .../{seeds/helpers.sql => seed.sql} | 0 .../materializations/snapshot_merge.sql | 1 - .../macros/relation_components/schema.sql | 21 ++ .../macros/relation_components/sort.sql | 16 ++ .../redshift/macros/relations/table.sql | 60 +++++ .../redshift/macros/relations/view.sql | 29 +++ .../macros/{ => utils}/timestamps.sql | 0 22 files changed, 467 insertions(+), 540 deletions(-) delete mode 100644 dbt/adapters/redshift/materialization_config/__init__.py delete mode 100644 dbt/adapters/redshift/materialization_config/base.py delete mode 100644 dbt/adapters/redshift/materialization_config/database.py delete mode 100644 dbt/adapters/redshift/materialization_config/policy.py delete mode 100644 dbt/adapters/redshift/relation.py create mode 100644 dbt/adapters/redshift/relation/models/__init__.py create mode 100644 dbt/adapters/redshift/relation/models/database.py rename dbt/adapters/redshift/{materialization_config => relation/models}/dist.py (77%) rename dbt/adapters/redshift/{materialization_config => relation/models}/materialized_view.py (50%) create mode 100644 dbt/adapters/redshift/relation/models/policy.py create mode 100644 dbt/adapters/redshift/relation/models/schema.py rename dbt/adapters/redshift/{materialization_config => relation/models}/sort.py (77%) rename dbt/include/redshift/macros/{relations.sql => get_relations.sql} (100%) rename dbt/include/redshift/macros/materializations/{seeds/helpers.sql => seed.sql} (100%) create mode 100644 dbt/include/redshift/macros/relation_components/schema.sql create mode 100644 dbt/include/redshift/macros/relation_components/sort.sql create mode 100644 dbt/include/redshift/macros/relations/table.sql create mode 100644 dbt/include/redshift/macros/relations/view.sql rename dbt/include/redshift/macros/{ => utils}/timestamps.sql (100%) diff --git a/dbt/adapters/redshift/impl.py b/dbt/adapters/redshift/impl.py index 0ceb931d0..b7fdf787d 100644 --- a/dbt/adapters/redshift/impl.py +++ b/dbt/adapters/redshift/impl.py @@ -1,17 +1,20 @@ +from collections import namedtuple from dataclasses import dataclass from typing import Optional, Set, Any, Dict, Type -from collections import namedtuple from dbt.adapters.base import PythonJobHelper from dbt.adapters.base.impl import AdapterConfig, ConstraintSupport from dbt.adapters.base.meta import available +from dbt.adapters.relation.factory import RelationFactory from dbt.adapters.sql import SQLAdapter from dbt.contracts.connection import AdapterResponse from dbt.contracts.graph.nodes import ConstraintType +from dbt.contracts.relation import RelationType from dbt.events import AdapterLogger import dbt.exceptions from dbt.adapters.redshift import RedshiftConnectionManager, RedshiftRelation +from dbt.adapters.redshift.relation import models as relation_models logger = AdapterLogger("Redshift") @@ -45,6 +48,19 @@ class RedshiftAdapter(SQLAdapter): ConstraintType.foreign_key: ConstraintSupport.NOT_ENFORCED, } + @property + def relation_factory(self): + return RelationFactory( + relation_models={ + RelationType.MaterializedView: relation_models.RedshiftMaterializedViewRelation, + }, + relation_changesets={ + RelationType.MaterializedView: relation_models.RedshiftMaterializedViewRelationChangeset, + }, + relation_can_be_renamed={RelationType.MaterializedView}, + render_policy=relation_models.RedshiftRenderPolicy, + ) + @classmethod def date_function(cls): return "getdate()" diff --git a/dbt/adapters/redshift/materialization_config/__init__.py b/dbt/adapters/redshift/materialization_config/__init__.py deleted file mode 100644 index 90555568e..000000000 --- a/dbt/adapters/redshift/materialization_config/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -from dbt.adapters.redshift.materialization_config.sort import ( - RedshiftSortConfig, - RedshiftSortConfigChange, -) -from dbt.adapters.redshift.materialization_config.dist import ( - RedshiftDistConfig, - RedshiftDistConfigChange, -) -from dbt.adapters.redshift.materialization_config.materialized_view import ( - RedshiftMaterializedViewConfig, - RedshiftAutoRefreshConfigChange, - RedshiftBackupConfigChange, - RedshiftMaterializedViewConfigChangeset, -) -from dbt.adapters.redshift.materialization_config.policies import ( - RedshiftIncludePolicy, - RedshiftQuotePolicy, - MAX_CHARACTERS_IN_IDENTIFIER, -) diff --git a/dbt/adapters/redshift/materialization_config/base.py b/dbt/adapters/redshift/materialization_config/base.py deleted file mode 100644 index 920cde6c2..000000000 --- a/dbt/adapters/redshift/materialization_config/base.py +++ /dev/null @@ -1,70 +0,0 @@ -from dataclasses import dataclass -from typing import Optional - -import agate -from dbt.adapters.base.relation import Policy -from dbt.adapters.relation_configs import ( - RelationConfigBase, - RelationResults, -) -from dbt.contracts.graph.nodes import ModelNode -from dbt.contracts.relation import ComponentName - -from dbt.adapters.redshift.materialization_config.policies import ( - RedshiftIncludePolicy, - RedshiftQuotePolicy, -) - - -@dataclass(frozen=True, eq=True, unsafe_hash=True) -class RedshiftRelationConfigBase(RelationConfigBase): - """ - This base class implements a few boilerplate methods and provides some light structure for Redshift relations. - """ - - @classmethod - def include_policy(cls) -> Policy: - return RedshiftIncludePolicy() - - @classmethod - def quote_policy(cls) -> Policy: - return RedshiftQuotePolicy() - - @classmethod - def from_model_node(cls, model_node: ModelNode) -> "RelationConfigBase": - relation_config = cls.parse_model_node(model_node) - relation = cls.from_dict(relation_config) - return relation - - @classmethod - def parse_model_node(cls, model_node: ModelNode) -> dict: - raise NotImplementedError( - "`parse_model_node()` needs to be implemented on this RelationConfigBase instance" - ) - - @classmethod - def from_relation_results(cls, relation_results: RelationResults) -> "RelationConfigBase": - relation_config = cls.parse_relation_results(relation_results) - relation = cls.from_dict(relation_config) - return relation - - @classmethod - def parse_relation_results(cls, relation_results: RelationResults) -> dict: - raise NotImplementedError( - "`parse_relation_results()` needs to be implemented on this RelationConfigBase instance" - ) - - @classmethod - def _render_part(cls, component: ComponentName, value: Optional[str]) -> Optional[str]: - if cls.include_policy().get_part(component) and value: - if cls.quote_policy().get_part(component): - return f'"{value}"' - return value.lower() - return None - - @classmethod - def _get_first_row(cls, results: agate.Table) -> agate.Row: - try: - return results.rows[0] - except IndexError: - return agate.Row(values=set()) diff --git a/dbt/adapters/redshift/materialization_config/database.py b/dbt/adapters/redshift/materialization_config/database.py deleted file mode 100644 index 145311ce5..000000000 --- a/dbt/adapters/redshift/materialization_config/database.py +++ /dev/null @@ -1,58 +0,0 @@ -from dbt.adapters.materialization_config import ( - DatabaseConfig, - RelationConfigValidationMixin, - RelationConfigValidationRule, -) - -from dbt.adapters.redshift.materialization_config.policy import redshift_conform_part - - -@dataclass(frozen=True, eq=True, unsafe_hash=True) -class RedshiftDatabaseConfig(DatabaseConfig): - """ - This config follow the specs found here: - https://docs.aws.amazon.com/redshift/latest/dg/r_CREATE_DATABASE.html - - The following parameters are configurable by dbt: - - name: name of the database - """ - - name: str - - @property - def validation_rules(self) -> Set[RelationConfigValidationRule]: - return { - RelationConfigValidationRule( - validation_check=len(self.name or "") > 0, - validation_error=DbtRuntimeError( - f"dbt-redshift requires a name for a database, received: {self.name}" - ), - ) - } - - @classmethod - def from_dict(cls, config_dict: dict) -> "RedshiftDatabaseConfig": - """ - Because this returns a frozen dataclass, this method should be overridden if additional parameters are supplied. - """ - kwargs_dict = {"name": redshift_conform_part(ComponentName.Database, config_dict["name"])} - database = super().from_dict(kwargs_dict) - assert isinstance(database, RedshiftDatabaseConfig) - return database - - @classmethod - def parse_model_node(cls, model_node: ModelNode) -> dict: - """ - Because this returns a `dict`, this method should be extended if additional parameters are supplied. - """ - config_dict = {"name": model_node.database} - super().parse_model_node() - return config_dict - - @classmethod - def parse_describe_relation_results(cls, describe_relation_results: agate.Row) -> dict: - """ - Because this returns a `dict`, this method should be extended if additional parameters are supplied. - """ - config_dict = {"name": describe_relation_results["database"]} - return config_dict diff --git a/dbt/adapters/redshift/materialization_config/policy.py b/dbt/adapters/redshift/materialization_config/policy.py deleted file mode 100644 index 7ec8e8acb..000000000 --- a/dbt/adapters/redshift/materialization_config/policy.py +++ /dev/null @@ -1,19 +0,0 @@ -from dataclasses import dataclass - -from dbt.adapters.base.relation import Policy - - -MAX_CHARACTERS_IN_IDENTIFIER = 127 - - -class RedshiftIncludePolicy(Policy): - database: bool = True - schema: bool = True - identifier: bool = True - - -@dataclass -class RedshiftQuotePolicy(Policy): - database: bool = True - schema: bool = True - identifier: bool = True diff --git a/dbt/adapters/redshift/relation.py b/dbt/adapters/redshift/relation.py deleted file mode 100644 index 27f12cacf..000000000 --- a/dbt/adapters/redshift/relation.py +++ /dev/null @@ -1,105 +0,0 @@ -from dataclasses import dataclass -from typing import Optional - -from dbt.adapters.base.relation import BaseRelation -from dbt.adapters.relation_configs import ( - RelationConfigBase, - RelationConfigChangeAction, - RelationResults, -) -from dbt.context.providers import RuntimeConfigObject -from dbt.contracts.graph.nodes import ModelNode -from dbt.contracts.relation import RelationType -from dbt.exceptions import DbtRuntimeError - -from dbt.adapters.redshift.materialization_config import ( - RedshiftMaterializedViewConfig, - RedshiftMaterializedViewConfigChangeset, - RedshiftAutoRefreshConfigChange, - RedshiftBackupConfigChange, - RedshiftDistConfigChange, - RedshiftSortConfigChange, - RedshiftIncludePolicy, - RedshiftQuotePolicy, - MAX_CHARACTERS_IN_IDENTIFIER, -) - - -@dataclass(frozen=True, eq=False, repr=False) -class RedshiftRelation(BaseRelation): - include_policy = RedshiftIncludePolicy # type: ignore - quote_policy = RedshiftQuotePolicy # type: ignore - relation_configs = { - RelationType.MaterializedView.value: RedshiftMaterializedViewConfig, - } - - def __post_init__(self): - # Check for length of Redshift table/view names. - # Check self.type to exclude test relation identifiers - if ( - self.identifier is not None - and self.type is not None - and len(self.identifier) > MAX_CHARACTERS_IN_IDENTIFIER - ): - raise DbtRuntimeError( - f"Relation name '{self.identifier}' " - f"is longer than {MAX_CHARACTERS_IN_IDENTIFIER} characters" - ) - - def relation_max_name_length(self): - return MAX_CHARACTERS_IN_IDENTIFIER - - @classmethod - def from_runtime_config(cls, runtime_config: RuntimeConfigObject) -> RelationConfigBase: - model_node: ModelNode = runtime_config.model - relation_type: str = model_node.config.materialized - - if relation_config := cls.relation_configs.get(relation_type): - return relation_config.from_model_node(model_node) - - raise DbtRuntimeError( - f"from_runtime_config() is not supported for the provided relation type: {relation_type}" - ) - - @classmethod - def materialized_view_config_changeset( - cls, relation_results: RelationResults, runtime_config: RuntimeConfigObject - ) -> Optional[RedshiftMaterializedViewConfigChangeset]: - config_change_collection = RedshiftMaterializedViewConfigChangeset() - - existing_materialized_view = RedshiftMaterializedViewConfig.from_relation_results( - relation_results - ) - new_materialized_view = RedshiftMaterializedViewConfig.from_model_node( - runtime_config.model - ) - assert isinstance(existing_materialized_view, RedshiftMaterializedViewConfig) - assert isinstance(new_materialized_view, RedshiftMaterializedViewConfig) - - if new_materialized_view.autorefresh != existing_materialized_view.autorefresh: - config_change_collection.autorefresh = RedshiftAutoRefreshConfigChange( - action=RelationConfigChangeAction.alter, - context=new_materialized_view.autorefresh, - ) - - if new_materialized_view.backup != existing_materialized_view.backup: - config_change_collection.backup = RedshiftBackupConfigChange( - action=RelationConfigChangeAction.alter, - context=new_materialized_view.backup, - ) - - if new_materialized_view.dist != existing_materialized_view.dist: - config_change_collection.dist = RedshiftDistConfigChange( - action=RelationConfigChangeAction.alter, - context=new_materialized_view.dist, - ) - - if new_materialized_view.sort != existing_materialized_view.sort: - config_change_collection.sort = RedshiftSortConfigChange( - action=RelationConfigChangeAction.alter, - context=new_materialized_view.sort, - ) - - if config_change_collection.has_changes: - return config_change_collection - return None diff --git a/dbt/adapters/redshift/relation/models/__init__.py b/dbt/adapters/redshift/relation/models/__init__.py new file mode 100644 index 000000000..aa21a54dc --- /dev/null +++ b/dbt/adapters/redshift/relation/models/__init__.py @@ -0,0 +1,12 @@ +from dbt.adapters.redshift.relation.models.database import RedshiftDatabaseRelation +from dbt.adapters.redshift.relation.models.dist import RedshiftDistRelation +from dbt.adapters.redshift.relation.models.materialized_view import ( + RedshiftMaterializedViewRelation, + RedshiftMaterializedViewRelationChangeset, +) +from dbt.adapters.redshift.relation.models.policy import ( + RedshiftRenderPolicy, + MAX_CHARACTERS_IN_IDENTIFIER, +) +from dbt.adapters.redshift.relation.models.schema import RedshiftSchemaRelation +from dbt.adapters.redshift.relation.models.sort import RedshiftSortRelation diff --git a/dbt/adapters/redshift/relation/models/database.py b/dbt/adapters/redshift/relation/models/database.py new file mode 100644 index 000000000..84f756549 --- /dev/null +++ b/dbt/adapters/redshift/relation/models/database.py @@ -0,0 +1,42 @@ +from dataclasses import dataclass +from typing import Set + +from dbt.adapters.relation.models import DatabaseRelation +from dbt.adapters.validation import ValidationMixin, ValidationRule +from dbt.exceptions import DbtRuntimeError + +from dbt.adapters.redshift.relation.models.policy import RedshiftRenderPolicy + + +@dataclass(frozen=True, eq=True, unsafe_hash=True) +class RedshiftDatabaseRelation(DatabaseRelation, ValidationMixin): + """ + This config follow the specs found here: + https://docs.aws.amazon.com/redshift/latest/dg/r_CREATE_DATABASE.html + + The following parameters are configurable by dbt: + - name: name of the database + """ + + # attribution + name: str + + # configuration + render = RedshiftRenderPolicy + + @property + def validation_rules(self) -> Set[ValidationRule]: + return { + ValidationRule( + validation_check=len(self.name or "") > 0, + validation_error=DbtRuntimeError( + f"dbt-redshift requires a name for a database, received: {self.name}" + ), + ) + } + + @classmethod + def from_dict(cls, config_dict) -> "RedshiftDatabaseRelation": + database = super().from_dict(config_dict) + assert isinstance(database, RedshiftDatabaseRelation) + return database diff --git a/dbt/adapters/redshift/materialization_config/dist.py b/dbt/adapters/redshift/relation/models/dist.py similarity index 77% rename from dbt/adapters/redshift/materialization_config/dist.py rename to dbt/adapters/redshift/relation/models/dist.py index 0ec3f72ae..a8523f052 100644 --- a/dbt/adapters/redshift/materialization_config/dist.py +++ b/dbt/adapters/redshift/relation/models/dist.py @@ -2,18 +2,16 @@ from typing import Optional, Set import agate -from dbt.adapters.relation_configs import ( - RelationConfigChange, - RelationConfigChangeAction, - RelationConfigValidationMixin, - RelationConfigValidationRule, +from dbt.adapters.relation.models import ( + RelationChange, + RelationChangeAction, + RelationComponent, ) +from dbt.adapters.validation import ValidationMixin, ValidationRule from dbt.contracts.graph.nodes import ModelNode from dbt.dataclass_schema import StrEnum from dbt.exceptions import DbtRuntimeError -from dbt.adapters.redshift.materialization_config.base import RedshiftRelationConfigBase - class RedshiftDistStyle(StrEnum): auto = "auto" @@ -27,7 +25,7 @@ def default(cls) -> "RedshiftDistStyle": @dataclass(frozen=True, eq=True, unsafe_hash=True) -class RedshiftDistConfig(RedshiftRelationConfigBase, RelationConfigValidationMixin): +class RedshiftDistRelation(RelationComponent, ValidationMixin): """ This config fallows the specs found here: https://docs.aws.amazon.com/redshift/latest/dg/r_CREATE_TABLE_NEW.html @@ -41,10 +39,10 @@ class RedshiftDistConfig(RedshiftRelationConfigBase, RelationConfigValidationMix distkey: Optional[str] = None @property - def validation_rules(self) -> Set[RelationConfigValidationRule]: + def validation_rules(self) -> Set[ValidationRule]: # index rules get run by default with the mixin return { - RelationConfigValidationRule( + ValidationRule( validation_check=not ( self.diststyle == RedshiftDistStyle.key and self.distkey is None ), @@ -52,7 +50,7 @@ def validation_rules(self) -> Set[RelationConfigValidationRule]: "A `RedshiftDistConfig` that specifies a `diststyle` of `key` must provide a value for `distkey`." ), ), - RelationConfigValidationRule( + ValidationRule( validation_check=not ( self.diststyle in (RedshiftDistStyle.auto, RedshiftDistStyle.even, RedshiftDistStyle.all) @@ -65,12 +63,12 @@ def validation_rules(self) -> Set[RelationConfigValidationRule]: } @classmethod - def from_dict(cls, config_dict) -> "RedshiftDistConfig": + def from_dict(cls, config_dict) -> "RedshiftDistRelation": kwargs_dict = { "diststyle": config_dict.get("diststyle"), "distkey": config_dict.get("distkey"), } - dist: "RedshiftDistConfig" = super().from_dict(kwargs_dict) # type: ignore + dist: "RedshiftDistRelation" = super().from_dict(kwargs_dict) # type: ignore return dist @classmethod @@ -107,12 +105,12 @@ def parse_model_node(cls, model_node: ModelNode) -> dict: return config @classmethod - def parse_relation_results(cls, relation_results_entry: agate.Row) -> dict: + def parse_describe_relation_results(cls, describe_relation_results: agate.Row) -> dict: """ Translate agate objects from the database into a standard dictionary. Args: - relation_results_entry: the description of the distkey and diststyle from the database in this format: + describe_relation_results: the description of the distkey and diststyle from the database in this format: agate.Row({ "diststyle": "", # e.g. EVEN | KEY(column1) | AUTO(ALL) | AUTO(KEY(id)) @@ -120,7 +118,7 @@ def parse_relation_results(cls, relation_results_entry: agate.Row) -> dict: Returns: a standard dictionary describing this `RedshiftDistConfig` instance """ - dist: str = relation_results_entry.get("diststyle") + dist: str = describe_relation_results.get("diststyle") try: # covers `AUTO`, `ALL`, `EVEN`, `KEY`, '', @@ -145,18 +143,18 @@ def parse_relation_results(cls, relation_results_entry: agate.Row) -> dict: @dataclass(frozen=True, eq=True, unsafe_hash=True) -class RedshiftDistConfigChange(RelationConfigChange, RelationConfigValidationMixin): - context: RedshiftDistConfig +class RedshiftDistRelationChange(RelationChange, ValidationMixin): + context: RedshiftDistRelation @property def requires_full_refresh(self) -> bool: return True @property - def validation_rules(self) -> Set[RelationConfigValidationRule]: + def validation_rules(self) -> Set[ValidationRule]: return { - RelationConfigValidationRule( - validation_check=(self.action == RelationConfigChangeAction.alter), + ValidationRule( + validation_check=(self.action == RelationChangeAction.alter), validation_error=DbtRuntimeError( "Invalid operation, only `alter` changes are supported for `distkey` / `diststyle`." ), diff --git a/dbt/adapters/redshift/materialization_config/materialized_view.py b/dbt/adapters/redshift/relation/models/materialized_view.py similarity index 50% rename from dbt/adapters/redshift/materialization_config/materialized_view.py rename to dbt/adapters/redshift/relation/models/materialized_view.py index 9d12e6448..e17f3ffac 100644 --- a/dbt/adapters/redshift/materialization_config/materialized_view.py +++ b/dbt/adapters/redshift/relation/models/materialized_view.py @@ -1,33 +1,37 @@ +from copy import deepcopy from dataclasses import dataclass -from typing import Optional, Set +from typing import Dict, Optional, Set import agate -from dbt.adapters.materialization_config import ( - RelationResults, - RelationConfigChange, - RelationConfigValidationMixin, - RelationConfigValidationRule, - MaterializationConfig, +from dbt.adapters.relation.models import ( + MaterializedViewRelation, + Relation, + RelationChange, + RelationChangeAction, + RelationChangeset, ) +from dbt.adapters.validation import ValidationMixin, ValidationRule from dbt.contracts.graph.nodes import ModelNode -from dbt.contracts.relation import ComponentName from dbt.exceptions import DbtRuntimeError -from dbt.adapters.redshift.materialization_config.base import RedshiftRelationConfigBase -from dbt.adapters.redshift.materialization_config.dist import ( - RedshiftDistConfig, +from dbt.adapters.redshift.relation.models.dist import ( + RedshiftDistRelation, + RedshiftDistRelationChange, RedshiftDistStyle, - RedshiftDistConfigChange, ) -from dbt.adapters.redshift.materialization_config.policies import MAX_CHARACTERS_IN_IDENTIFIER -from dbt.adapters.redshift.materialization_config.sort import ( - RedshiftSortConfig, - RedshiftSortConfigChange, +from dbt.adapters.redshift.relation.models.policy import ( + MAX_CHARACTERS_IN_IDENTIFIER, + RedshiftRenderPolicy, +) +from dbt.adapters.redshift.relation.models.schema import RedshiftSchemaRelation +from dbt.adapters.redshift.relation.models.sort import ( + RedshiftSortRelation, + RedshiftSortRelationChange, ) @dataclass(frozen=True, eq=True, unsafe_hash=True) -class RedshiftMaterializedViewConfig(MaterializationConfig, RelationConfigValidationMixin): +class RedshiftMaterializedViewRelation(MaterializedViewRelation, ValidationMixin): """ This config follow the specs found here: https://docs.aws.amazon.com/redshift/latest/dg/materialized-view-create-sql-command.html @@ -52,42 +56,41 @@ class RedshiftMaterializedViewConfig(MaterializationConfig, RelationConfigValida There are currently no non-configurable parameters. """ + # attribution name: str - schema: str - database: str + schema: RedshiftSchemaRelation query: str - backup: bool = True - dist: RedshiftDistConfig = RedshiftDistConfig(diststyle=RedshiftDistStyle.even) - sort: RedshiftSortConfig = RedshiftSortConfig() - autorefresh: bool = False - - @property - def path(self) -> str: - return ".".join( - part - for part in [self.database_name, self.schema_name, self.mv_name] - if part is not None - ) + backup: Optional[bool] = True + dist: RedshiftDistRelation = RedshiftDistRelation( + diststyle=RedshiftDistStyle.even, render=RedshiftRenderPolicy + ) + sort: RedshiftSortRelation = RedshiftSortRelation(render=RedshiftRenderPolicy) + autorefresh: Optional[bool] = False + + # configuration + render = RedshiftRenderPolicy + SchemaParser = RedshiftSchemaRelation # type: ignore + can_be_renamed = True @property - def validation_rules(self) -> Set[RelationConfigValidationRule]: + def validation_rules(self) -> Set[ValidationRule]: # sort and dist rules get run by default with the mixin return { - RelationConfigValidationRule( - validation_check=len(self.mv_name or "") <= MAX_CHARACTERS_IN_IDENTIFIER, + ValidationRule( + validation_check=len(self.name or "") <= MAX_CHARACTERS_IN_IDENTIFIER, validation_error=DbtRuntimeError( f"The materialized view name is more than {MAX_CHARACTERS_IN_IDENTIFIER} " - f"characters: {self.mv_name}" + f"characters: {self.name}" ), ), - RelationConfigValidationRule( + ValidationRule( validation_check=self.dist.diststyle != RedshiftDistStyle.auto, validation_error=DbtRuntimeError( "Redshift materialized views do not support a `diststyle` of `auto`." ), ), - RelationConfigValidationRule( - validation_check=len(self.mv_name if self.mv_name else "") <= 127, + ValidationRule( + validation_check=len(self.name if self.name else "") <= 127, validation_error=DbtRuntimeError( "Redshift does not support object names longer than 127 characters." ), @@ -95,56 +98,49 @@ def validation_rules(self) -> Set[RelationConfigValidationRule]: } @classmethod - def from_dict(cls, config_dict) -> "RedshiftMaterializedViewConfig": - kwargs_dict = { - "mv_name": cls._render_part(ComponentName.Identifier, config_dict.get("mv_name")), - "schema_name": cls._render_part(ComponentName.Schema, config_dict.get("schema_name")), - "database_name": cls._render_part( - ComponentName.Database, config_dict.get("database_name") - ), - "query": config_dict.get("query"), - "backup": config_dict.get("backup"), - "autorefresh": config_dict.get("autorefresh"), - } + def from_dict(cls, config_dict) -> "RedshiftMaterializedViewRelation": + # don't alter the incoming config + kwargs_dict = deepcopy(config_dict) # this preserves the materialized view-specific default of `even` over the general default of `auto` if dist := config_dict.get("dist"): - kwargs_dict.update({"dist": RedshiftDistConfig.from_dict(dist)}) + kwargs_dict.update({"dist": RedshiftDistRelation.from_dict(dist)}) if sort := config_dict.get("sort"): - kwargs_dict.update({"sort": RedshiftSortConfig.from_dict(sort)}) + kwargs_dict.update({"sort": RedshiftSortRelation.from_dict(sort)}) - materialized_view: "RedshiftMaterializedViewConfig" = super().from_dict(kwargs_dict) # type: ignore + materialized_view = super().from_dict(kwargs_dict) + assert isinstance(materialized_view, RedshiftMaterializedViewRelation) return materialized_view @classmethod def parse_model_node(cls, model_node: ModelNode) -> dict: - config_dict = { - "mv_name": model_node.identifier, - "schema_name": model_node.schema, - "database_name": model_node.database, - "backup": model_node.config.get("backup"), - "autorefresh": model_node.config.get("auto_refresh"), - } + config_dict = super().parse_model_node(model_node) - if query := model_node.compiled_code: - config_dict.update({"query": query.strip()}) + config_dict.update( + { + "backup": model_node.config.get("backup"), + "autorefresh": model_node.config.get("auto_refresh"), + } + ) if model_node.config.get("dist"): - config_dict.update({"dist": RedshiftDistConfig.parse_model_node(model_node)}) + config_dict.update({"dist": RedshiftDistRelation.parse_model_node(model_node)}) if model_node.config.get("sort"): - config_dict.update({"sort": RedshiftSortConfig.parse_model_node(model_node)}) + config_dict.update({"sort": RedshiftSortRelation.parse_model_node(model_node)}) return config_dict @classmethod - def parse_relation_results(cls, relation_results: RelationResults) -> dict: + def parse_describe_relation_results( + cls, describe_relation_results: Dict[str, agate.Table] + ) -> dict: """ Translate agate objects from the database into a standard dictionary. Args: - relation_results: the description of the materialized view from the database in this format: + describe_relation_results: the description of the materialized view from the database in this format: { "materialized_view": agate.Table( @@ -166,30 +162,29 @@ def parse_relation_results(cls, relation_results: RelationResults) -> dict: Returns: a standard dictionary describing this `RedshiftMaterializedViewConfig` instance """ - materialized_view: agate.Row = cls._get_first_row( - relation_results.get("materialized_view") + config_dict = super().parse_describe_relation_results(describe_relation_results) + + materialized_view: agate.Row = describe_relation_results["materialized_view"].rows[0] + query: agate.Row = describe_relation_results["query"].rows[0] + + config_dict.update( + { + "autorefresh": {"t": True, "f": False}.get(materialized_view.get("autorefresh")), + "query": cls._parse_query(query.get("definition")), + } ) - query: agate.Row = cls._get_first_row(relation_results.get("query")) - - config_dict = { - "mv_name": materialized_view.get("table"), - "schema_name": materialized_view.get("schema"), - "database_name": materialized_view.get("database"), - "autorefresh": {"t": True, "f": False}.get(materialized_view.get("autorefresh")), - "query": cls._parse_query(query.get("definition")), - } # the default for materialized views differs from the default for diststyle in general # only set it if we got a value if materialized_view.get("diststyle"): config_dict.update( - {"dist": RedshiftDistConfig.parse_relation_results(materialized_view)} + {"dist": RedshiftDistRelation.parse_describe_relation_results(materialized_view)} ) # TODO: this only shows the first column in the sort key if materialized_view.get("sortkey1"): config_dict.update( - {"sort": RedshiftSortConfig.parse_relation_results(materialized_view)} + {"sort": RedshiftSortRelation.parse_describe_relation_results(materialized_view)} ) return config_dict @@ -222,7 +217,7 @@ def _parse_query(cls, query: str) -> str: @dataclass(frozen=True, eq=True, unsafe_hash=True) -class RedshiftAutoRefreshConfigChange(RelationConfigChange): +class RedshiftAutoRefreshRelationChange(RelationChange): context: Optional[bool] = None @property @@ -231,7 +226,7 @@ def requires_full_refresh(self) -> bool: @dataclass(frozen=True, eq=True, unsafe_hash=True) -class RedshiftBackupConfigChange(RelationConfigChange): +class RedshiftBackupRelationChange(RelationChange): context: Optional[bool] = None @property @@ -240,11 +235,67 @@ def requires_full_refresh(self) -> bool: @dataclass -class RedshiftMaterializedViewConfigChangeset: - backup: Optional[RedshiftBackupConfigChange] = None - dist: Optional[RedshiftDistConfigChange] = None - sort: Optional[RedshiftSortConfigChange] = None - autorefresh: Optional[RedshiftAutoRefreshConfigChange] = None +class RedshiftMaterializedViewRelationChangeset(RelationChangeset): + backup: Optional[RedshiftBackupRelationChange] = None + dist: Optional[RedshiftDistRelationChange] = None + sort: Optional[RedshiftSortRelationChange] = None + autorefresh: Optional[RedshiftAutoRefreshRelationChange] = None + + @classmethod + def parse_relations(cls, existing_relation: Relation, target_relation: Relation) -> dict: + try: + assert isinstance(existing_relation, RedshiftMaterializedViewRelation) + assert isinstance(target_relation, RedshiftMaterializedViewRelation) + except AssertionError: + raise DbtRuntimeError( + f"Two Redshift materialized view relations were expected, but received:\n" + f" existing: {existing_relation}\n" + f" new: {target_relation}\n" + ) + + config_dict = super().parse_relations(existing_relation, target_relation) + + if target_relation.autorefresh != existing_relation.autorefresh: + config_dict.update( + { + "autorefresh": RedshiftAutoRefreshRelationChange( + action=RelationChangeAction.alter, + context=target_relation.autorefresh, + ) + } + ) + + if target_relation.backup != existing_relation.backup: + config_dict.update( + { + "backup": RedshiftBackupRelationChange( + action=RelationChangeAction.alter, + context=target_relation.backup, + ) + } + ) + + if target_relation.dist != existing_relation.dist: + config_dict.update( + { + "dist": RedshiftDistRelationChange( + action=RelationChangeAction.alter, + context=target_relation.dist, + ) + } + ) + + if target_relation.sort != existing_relation.sort: + config_dict.update( + { + "sort": RedshiftSortRelationChange( + action=RelationChangeAction.alter, + context=target_relation.sort, + ) + } + ) + + return config_dict @property def requires_full_refresh(self) -> bool: @@ -258,12 +309,5 @@ def requires_full_refresh(self) -> bool: ) @property - def has_changes(self) -> bool: - return any( - { - self.backup if self.backup else False, - self.dist if self.dist else False, - self.sort if self.sort else False, - self.autorefresh if self.autorefresh else False, - } - ) + def is_empty(self) -> bool: + return not any({self.backup, self.dist, self.sort, self.autorefresh}) and super().is_empty diff --git a/dbt/adapters/redshift/relation/models/policy.py b/dbt/adapters/redshift/relation/models/policy.py new file mode 100644 index 000000000..4210b00b1 --- /dev/null +++ b/dbt/adapters/redshift/relation/models/policy.py @@ -0,0 +1,27 @@ +from dataclasses import dataclass + +from dbt.adapters.relation.models import IncludePolicy, QuotePolicy, RenderPolicy + + +MAX_CHARACTERS_IN_IDENTIFIER = 127 + + +class RedshiftIncludePolicy(IncludePolicy): + database: bool = True + schema: bool = True + identifier: bool = True + + +@dataclass +class RedshiftQuotePolicy(QuotePolicy): + database: bool = True + schema: bool = True + identifier: bool = True + + +RedshiftRenderPolicy = RenderPolicy( + quote_policy=RedshiftQuotePolicy(), + include_policy=RedshiftIncludePolicy(), + quote_character='"', + delimiter=".", +) diff --git a/dbt/adapters/redshift/relation/models/schema.py b/dbt/adapters/redshift/relation/models/schema.py new file mode 100644 index 000000000..4bb252b0b --- /dev/null +++ b/dbt/adapters/redshift/relation/models/schema.py @@ -0,0 +1,49 @@ +from dataclasses import dataclass +from typing import Set + +from dbt.adapters.relation.models import SchemaRelation +from dbt.adapters.validation import ValidationMixin, ValidationRule +from dbt.exceptions import DbtRuntimeError + +from dbt.adapters.redshift.relation.models.database import RedshiftDatabaseRelation +from dbt.adapters.redshift.relation.models.policy import RedshiftRenderPolicy + + +@dataclass(frozen=True, eq=True, unsafe_hash=True) +class RedshiftSchemaRelation(SchemaRelation, ValidationMixin): + """ + This config follow the specs found here: + https://docs.aws.amazon.com/redshift/latest/dg/r_CREATE_SCHEMA.html + + The following parameters are configurable by dbt: + - name: name of the schema + - database_name: name of the database + """ + + # attribution + name: str + + # configuration + render = RedshiftRenderPolicy + DatabaseParser = RedshiftDatabaseRelation # type: ignore + + @classmethod + def from_dict(cls, config_dict) -> "RedshiftSchemaRelation": + schema = super().from_dict(config_dict) + assert isinstance(schema, RedshiftSchemaRelation) + return schema + + @property + def validation_rules(self) -> Set[ValidationRule]: + """ + Returns: a set of rules that should evaluate to `True` (i.e. False == validation failure) + """ + return { + ValidationRule( + validation_check=len(self.name or "") > 0, + validation_error=DbtRuntimeError( + f"dbt-redshift requires a name to reference a schema, received:\n" + f" schema: {self.name}\n" + ), + ), + } diff --git a/dbt/adapters/redshift/materialization_config/sort.py b/dbt/adapters/redshift/relation/models/sort.py similarity index 77% rename from dbt/adapters/redshift/materialization_config/sort.py rename to dbt/adapters/redshift/relation/models/sort.py index 1bd9b5f01..c07b3b71a 100644 --- a/dbt/adapters/redshift/materialization_config/sort.py +++ b/dbt/adapters/redshift/relation/models/sort.py @@ -1,18 +1,19 @@ +from copy import deepcopy from dataclasses import dataclass from typing import Optional, FrozenSet, Set import agate -from dbt.adapters.relation_configs import ( - RelationConfigChange, - RelationConfigChangeAction, - RelationConfigValidationMixin, - RelationConfigValidationRule, +from dbt.adapters.relation.models import ( + RelationChange, + RelationChangeAction, + RelationComponent, ) +from dbt.adapters.validation import ValidationMixin, ValidationRule from dbt.contracts.graph.nodes import ModelNode from dbt.dataclass_schema import StrEnum from dbt.exceptions import DbtRuntimeError -from dbt.adapters.redshift.materialization_config.base import RedshiftRelationConfigBase +from dbt.adapters.redshift.relation.models.policy import RedshiftRenderPolicy class RedshiftSortStyle(StrEnum): @@ -30,7 +31,7 @@ def default_with_columns(cls) -> "RedshiftSortStyle": @dataclass(frozen=True, eq=True, unsafe_hash=True) -class RedshiftSortConfig(RedshiftRelationConfigBase, RelationConfigValidationMixin): +class RedshiftSortRelation(RelationComponent, ValidationMixin): """ This config fallows the specs found here: https://docs.aws.amazon.com/redshift/latest/dg/r_CREATE_TABLE_NEW.html @@ -45,6 +46,9 @@ class RedshiftSortConfig(RedshiftRelationConfigBase, RelationConfigValidationMix sortstyle: Optional[RedshiftSortStyle] = None sortkey: Optional[FrozenSet[str]] = None + # configuration + render = RedshiftRenderPolicy + def __post_init__(self): # maintains `frozen=True` while allowing for a variable default on `sort_type` if self.sortstyle is None and self.sortkey is None: @@ -54,10 +58,10 @@ def __post_init__(self): super().__post_init__() @property - def validation_rules(self) -> Set[RelationConfigValidationRule]: + def validation_rules(self) -> Set[ValidationRule]: # index rules get run by default with the mixin return { - RelationConfigValidationRule( + ValidationRule( validation_check=not ( self.sortstyle == RedshiftSortStyle.auto and self.sortkey is not None ), @@ -65,7 +69,7 @@ def validation_rules(self) -> Set[RelationConfigValidationRule]: "A `RedshiftSortConfig` that specifies a `sortkey` does not support the `sortstyle` of `auto`." ), ), - RelationConfigValidationRule( + ValidationRule( validation_check=not ( self.sortstyle in (RedshiftSortStyle.compound, RedshiftSortStyle.interleaved) and self.sortkey is None @@ -74,7 +78,7 @@ def validation_rules(self) -> Set[RelationConfigValidationRule]: "A `sortstyle` of `compound` or `interleaved` requires a `sortkey` to be provided." ), ), - RelationConfigValidationRule( + ValidationRule( validation_check=not ( self.sortstyle == RedshiftSortStyle.compound and self.sortkey is not None @@ -84,7 +88,7 @@ def validation_rules(self) -> Set[RelationConfigValidationRule]: "A compound `sortkey` only supports 400 columns." ), ), - RelationConfigValidationRule( + ValidationRule( validation_check=not ( self.sortstyle == RedshiftSortStyle.interleaved and self.sortkey is not None @@ -97,12 +101,19 @@ def validation_rules(self) -> Set[RelationConfigValidationRule]: } @classmethod - def from_dict(cls, config_dict) -> "RedshiftSortConfig": - kwargs_dict = { - "sortstyle": config_dict.get("sortstyle"), - "sortkey": frozenset(column for column in config_dict.get("sortkey", {})), - } - sort: "RedshiftSortConfig" = super().from_dict(kwargs_dict) # type: ignore + def from_dict(cls, config_dict) -> "RedshiftSortRelation": + # don't alter the incoming config + kwargs_dict = deepcopy(config_dict) + + kwargs_dict.update( + { + "sortstyle": config_dict.get("sortstyle"), + "sortkey": frozenset(column for column in config_dict.get("sortkey", {})), + } + ) + + sort = super().from_dict(kwargs_dict) + assert isinstance(sort, RedshiftSortRelation) return sort @classmethod @@ -135,7 +146,7 @@ def parse_model_node(cls, model_node: ModelNode) -> dict: return config_dict @classmethod - def parse_relation_results(cls, relation_results_entry: agate.Row) -> dict: + def parse_describe_relation_results(cls, describe_relation_results: agate.Row) -> dict: """ Translate agate objects from the database into a standard dictionary. @@ -144,7 +155,7 @@ def parse_relation_results(cls, relation_results_entry: agate.Row) -> dict: Processing of `sortstyle` has been omitted here, which means it's the default (compound). Args: - relation_results_entry: the description of the sortkey and sortstyle from the database in this format: + describe_relation_results: the description of the sortkey and sortstyle from the database in this format: agate.Row({ ..., @@ -154,24 +165,24 @@ def parse_relation_results(cls, relation_results_entry: agate.Row) -> dict: Returns: a standard dictionary describing this `RedshiftSortConfig` instance """ - if sortkey := relation_results_entry.get("sortkey1"): + if sortkey := describe_relation_results.get("sortkey1"): return {"sortkey": {sortkey}} return {} @dataclass(frozen=True, eq=True, unsafe_hash=True) -class RedshiftSortConfigChange(RelationConfigChange, RelationConfigValidationMixin): - context: RedshiftSortConfig +class RedshiftSortRelationChange(RelationChange, ValidationMixin): + context: RedshiftSortRelation @property def requires_full_refresh(self) -> bool: return True @property - def validation_rules(self) -> Set[RelationConfigValidationRule]: + def validation_rules(self) -> Set[ValidationRule]: return { - RelationConfigValidationRule( - validation_check=(self.action == RelationConfigChangeAction.alter), + ValidationRule( + validation_check=(self.action == RelationChangeAction.alter), validation_error=DbtRuntimeError( "Invalid operation, only `alter` changes are supported for `sortkey` / `sortstyle`." ), diff --git a/dbt/include/redshift/macros/adapters.sql b/dbt/include/redshift/macros/adapters.sql index 62813852b..28373232d 100644 --- a/dbt/include/redshift/macros/adapters.sql +++ b/dbt/include/redshift/macros/adapters.sql @@ -1,110 +1,3 @@ - -{% macro dist(dist) %} - {%- if dist is not none -%} - {%- set dist = dist.strip().lower() -%} - - {%- if dist in ['all', 'even'] -%} - diststyle {{ dist }} - {%- elif dist == "auto" -%} - {%- else -%} - diststyle key distkey ({{ dist }}) - {%- endif -%} - - {%- endif -%} -{%- endmacro -%} - - -{% macro sort(sort_type, sort) %} - {%- if sort is not none %} - {{ sort_type | default('compound', boolean=true) }} sortkey( - {%- if sort is string -%} - {%- set sort = [sort] -%} - {%- endif -%} - {%- for item in sort -%} - {{ item }} - {%- if not loop.last -%},{%- endif -%} - {%- endfor -%} - ) - {%- endif %} -{%- endmacro -%} - - -{% macro redshift__create_table_as(temporary, relation, sql) -%} - - {%- set _dist = config.get('dist') -%} - {%- set _sort_type = config.get( - 'sort_type', - validator=validation.any['compound', 'interleaved']) -%} - {%- set _sort = config.get( - 'sort', - validator=validation.any[list, basestring]) -%} - {%- set sql_header = config.get('sql_header', none) -%} - {%- set backup = config.get('backup') -%} - - {{ sql_header if sql_header is not none }} - - {%- set contract_config = config.get('contract') -%} - {%- if contract_config.enforced -%} - - create {% if temporary -%}temporary{%- endif %} table - {{ relation.include(database=(not temporary), schema=(not temporary)) }} - {{ get_table_columns_and_constraints() }} - {{ get_assert_columns_equivalent(sql) }} - {%- set sql = get_select_subquery(sql) %} - {% if backup == false -%}backup no{%- endif %} - {{ dist(_dist) }} - {{ sort(_sort_type, _sort) }} - ; - - insert into {{ relation.include(database=(not temporary), schema=(not temporary)) }} - ( - {{ sql }} - ) - ; - - {%- else %} - - create {% if temporary -%}temporary{%- endif %} table - {{ relation.include(database=(not temporary), schema=(not temporary)) }} - {% if backup == false -%}backup no{%- endif %} - {{ dist(_dist) }} - {{ sort(_sort_type, _sort) }} - as ( - {{ sql }} - ); - - {%- endif %} -{%- endmacro %} - - -{% macro redshift__create_view_as(relation, sql) -%} - {%- set binding = config.get('bind', default=True) -%} - - {% set bind_qualifier = '' if binding else 'with no schema binding' %} - {%- set sql_header = config.get('sql_header', none) -%} - - {{ sql_header if sql_header is not none }} - - create view {{ relation }} - {%- set contract_config = config.get('contract') -%} - {%- if contract_config.enforced -%} - {{ get_assert_columns_equivalent(sql) }} - {%- endif %} as ( - {{ sql }} - ) {{ bind_qualifier }}; -{% endmacro %} - - -{% macro redshift__create_schema(relation) -%} - {{ postgres__create_schema(relation) }} -{% endmacro %} - - -{% macro redshift__drop_schema(relation) -%} - {{ postgres__drop_schema(relation) }} -{% endmacro %} - - {% macro redshift__get_columns_in_relation(relation) -%} {% call statement('get_columns_in_relation', fetch_result=True) %} with bound_views as ( @@ -255,16 +148,6 @@ {%- endmacro %} -{% macro redshift__list_schemas(database) -%} - {{ return(postgres__list_schemas(database)) }} -{%- endmacro %} - - -{% macro redshift__check_schema_exists(information_schema, schema) -%} - {{ return(postgres__check_schema_exists(information_schema, schema)) }} -{%- endmacro %} - - {% macro redshift__persist_docs(relation, model, for_relation, for_columns) -%} {% if for_relation and config.persist_relation_docs() and model.description %} {% do run_query(alter_relation_comment(relation, model.description)) %} @@ -313,12 +196,3 @@ {% endif %} {% endmacro %} - - -{% macro redshift__get_drop_relation_sql(relation) %} - {%- if relation.is_materialized_view -%} - {{ redshift__drop_materialized_view(relation) }} - {%- else -%} - drop {{ relation.type }} if exists {{ relation }} cascade - {%- endif -%} -{% endmacro %} diff --git a/dbt/include/redshift/macros/relations.sql b/dbt/include/redshift/macros/get_relations.sql similarity index 100% rename from dbt/include/redshift/macros/relations.sql rename to dbt/include/redshift/macros/get_relations.sql diff --git a/dbt/include/redshift/macros/materializations/seeds/helpers.sql b/dbt/include/redshift/macros/materializations/seed.sql similarity index 100% rename from dbt/include/redshift/macros/materializations/seeds/helpers.sql rename to dbt/include/redshift/macros/materializations/seed.sql diff --git a/dbt/include/redshift/macros/materializations/snapshot_merge.sql b/dbt/include/redshift/macros/materializations/snapshot_merge.sql index eda314727..f3be9aa8b 100644 --- a/dbt/include/redshift/macros/materializations/snapshot_merge.sql +++ b/dbt/include/redshift/macros/materializations/snapshot_merge.sql @@ -1,4 +1,3 @@ - {% macro redshift__snapshot_merge_sql(target, source, insert_cols) -%} {{ postgres__snapshot_merge_sql(target, source, insert_cols) }} {% endmacro %} diff --git a/dbt/include/redshift/macros/relation_components/schema.sql b/dbt/include/redshift/macros/relation_components/schema.sql new file mode 100644 index 000000000..f5ba889a0 --- /dev/null +++ b/dbt/include/redshift/macros/relation_components/schema.sql @@ -0,0 +1,21 @@ +{# /* + These are `BaseRelation` versions. The `BaseRelation` workflows are different. +*/ #} +{% macro redshift__create_schema(relation) -%} + {{ postgres__create_schema(relation) }} +{% endmacro %} + + +{% macro redshift__drop_schema(relation) -%} + {{ postgres__drop_schema(relation) }} +{% endmacro %} + + +{% macro redshift__list_schemas(database) -%} + {{ return(postgres__list_schemas(database)) }} +{%- endmacro %} + + +{% macro redshift__check_schema_exists(information_schema, schema) -%} + {{ return(postgres__check_schema_exists(information_schema, schema)) }} +{%- endmacro %} diff --git a/dbt/include/redshift/macros/relation_components/sort.sql b/dbt/include/redshift/macros/relation_components/sort.sql new file mode 100644 index 000000000..3015081f8 --- /dev/null +++ b/dbt/include/redshift/macros/relation_components/sort.sql @@ -0,0 +1,16 @@ +{# /* + These are `BaseRelation` versions. The `BaseRelation` workflows are different. +*/ #} +{% macro sort(sort_type, sort) %} + {%- if sort is not none %} + {{ sort_type | default('compound', boolean=true) }} sortkey( + {%- if sort is string -%} + {%- set sort = [sort] -%} + {%- endif -%} + {%- for item in sort -%} + {{ item }} + {%- if not loop.last -%},{%- endif -%} + {%- endfor -%} + ) + {%- endif %} +{%- endmacro -%} diff --git a/dbt/include/redshift/macros/relations/table.sql b/dbt/include/redshift/macros/relations/table.sql new file mode 100644 index 000000000..924592d05 --- /dev/null +++ b/dbt/include/redshift/macros/relations/table.sql @@ -0,0 +1,60 @@ +{%- macro redshift_drop_table_template(table) -%} + drop table if exists {{ table.fully_qualified_path }} cascade +{%- endmacro -%} + + +{%- macro redshift__rename_table_template(table, new_name) -%} + alter table {{ table.fully_qualified_path }} rename to {{ new_name }} +{%- endmacro -%} + + +{# /* + These are `BaseRelation` versions. The `BaseRelation` workflows are different. +*/ #} + +{% macro redshift__create_table_as(temporary, relation, sql) -%} + + {%- set _dist = config.get('dist') -%} + {%- set _sort_type = config.get( + 'sort_type', + validator=validation.any['compound', 'interleaved']) -%} + {%- set _sort = config.get( + 'sort', + validator=validation.any[list, basestring]) -%} + {%- set sql_header = config.get('sql_header', none) -%} + {%- set backup = config.get('backup') -%} + + {{ sql_header if sql_header is not none }} + + {%- set contract_config = config.get('contract') -%} + {%- if contract_config.enforced -%} + + create {% if temporary -%}temporary{%- endif %} table + {{ relation.include(database=(not temporary), schema=(not temporary)) }} + {{ get_table_columns_and_constraints() }} + {{ get_assert_columns_equivalent(sql) }} + {%- set sql = get_select_subquery(sql) %} + {% if backup == false -%}backup no{%- endif %} + {{ dist(_dist) }} + {{ sort(_sort_type, _sort) }} + ; + + insert into {{ relation.include(database=(not temporary), schema=(not temporary)) }} + ( + {{ sql }} + ) + ; + + {%- else %} + + create {% if temporary -%}temporary{%- endif %} table + {{ relation.include(database=(not temporary), schema=(not temporary)) }} + {% if backup == false -%}backup no{%- endif %} + {{ dist(_dist) }} + {{ sort(_sort_type, _sort) }} + as ( + {{ sql }} + ); + + {%- endif %} +{%- endmacro %} diff --git a/dbt/include/redshift/macros/relations/view.sql b/dbt/include/redshift/macros/relations/view.sql new file mode 100644 index 000000000..5fe65fe60 --- /dev/null +++ b/dbt/include/redshift/macros/relations/view.sql @@ -0,0 +1,29 @@ +{%- macro redshift__drop_view_template(view) -%} + drop view if exists {{ view.fully_qualified_path }} cascade +{%- endmacro -%} + + +{%- macro redshift__rename_view_template(view, new_name) -%} + alter view {{ view.fully_qualified_path }} rename to {{ new_name }} +{%- endmacro -%} + + +{# /* + These are `BaseRelation` versions. The `BaseRelation` workflows are different. +*/ #} +{% macro redshift__create_view_as(relation, sql) -%} + {%- set binding = config.get('bind', default=True) -%} + + {% set bind_qualifier = '' if binding else 'with no schema binding' %} + {%- set sql_header = config.get('sql_header', none) -%} + + {{ sql_header if sql_header is not none }} + + create view {{ relation }} + {%- set contract_config = config.get('contract') -%} + {%- if contract_config.enforced -%} + {{ get_assert_columns_equivalent(sql) }} + {%- endif %} as ( + {{ sql }} + ) {{ bind_qualifier }}; +{% endmacro %} diff --git a/dbt/include/redshift/macros/timestamps.sql b/dbt/include/redshift/macros/utils/timestamps.sql similarity index 100% rename from dbt/include/redshift/macros/timestamps.sql rename to dbt/include/redshift/macros/utils/timestamps.sql From 9f62fb0e49c3f14bc2e4e72dfab0f9c7042e6632 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 11 Jul 2023 01:10:24 -0400 Subject: [PATCH 03/12] new relation/materialization first draft --- dbt/adapters/redshift/materialization.py | 0 .../redshift/materialization_config/schema.py | 0 dbt/adapters/redshift/relation/__init__.py | 25 ++++ .../materializations/materialized_view.sql | 106 --------------- .../macros/relation_components/dist.sql | 16 +++ .../macros/relations/materialized_view.sql | 122 ++++++++++++++++++ 6 files changed, 163 insertions(+), 106 deletions(-) delete mode 100644 dbt/adapters/redshift/materialization.py delete mode 100644 dbt/adapters/redshift/materialization_config/schema.py create mode 100644 dbt/adapters/redshift/relation/__init__.py delete mode 100644 dbt/include/redshift/macros/materializations/materialized_view.sql create mode 100644 dbt/include/redshift/macros/relation_components/dist.sql create mode 100644 dbt/include/redshift/macros/relations/materialized_view.sql diff --git a/dbt/adapters/redshift/materialization.py b/dbt/adapters/redshift/materialization.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/dbt/adapters/redshift/materialization_config/schema.py b/dbt/adapters/redshift/materialization_config/schema.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/dbt/adapters/redshift/relation/__init__.py b/dbt/adapters/redshift/relation/__init__.py new file mode 100644 index 000000000..af04d34fa --- /dev/null +++ b/dbt/adapters/redshift/relation/__init__.py @@ -0,0 +1,25 @@ +from dataclasses import dataclass + +from dbt.adapters.base.relation import BaseRelation +from dbt.exceptions import DbtRuntimeError + +from dbt.adapters.redshift.relation.models import MAX_CHARACTERS_IN_IDENTIFIER + + +@dataclass(frozen=True, eq=False, repr=False) +class RedshiftRelation(BaseRelation): + def __post_init__(self): + # Check for length of Redshift table/view names. + # Check self.type to exclude test relation identifiers + if ( + self.identifier is not None + and self.type is not None + and len(self.identifier) > MAX_CHARACTERS_IN_IDENTIFIER + ): + raise DbtRuntimeError( + f"Relation name '{self.identifier}' " + f"is longer than {MAX_CHARACTERS_IN_IDENTIFIER} characters" + ) + + def relation_max_name_length(self): + return MAX_CHARACTERS_IN_IDENTIFIER diff --git a/dbt/include/redshift/macros/materializations/materialized_view.sql b/dbt/include/redshift/macros/materializations/materialized_view.sql deleted file mode 100644 index 6343c1a7b..000000000 --- a/dbt/include/redshift/macros/materializations/materialized_view.sql +++ /dev/null @@ -1,106 +0,0 @@ -{% macro redshift__get_alter_materialized_view_as_sql( - relation, - configuration_changes, - sql, - existing_relation, - backup_relation, - intermediate_relation -) %} - - -- apply a full refresh immediately if needed - {% if configuration_changes.requires_full_refresh %} - - {{ get_replace_materialized_view_as_sql(relation, sql, existing_relation, backup_relation, intermediate_relation) }} - - -- otherwise apply individual changes as needed - {% else %} - - {%- set autorefresh = configuration_changes.autorefresh -%} - {%- if autorefresh -%}{{- log('Applying UPDATE AUTOREFRESH to: ' ~ relation) -}}{%- endif -%} - - alter materialized view {{ relation }} - auto refresh {% if autorefresh.context %}yes{% else %}no{% endif %} - - {%- endif -%} - -{% endmacro %} - - -{% macro redshift__get_create_materialized_view_as_sql(relation, sql) %} - - {%- set materialized_view = relation.from_runtime_config(config) -%} - - create materialized view {{ materialized_view.path }} - backup {% if materialized_view.backup %}yes{% else %}no{% endif %} - diststyle {{ materialized_view.dist.diststyle }} - {% if materialized_view.dist.distkey %}distkey ({{ materialized_view.dist.distkey }}){% endif %} - {% if materialized_view.sort.sortkey %}sortkey ({{ ','.join(materialized_view.sort.sortkey) }}){% endif %} - auto refresh {% if materialized_view.auto_refresh %}yes{% else %}no{% endif %} - as ( - {{ materialized_view.query }} - ); - -{% endmacro %} - - -{% macro redshift__get_replace_materialized_view_as_sql(relation, sql, existing_relation, backup_relation, intermediate_relation) %} - {{ redshift__get_drop_relation_sql(existing_relation) }}; - {{ get_create_materialized_view_as_sql(relation, sql) }} -{% endmacro %} - - -{% macro redshift__get_materialized_view_configuration_changes(existing_relation, new_config) %} - {% set _existing_materialized_view = redshift__describe_materialized_view(existing_relation) %} - {% set _configuration_changes = existing_relation.materialized_view_config_changeset(_existing_materialized_view, new_config) %} - {% do return(_configuration_changes) %} -{% endmacro %} - - -{% macro redshift__refresh_materialized_view(relation) -%} - refresh materialized view {{ relation }} -{% endmacro %} - - -{% macro redshift__describe_materialized_view(relation) %} - {#- - These need to be separate queries because redshift will not let you run queries - against svv_table_info and pg_views in the same query. The same is true of svv_redshift_columns. - -#} - - {%- set _materialized_view_sql -%} - select - tb.database, - tb.schema, - tb.table, - tb.diststyle, - tb.sortkey1, - mv.autorefresh - from svv_table_info tb - left join stv_mv_info mv - on mv.db_name = tb.database - and mv.schema = tb.schema - and mv.name = tb.table - where tb.table ilike '{{ relation.identifier }}' - and tb.schema ilike '{{ relation.schema }}' - and tb.database ilike '{{ relation.database }}' - {%- endset %} - {% set _materialized_view = run_query(_materialized_view_sql) %} - - {%- set _query_sql -%} - select - vw.definition - from pg_views vw - where vw.viewname = '{{ relation.identifier }}' - and vw.schemaname = '{{ relation.schema }}' - and vw.definition ilike '%create materialized view%' - {%- endset %} - {% set _query = run_query(_query_sql) %} - - {% do return({'materialized_view': _materialized_view, 'query': _query}) %} - -{% endmacro %} - - -{% macro redshift__drop_materialized_view(relation) -%} - drop materialized view if exists {{ relation }} -{%- endmacro %} diff --git a/dbt/include/redshift/macros/relation_components/dist.sql b/dbt/include/redshift/macros/relation_components/dist.sql new file mode 100644 index 000000000..af6e62084 --- /dev/null +++ b/dbt/include/redshift/macros/relation_components/dist.sql @@ -0,0 +1,16 @@ +{# /* + These are `BaseRelation` versions. The `BaseRelation` workflows are different. +*/ #} +{% macro dist(dist) %} + {%- if dist is not none -%} + {%- set dist = dist.strip().lower() -%} + + {%- if dist in ['all', 'even'] -%} + diststyle {{ dist }} + {%- elif dist == "auto" -%} + {%- else -%} + diststyle key distkey ({{ dist }}) + {%- endif -%} + + {%- endif -%} +{%- endmacro -%} diff --git a/dbt/include/redshift/macros/relations/materialized_view.sql b/dbt/include/redshift/macros/relations/materialized_view.sql new file mode 100644 index 000000000..cd20bec9a --- /dev/null +++ b/dbt/include/redshift/macros/relations/materialized_view.sql @@ -0,0 +1,122 @@ +{#- /* + This file contains DDL that gets consumed in the default materialized view materialization in `dbt-core`. + These macros could be used elsewhere as they do not care that they are being called by a materialization; + but the original intention was to support the materialization of materialized views. These macros represent + the basic interactions dbt-postgres requires of materialized views in Postgres: + - ALTER + - CREATE + - DESCRIBE + - DROP + - REFRESH + - RENAME + These macros all take a `RedshiftMaterializedViewRelation` instance as an input. This class can be found in: + `dbt/adapters/redshift/relation/models/materialized_view.py` + + Used in: + `dbt/include/global_project/macros/materializations/models/materialized_view/materialized_view.sql` + Uses: + `dbt/adapters/redshift/relation/factory.py` +*/ -#} + +{% macro redshift__alter_materialized_view_template(existing_materialized_view, target_materialized_view) %} + + {%- if target_materialized_view == existing_materialized_view -%} + {{- exceptions.warn("No changes were identified for: " ~ existing_materialized_view) -}} + + {%- else -%} + {%- set _changeset = adapter.make_changeset(existing_materialized_view, target_materialized_view) -%} + + {% if _changeset.requires_full_refresh %} + {{ replace_template(existing_materialized_view, target_materialized_view) }} + + {% else %} + + {%- set autorefresh = _changeset.autorefresh -%} + {%- if autorefresh -%} + {{- log('Applying UPDATE AUTOREFRESH to: ' ~ existing_materialized_view) -}} + alter materialized view {{ existing_materialized_view.fully_qualified_path }} + auto refresh {% if autorefresh.context %}yes{% else %}no{% endif %} + {%- endif -%} + + {%- endif -%} + {%- endif -%} + +{% endmacro %} + + +{% macro redshift__create_materialized_view_template(materialized_view) %} + + create materialized view {{ materialized_view.fully_qualified_path }} + backup {% if materialized_view.backup %}yes{% else %}no{% endif %} + diststyle {{ materialized_view.dist.diststyle }} + {% if materialized_view.dist.distkey %}distkey ({{ materialized_view.dist.distkey }}){% endif %} + {% if materialized_view.sort.sortkey %}sortkey ({{ ','.join(materialized_view.sort.sortkey) }}){% endif %} + auto refresh {% if materialized_view.auto_refresh %}yes{% else %}no{% endif %} + as ( + {{ materialized_view.query }} + ); + +{% endmacro %} + + +{% macro redshift__describe_materialized_view_template(materialized_view) %} + {#- + These need to be separate queries because redshift will not let you run queries + against svv_table_info and pg_views in the same query. The same is true of svv_redshift_columns. + -#} + + {%- set _materialized_view_sql -%} + select + tb.database, + tb.schema, + tb.table, + tb.diststyle, + tb.sortkey1, + mv.autorefresh + from svv_table_info tb + left join stv_mv_info mv + on mv.db_name = tb.database + and mv.schema = tb.schema + and mv.name = tb.table + where tb.table ilike '{{ materialized_view.name }}' + and tb.schema ilike '{{ materialized_view.schema_name }}' + and tb.database ilike '{{ materialized_view.database_name }}' + {%- endset %} + {% set _materialized_view = run_query(_materialized_view_sql) %} + + {%- set _query_sql -%} + select + vw.definition + from pg_views vw + where vw.viewname = '{{ materialized_view.name }}' + and vw.schemaname = '{{ materialized_view.schema_name }}' + and vw.definition ilike '%create materialized view%' + {%- endset %} + {% set _query = run_query(_query_sql) %} + + {% do return({'materialized_view': _materialized_view, 'query': _query}) %} + +{% endmacro %} + + +{% macro redshift__drop_materialized_view_template(materialized_view) -%} + drop materialized view if exists {{ materialized_view.fully_qualified_path }} +{%- endmacro %} + + +{% macro redshift__refresh_materialized_view_template(materialized_view) -%} + refresh materialized view {{ materialized_view.fully_qualified_path }} +{% endmacro %} + + +{%- macro postgres__rename_materialized_view_template(materialized_view, new_name) -%} + + {%- if adapter.is_relation_model(materialized_view) -%} + {%- set fully_qualified_path = materialized_view.fully_qualified_path -%} + {%- else -%} + {%- set fully_qualified_path = materialized_view -%} + {%- endif -%} + + alter materialized view {{ fully_qualified_path }} rename to {{ new_name }} + +{%- endmacro -%} From 43678c7e1641948dc3d6671603f814b6a7804724 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 11 Jul 2023 01:21:39 -0400 Subject: [PATCH 04/12] setup initial tests, need to update for redshift --- tests/unit/conftest.py | 194 ++++++++++++++++++ .../test_materialization.py | 34 +++ .../test_materialization_factory.py | 36 ++++ .../model_tests/test_database.py | 24 +++ .../relation_tests/model_tests/test_index.py | 25 +++ .../model_tests/test_materialized_view.py | 88 ++++++++ .../relation_tests/model_tests/test_schema.py | 31 +++ .../relation_tests/test_relation_factory.py | 79 +++++++ 8 files changed, 511 insertions(+) create mode 100644 tests/unit/conftest.py create mode 100644 tests/unit/materialization_tests/test_materialization.py create mode 100644 tests/unit/materialization_tests/test_materialization_factory.py create mode 100644 tests/unit/relation_tests/model_tests/test_database.py create mode 100644 tests/unit/relation_tests/model_tests/test_index.py create mode 100644 tests/unit/relation_tests/model_tests/test_materialized_view.py create mode 100644 tests/unit/relation_tests/model_tests/test_schema.py create mode 100644 tests/unit/relation_tests/test_relation_factory.py diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py new file mode 100644 index 000000000..5833cfa4a --- /dev/null +++ b/tests/unit/conftest.py @@ -0,0 +1,194 @@ +from dataclasses import dataclass + +import agate +import pytest + +from dbt.adapters.materialization.factory import MaterializationFactory +from dbt.adapters.materialization.models import ( + MaterializationType, + MaterializedViewMaterialization, +) +from dbt.adapters.relation.factory import RelationFactory +from dbt.contracts.files import FileHash +from dbt.contracts.graph.model_config import OnConfigurationChangeOption +from dbt.contracts.graph.nodes import DependsOn, ModelNode, NodeConfig +from dbt.contracts.relation import RelationType +from dbt.node_types import NodeType + +from dbt.adapters.redshift.relation import models + + +@pytest.fixture +def relation_factory(): + return RelationFactory( + relation_models={ + RelationType.MaterializedView: models.RedshiftMaterializedViewRelation, + }, + relation_changesets={ + RelationType.MaterializedView: models.RedshiftMaterializedViewRelationChangeset, + }, + relation_can_be_renamed={RelationType.MaterializedView}, + render_policy=models.RedshiftRenderPolicy, + ) + + +@pytest.fixture +def materialization_factory(relation_factory): + return MaterializationFactory( + relation_factory=relation_factory, + materialization_map={ + MaterializationType.MaterializedView: MaterializedViewMaterialization + }, + ) + + +@pytest.fixture +def materialized_view_stub(relation_factory): + return relation_factory.make_stub( + name="my_materialized_view", + schema_name="my_schema", + database_name="my_database", + relation_type=RelationType.MaterializedView, + ) + + +@pytest.fixture +def view_stub(relation_factory): + return relation_factory.make_stub( + name="my_view", + schema_name="my_schema", + database_name="my_database", + relation_type=RelationType.View, + ) + + +@pytest.fixture +def materialized_view_describe_relation_results(): + # TODO separate query and add in sort/dist info + materialized_view_agate = agate.Table.from_object( + [ + { + "name": "my_materialized_view", + "schema_name": "my_schema", + "database_name": "my_database", + "query": "select 42 from meaning_of_life", + } + ] + ) + return {"materialized_view": materialized_view_agate} + + +@pytest.fixture +def materialized_view_model_node(): + return ModelNode( + alias="my_materialized_view", + name="my_materialized_view", + database="my_database", + schema="my_schema", + resource_type=NodeType.Model, + unique_id="model.root.my_materialized_view", + fqn=["root", "my_materialized_view"], + package_name="root", + original_file_path="my_materialized_view.sql", + refs=[], + sources=[], + depends_on=DependsOn(), + config=NodeConfig.from_dict( + { + "enabled": True, + "materialized": "materialized_view", + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "vars": {}, + "quoting": {}, + "column_types": {}, + "tags": [], + # TODO replace this with sort/dist info + "indexes": [ + {"columns": ["id", "value"], "type": "hash"}, + {"columns": ["id"], "unique": True}, + ], + } + ), + tags=[], + path="my_materialized_view.sql", + language="sql", + raw_code="select 42 from meaning_of_life", + compiled_code="select 42 from meaning_of_life", + description="", + columns={}, + checksum=FileHash.from_contents(""), + ) + + +@pytest.fixture +def materialized_view_relation(relation_factory, materialized_view_describe_relation_results): + return relation_factory.make_from_describe_relation_results( + materialized_view_describe_relation_results, RelationType.MaterializedView + ) + + +@pytest.fixture +def materialized_view_runtime_config(materialized_view_model_node): + """ + This is not actually a `RuntimeConfigObject`. It's an object that has attribution that looks like + a boiled down version of a RuntimeConfigObject. + + TODO: replace this with an actual `RuntimeConfigObject` + """ + + @dataclass() + class RuntimeConfigObject: + model: ModelNode + full_refresh: bool + grants: dict + on_configuration_change: OnConfigurationChangeOption + + def get(self, attribute: str, default=None): + return getattr(self, attribute, default) + + return RuntimeConfigObject( + model=materialized_view_model_node, + full_refresh=False, + grants={}, + on_configuration_change=OnConfigurationChangeOption.Continue, + ) + + +""" +Make sure the fixtures at least work, more thorough testing is done elsewhere +""" + + +def test_relation_factory(relation_factory): + assert ( + relation_factory._get_parser(RelationType.MaterializedView) + == models.RedshiftMaterializedViewRelation + ) + + +def test_materialization_factory(materialization_factory): + redshift_parser = materialization_factory.relation_factory._get_parser( + RelationType.MaterializedView + ) + assert redshift_parser == models.RedshiftMaterializedViewRelation + + +def test_materialized_view_stub(materialized_view_stub): + assert materialized_view_stub.name == "my_materialized_view" + + +def test_materialized_view_model_node(materialized_view_model_node): + assert materialized_view_model_node.name == "my_materialized_view" + + +def test_materialized_view_runtime_config(materialized_view_runtime_config): + assert materialized_view_runtime_config.get("full_refresh", False) is False + assert materialized_view_runtime_config.get("on_configuration_change", "apply") == "continue" + assert materialized_view_runtime_config.model.name == "my_materialized_view" + + +def test_materialized_view_relation(materialized_view_relation): + assert materialized_view_relation.type == RelationType.MaterializedView + assert materialized_view_relation.name == "my_materialized_view" diff --git a/tests/unit/materialization_tests/test_materialization.py b/tests/unit/materialization_tests/test_materialization.py new file mode 100644 index 000000000..f8c51675c --- /dev/null +++ b/tests/unit/materialization_tests/test_materialization.py @@ -0,0 +1,34 @@ +from dataclasses import replace + +from dbt.adapters.materialization.models import ( + MaterializedViewMaterialization, + MaterializationBuildStrategy, +) + + +def test_materialized_view_create(materialized_view_runtime_config, relation_factory): + materialization = MaterializedViewMaterialization.from_runtime_config( + materialized_view_runtime_config, relation_factory + ) + assert materialization.build_strategy == MaterializationBuildStrategy.Create + assert materialization.should_revoke_grants is False + + +def test_materialized_view_replace(materialized_view_runtime_config, relation_factory, view_stub): + materialization = MaterializedViewMaterialization.from_runtime_config( + materialized_view_runtime_config, relation_factory, view_stub + ) + assert materialization.build_strategy == MaterializationBuildStrategy.Replace + assert materialization.should_revoke_grants is True + + +def test_materialized_view_alter( + materialized_view_runtime_config, relation_factory, materialized_view_relation +): + altered_materialized_view = replace(materialized_view_relation, indexes={}) + + materialization = MaterializedViewMaterialization.from_runtime_config( + materialized_view_runtime_config, relation_factory, altered_materialized_view + ) + assert materialization.build_strategy == MaterializationBuildStrategy.Alter + assert materialization.should_revoke_grants is True diff --git a/tests/unit/materialization_tests/test_materialization_factory.py b/tests/unit/materialization_tests/test_materialization_factory.py new file mode 100644 index 000000000..f176dca4a --- /dev/null +++ b/tests/unit/materialization_tests/test_materialization_factory.py @@ -0,0 +1,36 @@ +from dbt.adapters.materialization.models import MaterializationType +from dbt.contracts.relation import RelationType + +from dbt.adapters.redshift.relation import models as relation_models + + +def test_make_from_runtime_config(materialization_factory, materialized_view_runtime_config): + materialization = materialization_factory.make_from_runtime_config( + runtime_config=materialized_view_runtime_config, + materialization_type=MaterializationType.MaterializedView, + existing_relation_stub=None, + ) + assert materialization.type == MaterializationType.MaterializedView + + materialized_view = materialization.target_relation + assert materialized_view.type == RelationType.MaterializedView + + assert materialized_view.name == "my_materialized_view" + assert materialized_view.schema_name == "my_schema" + assert materialized_view.database_name == "my_database" + assert materialized_view.query == "select 42 from meaning_of_life" + + index_1 = relation_models.RedshiftIndexRelation( + column_names=frozenset({"id", "value"}), + method=relation_models.RedshiftIndexMethod.hash, + unique=False, + render=relation_models.RedshiftRenderPolicy, + ) + index_2 = relation_models.RedshiftIndexRelation( + column_names=frozenset({"id"}), + method=relation_models.RedshiftIndexMethod.btree, + unique=True, + render=relation_models.RedshiftRenderPolicy, + ) + assert index_1 in materialized_view.indexes + assert index_2 in materialized_view.indexes diff --git a/tests/unit/relation_tests/model_tests/test_database.py b/tests/unit/relation_tests/model_tests/test_database.py new file mode 100644 index 000000000..e0432512a --- /dev/null +++ b/tests/unit/relation_tests/model_tests/test_database.py @@ -0,0 +1,24 @@ +from typing import Type + +import pytest +from dbt.exceptions import DbtRuntimeError + +from dbt.adapters.redshift.relation.models import RedshiftDatabaseRelation + + +@pytest.mark.parametrize( + "config_dict,exception", + [ + ({"name": "my_database"}, None), + ({"name": ""}, DbtRuntimeError), + ({"wrong_name": "my_database"}, DbtRuntimeError), + ({}, DbtRuntimeError), + ], +) +def test_make_database(config_dict: dict, exception: Type[Exception]): + if exception: + with pytest.raises(exception): + RedshiftDatabaseRelation.from_dict(config_dict) + else: + my_database = RedshiftDatabaseRelation.from_dict(config_dict) + assert my_database.name == config_dict.get("name") diff --git a/tests/unit/relation_tests/model_tests/test_index.py b/tests/unit/relation_tests/model_tests/test_index.py new file mode 100644 index 000000000..11cd355b2 --- /dev/null +++ b/tests/unit/relation_tests/model_tests/test_index.py @@ -0,0 +1,25 @@ +from typing import Type + +import pytest +from dbt.exceptions import DbtRuntimeError + +from dbt.adapters.redshift.relation.models import RedshiftIndexRelation + + +@pytest.mark.parametrize( + "config_dict,exception", + [ + ({"column_names": frozenset({"id", "value"}), "method": "hash", "unique": False}, None), + ({"column_names": frozenset("id"), "method": "btree", "unique": True}, None), + ({}, DbtRuntimeError), + ({"method": "btree", "unique": True}, DbtRuntimeError), + ], +) +# TODO replace this with sort/dist stuff +def test_create_index(config_dict: dict, exception: Type[Exception]): + if exception: + with pytest.raises(exception): + RedshiftIndexRelation.from_dict(config_dict) + else: + my_index = RedshiftIndexRelation.from_dict(config_dict) + assert my_index.column_names == config_dict.get("column_names") diff --git a/tests/unit/relation_tests/model_tests/test_materialized_view.py b/tests/unit/relation_tests/model_tests/test_materialized_view.py new file mode 100644 index 000000000..7166154f8 --- /dev/null +++ b/tests/unit/relation_tests/model_tests/test_materialized_view.py @@ -0,0 +1,88 @@ +from dataclasses import replace +from typing import Type + +import pytest + +from dbt.exceptions import DbtRuntimeError + +from dbt.adapters.redshift.relation.models import ( + RedshiftMaterializedViewRelation, + RedshiftMaterializedViewRelationChangeset, +) + + +@pytest.mark.parametrize( + "config_dict,exception", + [ + ( + { + "name": "my_materialized_view", + "schema": { + "name": "my_schema", + "database": {"name": "my_database"}, + }, + "query": "select 1 from my_favoriate_table", + }, + None, + ), + ( + { + "name": "my_indexed_materialized_view", + "schema": { + "name": "my_schema", + "database": {"name": "my_database"}, + }, + "query": "select 42 from meaning_of_life", + "indexes": [ + { + "column_names": frozenset({"id", "value"}), + "method": "hash", + "unique": False, + }, + {"column_names": frozenset({"id"}), "method": "btree", "unique": True}, + ], + }, + None, + ), + ( + { + "my_name": "my_materialized_view", + "schema": { + "name": "my_schema", + "database": {"name": "my_database"}, + }, + }, + DbtRuntimeError, + ), + ], +) +def test_create_materialized_view(config_dict: dict, exception: Type[Exception]): + if exception: + with pytest.raises(exception): + RedshiftMaterializedViewRelation.from_dict(config_dict) + else: + my_materialized_view = RedshiftMaterializedViewRelation.from_dict(config_dict) + assert my_materialized_view.name == config_dict.get("name") + assert my_materialized_view.schema_name == config_dict.get("schema").get("name") + assert my_materialized_view.database_name == config_dict.get("schema").get("database").get( + "name" + ) + assert my_materialized_view.query == config_dict.get("query") + if indexes := config_dict.get("indexes"): + parsed = {(index.method, index.unique) for index in my_materialized_view.indexes} + raw = {(index.get("method"), index.get("unique")) for index in indexes} + assert parsed == raw + + +def test_create_materialized_view_changeset(materialized_view_relation): + existing_materialized_view = replace(materialized_view_relation) + + # pulled from `./dbt_postgres_tests/conftest.py` + # TODO update with sort/dist/autorefresh/backup stuff + target_materialized_view = replace(existing_materialized_view, indexes=frozenset({})) + + changeset = RedshiftMaterializedViewRelationChangeset.from_relations( + existing_materialized_view, target_materialized_view + ) + assert changeset.is_empty is False + assert changeset.requires_full_refresh is False diff --git a/tests/unit/relation_tests/model_tests/test_schema.py b/tests/unit/relation_tests/model_tests/test_schema.py new file mode 100644 index 000000000..64a19ecbe --- /dev/null +++ b/tests/unit/relation_tests/model_tests/test_schema.py @@ -0,0 +1,31 @@ +from typing import Type + +import pytest +from dbt.exceptions import DbtRuntimeError + +from dbt.adapters.redshift.relation.models import RedshiftSchemaRelation + + +@pytest.mark.parametrize( + "config_dict,exception", + [ + ({"name": "my_schema", "database": {"name": "my_database"}}, None), + ({"name": "my_schema", "database": None}, DbtRuntimeError), + ({"name": "my_schema"}, DbtRuntimeError), + ({"name": "", "database": {"name": "my_database"}}, DbtRuntimeError), + ({"wrong_name": "my_database", "database": {"name": "my_database"}}, DbtRuntimeError), + ( + {"name": "my_schema", "database": {"name": "my_database"}, "meaning_of_life": 42}, + DbtRuntimeError, + ), + ({}, DbtRuntimeError), + ], +) +def test_make_schema(config_dict: dict, exception: Type[Exception]): + if exception: + with pytest.raises(exception): + RedshiftSchemaRelation.from_dict(config_dict) + else: + my_schema = RedshiftSchemaRelation.from_dict(config_dict) + assert my_schema.name == config_dict.get("name") + assert my_schema.database_name == config_dict.get("database").get("name") diff --git a/tests/unit/relation_tests/test_relation_factory.py b/tests/unit/relation_tests/test_relation_factory.py new file mode 100644 index 000000000..3a35152f1 --- /dev/null +++ b/tests/unit/relation_tests/test_relation_factory.py @@ -0,0 +1,79 @@ +""" +Uses the following fixtures in `unit/dbt_redshift_tests/conftest.py`: +- `relation_factory` +- `materialized_view_stub` +""" + +from dbt.contracts.relation import RelationType + +from dbt.adapters.postgres.relation import models + + +def test_make_stub(materialized_view_stub): + assert materialized_view_stub.name == "my_materialized_view" + assert materialized_view_stub.schema_name == "my_schema" + assert materialized_view_stub.database_name == "my_database" + assert materialized_view_stub.type == "materialized_view" + assert materialized_view_stub.can_be_renamed is True + + +def test_make_backup_stub(relation_factory, materialized_view_stub): + backup_stub = relation_factory.make_backup_stub(materialized_view_stub) + assert backup_stub.name == '"my_materialized_view__dbt_backup"' + + +def test_make_intermediate(relation_factory, materialized_view_stub): + intermediate_relation = relation_factory.make_intermediate(materialized_view_stub) + assert intermediate_relation.name == '"my_materialized_view__dbt_tmp"' + + +def test_make_from_describe_relation_results( + relation_factory, materialized_view_describe_relation_results +): + materialized_view = relation_factory.make_from_describe_relation_results( + materialized_view_describe_relation_results, RelationType.MaterializedView + ) + + assert materialized_view.name == "my_materialized_view" + assert materialized_view.schema_name == "my_schema" + assert materialized_view.database_name == "my_database" + assert materialized_view.query == "select 42 from meaning_of_life" + + index_1 = models.RedshiftIndexRelation( + column_names=frozenset({"id", "value"}), + method=models.RedshiftIndexMethod.hash, + unique=False, + render=models.RedshiftRenderPolicy, + ) + index_2 = models.RedshiftIndexRelation( + column_names=frozenset({"id"}), + method=models.RedshiftIndexMethod.btree, + unique=True, + render=models.RedshiftRenderPolicy, + ) + assert index_1 in materialized_view.indexes + assert index_2 in materialized_view.indexes + + +def test_make_from_model_node(relation_factory, materialized_view_model_node): + materialized_view = relation_factory.make_from_model_node(materialized_view_model_node) + + assert materialized_view.name == "my_materialized_view" + assert materialized_view.schema_name == "my_schema" + assert materialized_view.database_name == "my_database" + assert materialized_view.query == "select 42 from meaning_of_life" + + index_1 = models.RedshiftIndexRelation( + column_names=frozenset({"id", "value"}), + method=models.RedshiftIndexMethod.hash, + unique=False, + render=models.RedshiftRenderPolicy, + ) + index_2 = models.RedshiftIndexRelation( + column_names=frozenset({"id"}), + method=models.RedshiftIndexMethod.btree, + unique=True, + render=models.RedshiftRenderPolicy, + ) + assert index_1 in materialized_view.indexes + assert index_2 in materialized_view.indexes From f138912ef7ba6adc35403d44e80a12b62d7f6a75 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 11 Jul 2023 01:30:37 -0400 Subject: [PATCH 05/12] changie --- .changes/unreleased/Features-20230711-013016.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Features-20230711-013016.yaml diff --git a/.changes/unreleased/Features-20230711-013016.yaml b/.changes/unreleased/Features-20230711-013016.yaml new file mode 100644 index 000000000..7c4315a22 --- /dev/null +++ b/.changes/unreleased/Features-20230711-013016.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Add materialized view support with the new change management framework +time: 2023-07-11T01:30:16.22273-04:00 +custom: + Author: mikealfare + Issue: dbt-labs/dbt-core#6911 From 2437891fa45413cc0e4c6b8a9cd1addf53da54a4 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 12 Jul 2023 01:51:44 -0400 Subject: [PATCH 06/12] implement adap-608 framework from core for materialized views --- dbt/adapters/redshift/impl.py | 5 +- .../redshift/relation/models/__init__.py | 10 +- dbt/adapters/redshift/relation/models/dist.py | 22 +- .../relation/models/materialized_view.py | 57 ++- dbt/adapters/redshift/relation/models/sort.py | 24 +- .../macros/materializations/table.sql | 77 ++++ .../redshift/macros/materializations/view.sql | 72 +++ .../macros/relations/materialized_view.sql | 30 +- .../redshift/macros/relations/table.sql | 2 +- .../redshift/macros/relations/view.sql | 3 +- .../materialized_view_tests/conftest.py | 59 +++ .../adapter/materialized_view_tests/files.py | 33 ++ .../materialized_view_tests/fixtures.py | 85 ---- .../test_materialized_views.py | 416 ++++++++++-------- .../adapter/materialized_view_tests/utils.py | 84 ++++ tests/unit/conftest.py | 37 +- .../test_materialization.py | 10 +- .../test_materialization_factory.py | 28 +- .../relation_tests/model_tests/test_dist.py | 30 ++ .../relation_tests/model_tests/test_index.py | 25 -- .../model_tests/test_materialized_view.py | 97 +++- .../relation_tests/model_tests/test_sort.py | 30 ++ .../relation_tests/test_relation_factory.py | 85 ++-- 23 files changed, 858 insertions(+), 463 deletions(-) create mode 100644 dbt/include/redshift/macros/materializations/table.sql create mode 100644 dbt/include/redshift/macros/materializations/view.sql create mode 100644 tests/functional/adapter/materialized_view_tests/conftest.py create mode 100644 tests/functional/adapter/materialized_view_tests/files.py delete mode 100644 tests/functional/adapter/materialized_view_tests/fixtures.py create mode 100644 tests/functional/adapter/materialized_view_tests/utils.py create mode 100644 tests/unit/relation_tests/model_tests/test_dist.py delete mode 100644 tests/unit/relation_tests/model_tests/test_index.py create mode 100644 tests/unit/relation_tests/model_tests/test_sort.py diff --git a/dbt/adapters/redshift/impl.py b/dbt/adapters/redshift/impl.py index b7fdf787d..507085931 100644 --- a/dbt/adapters/redshift/impl.py +++ b/dbt/adapters/redshift/impl.py @@ -57,7 +57,10 @@ def relation_factory(self): relation_changesets={ RelationType.MaterializedView: relation_models.RedshiftMaterializedViewRelationChangeset, }, - relation_can_be_renamed={RelationType.MaterializedView}, + relation_can_be_renamed={ + RelationType.Table, + RelationType.View, + }, render_policy=relation_models.RedshiftRenderPolicy, ) diff --git a/dbt/adapters/redshift/relation/models/__init__.py b/dbt/adapters/redshift/relation/models/__init__.py index aa21a54dc..bf98b30ad 100644 --- a/dbt/adapters/redshift/relation/models/__init__.py +++ b/dbt/adapters/redshift/relation/models/__init__.py @@ -1,5 +1,8 @@ from dbt.adapters.redshift.relation.models.database import RedshiftDatabaseRelation -from dbt.adapters.redshift.relation.models.dist import RedshiftDistRelation +from dbt.adapters.redshift.relation.models.dist import ( + RedshiftDistRelation, + RedshiftDistStyle, +) from dbt.adapters.redshift.relation.models.materialized_view import ( RedshiftMaterializedViewRelation, RedshiftMaterializedViewRelationChangeset, @@ -9,4 +12,7 @@ MAX_CHARACTERS_IN_IDENTIFIER, ) from dbt.adapters.redshift.relation.models.schema import RedshiftSchemaRelation -from dbt.adapters.redshift.relation.models.sort import RedshiftSortRelation +from dbt.adapters.redshift.relation.models.sort import ( + RedshiftSortRelation, + RedshiftSortStyle, +) diff --git a/dbt/adapters/redshift/relation/models/dist.py b/dbt/adapters/redshift/relation/models/dist.py index a8523f052..0f85f6004 100644 --- a/dbt/adapters/redshift/relation/models/dist.py +++ b/dbt/adapters/redshift/relation/models/dist.py @@ -1,3 +1,4 @@ +from copy import deepcopy from dataclasses import dataclass from typing import Optional, Set @@ -12,6 +13,8 @@ from dbt.dataclass_schema import StrEnum from dbt.exceptions import DbtRuntimeError +from dbt.adapters.redshift.relation.models.policy import RedshiftRenderPolicy + class RedshiftDistStyle(StrEnum): auto = "auto" @@ -35,9 +38,13 @@ class RedshiftDistRelation(RelationComponent, ValidationMixin): - distkey: the column to use for the dist key if `dist_style` is `key` """ + # attribution diststyle: Optional[RedshiftDistStyle] = RedshiftDistStyle.default() distkey: Optional[str] = None + # configuration + render = RedshiftRenderPolicy + @property def validation_rules(self) -> Set[ValidationRule]: # index rules get run by default with the mixin @@ -64,11 +71,14 @@ def validation_rules(self) -> Set[ValidationRule]: @classmethod def from_dict(cls, config_dict) -> "RedshiftDistRelation": - kwargs_dict = { - "diststyle": config_dict.get("diststyle"), - "distkey": config_dict.get("distkey"), - } - dist: "RedshiftDistRelation" = super().from_dict(kwargs_dict) # type: ignore + # don't alter the incoming config + kwargs_dict = deepcopy(config_dict) + + if diststyle := config_dict.get("diststyle"): + kwargs_dict.update({"diststyle": RedshiftDistStyle(diststyle)}) + + dist = super().from_dict(kwargs_dict) + assert isinstance(dist, RedshiftDistRelation) return dist @classmethod @@ -118,7 +128,7 @@ def parse_describe_relation_results(cls, describe_relation_results: agate.Row) - Returns: a standard dictionary describing this `RedshiftDistConfig` instance """ - dist: str = describe_relation_results.get("diststyle") + dist: str = describe_relation_results.get("dist") try: # covers `AUTO`, `ALL`, `EVEN`, `KEY`, '', diff --git a/dbt/adapters/redshift/relation/models/materialized_view.py b/dbt/adapters/redshift/relation/models/materialized_view.py index e17f3ffac..f0345ff40 100644 --- a/dbt/adapters/redshift/relation/models/materialized_view.py +++ b/dbt/adapters/redshift/relation/models/materialized_view.py @@ -5,10 +5,10 @@ import agate from dbt.adapters.relation.models import ( MaterializedViewRelation, + MaterializedViewRelationChangeset, Relation, RelationChange, RelationChangeAction, - RelationChangeset, ) from dbt.adapters.validation import ValidationMixin, ValidationRule from dbt.contracts.graph.nodes import ModelNode @@ -61,16 +61,14 @@ class RedshiftMaterializedViewRelation(MaterializedViewRelation, ValidationMixin schema: RedshiftSchemaRelation query: str backup: Optional[bool] = True - dist: RedshiftDistRelation = RedshiftDistRelation( - diststyle=RedshiftDistStyle.even, render=RedshiftRenderPolicy - ) - sort: RedshiftSortRelation = RedshiftSortRelation(render=RedshiftRenderPolicy) + dist: RedshiftDistRelation = RedshiftDistRelation.from_dict({"diststyle": "even"}) + sort: RedshiftSortRelation = RedshiftSortRelation.from_dict({}) autorefresh: Optional[bool] = False # configuration render = RedshiftRenderPolicy SchemaParser = RedshiftSchemaRelation # type: ignore - can_be_renamed = True + can_be_renamed = False @property def validation_rules(self) -> Set[ValidationRule]: @@ -119,8 +117,8 @@ def parse_model_node(cls, model_node: ModelNode) -> dict: config_dict.update( { - "backup": model_node.config.get("backup"), - "autorefresh": model_node.config.get("auto_refresh"), + "backup": model_node.config.extra.get("backup"), + "autorefresh": model_node.config.extra.get("autorefresh"), } ) @@ -145,11 +143,11 @@ def parse_describe_relation_results( { "materialized_view": agate.Table( agate.Row({ - "database": "", - "schema": "", - "table": "", - "diststyle": "", # e.g. EVEN | KEY(column1) | AUTO(ALL) | AUTO(KEY(id)), - "sortkey1": "", + "database_name": "", + "schema_name": "", + "name": "", + "dist": "", # e.g. EVEN | KEY(column1) | AUTO(ALL) | AUTO(KEY(id)), + "sortkey": "", "autorefresh: any("t", "f"), }) ), @@ -162,33 +160,45 @@ def parse_describe_relation_results( Returns: a standard dictionary describing this `RedshiftMaterializedViewConfig` instance """ + # merge these because the base class assumes `query` is on the same record as `name`, `schema_name` and + # `database_name` + describe_relation_results = cls._combine_describe_relation_results_tables( + describe_relation_results + ) config_dict = super().parse_describe_relation_results(describe_relation_results) materialized_view: agate.Row = describe_relation_results["materialized_view"].rows[0] - query: agate.Row = describe_relation_results["query"].rows[0] - config_dict.update( { - "autorefresh": {"t": True, "f": False}.get(materialized_view.get("autorefresh")), - "query": cls._parse_query(query.get("definition")), + "autorefresh": materialized_view.get("autorefresh"), + "query": cls._parse_query(materialized_view.get("query")), } ) # the default for materialized views differs from the default for diststyle in general # only set it if we got a value - if materialized_view.get("diststyle"): + if materialized_view.get("dist"): config_dict.update( {"dist": RedshiftDistRelation.parse_describe_relation_results(materialized_view)} ) # TODO: this only shows the first column in the sort key - if materialized_view.get("sortkey1"): + if materialized_view.get("sortkey"): config_dict.update( {"sort": RedshiftSortRelation.parse_describe_relation_results(materialized_view)} ) return config_dict + @classmethod + def _combine_describe_relation_results_tables( + cls, describe_relation_results: Dict[str, agate.Table] + ) -> Dict[str, agate.Table]: + materialized_view_table: agate.Table = describe_relation_results["materialized_view"] + query_table: agate.Table = describe_relation_results["query"] + combined_table: agate.Table = materialized_view_table.join(query_table, full_outer=True) + return {"materialized_view": combined_table} + @classmethod def _parse_query(cls, query: str) -> str: """ @@ -211,9 +221,10 @@ def _parse_query(cls, query: str) -> str: select * from my_base_table """ - open_paren = query.find("as (") + len("as (") - close_paren = query.find(");") - return query[open_paren:close_paren].strip() + return query + # open_paren = query.find("as (") + # close_paren = query.find(");") + # return query[open_paren:close_paren].strip() @dataclass(frozen=True, eq=True, unsafe_hash=True) @@ -235,7 +246,7 @@ def requires_full_refresh(self) -> bool: @dataclass -class RedshiftMaterializedViewRelationChangeset(RelationChangeset): +class RedshiftMaterializedViewRelationChangeset(MaterializedViewRelationChangeset): backup: Optional[RedshiftBackupRelationChange] = None dist: Optional[RedshiftDistRelationChange] = None sort: Optional[RedshiftSortRelationChange] = None diff --git a/dbt/adapters/redshift/relation/models/sort.py b/dbt/adapters/redshift/relation/models/sort.py index c07b3b71a..26939d4fd 100644 --- a/dbt/adapters/redshift/relation/models/sort.py +++ b/dbt/adapters/redshift/relation/models/sort.py @@ -1,5 +1,5 @@ from copy import deepcopy -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Optional, FrozenSet, Set import agate @@ -43,15 +43,16 @@ class RedshiftSortRelation(RelationComponent, ValidationMixin): - sort_key: the column(s) to use for the sort key; cannot be combined with `sort_type=auto` """ + # attribution sortstyle: Optional[RedshiftSortStyle] = None - sortkey: Optional[FrozenSet[str]] = None + sortkey: Optional[FrozenSet[str]] = field(default_factory=frozenset) # type: ignore # configuration render = RedshiftRenderPolicy def __post_init__(self): # maintains `frozen=True` while allowing for a variable default on `sort_type` - if self.sortstyle is None and self.sortkey is None: + if self.sortstyle is None and self.sortkey == frozenset(): object.__setattr__(self, "sortstyle", RedshiftSortStyle.default()) elif self.sortstyle is None: object.__setattr__(self, "sortstyle", RedshiftSortStyle.default_with_columns()) @@ -63,7 +64,7 @@ def validation_rules(self) -> Set[ValidationRule]: return { ValidationRule( validation_check=not ( - self.sortstyle == RedshiftSortStyle.auto and self.sortkey is not None + self.sortstyle == RedshiftSortStyle.auto and self.sortkey != frozenset() ), validation_error=DbtRuntimeError( "A `RedshiftSortConfig` that specifies a `sortkey` does not support the `sortstyle` of `auto`." @@ -72,7 +73,7 @@ def validation_rules(self) -> Set[ValidationRule]: ValidationRule( validation_check=not ( self.sortstyle in (RedshiftSortStyle.compound, RedshiftSortStyle.interleaved) - and self.sortkey is None + and self.sortkey == frozenset() ), validation_error=DbtRuntimeError( "A `sortstyle` of `compound` or `interleaved` requires a `sortkey` to be provided." @@ -105,12 +106,11 @@ def from_dict(cls, config_dict) -> "RedshiftSortRelation": # don't alter the incoming config kwargs_dict = deepcopy(config_dict) - kwargs_dict.update( - { - "sortstyle": config_dict.get("sortstyle"), - "sortkey": frozenset(column for column in config_dict.get("sortkey", {})), - } - ) + if sortstyle := config_dict.get("sortstyle"): + kwargs_dict.update({"sortstyle": RedshiftSortStyle(sortstyle)}) + + if sortkey := config_dict.get("sortkey"): + kwargs_dict.update({"sortkey": frozenset(column for column in sortkey)}) sort = super().from_dict(kwargs_dict) assert isinstance(sort, RedshiftSortRelation) @@ -165,7 +165,7 @@ def parse_describe_relation_results(cls, describe_relation_results: agate.Row) - Returns: a standard dictionary describing this `RedshiftSortConfig` instance """ - if sortkey := describe_relation_results.get("sortkey1"): + if sortkey := describe_relation_results.get("sortkey"): return {"sortkey": {sortkey}} return {} diff --git a/dbt/include/redshift/macros/materializations/table.sql b/dbt/include/redshift/macros/materializations/table.sql new file mode 100644 index 000000000..a3b8bacdb --- /dev/null +++ b/dbt/include/redshift/macros/materializations/table.sql @@ -0,0 +1,77 @@ +{% /* + + Ideally we don't overwrite materializations from dbt-core. However, the implementation of materialized views + requires this, at least for now. There are two issues that lead to this. First, Redshift does not support + the renaming of materialized views. That means we cannot back them up when replacing them. If the relation + that's replacing it is another materialized view, we can control for that since the materialization for + materialized views in dbt-core is flexible. That brings us to the second issue. The materialization for table + has the backup/deploy portion built into it; it's one single macro; replacing that has two options. We + can either break apart the macro in dbt-core, which could have unintended downstream effects for all + adapters. Or we can copy this here and keep it up to date with dbt-core until we resolve the larger issue. + We chose to go with the latter. + +*/ %} + +{% materialization table, adapter='redshift', supported_languages=['sql'] %} + + {%- set existing_relation = load_cached_relation(this) -%} + {%- set target_relation = this.incorporate(type='table') %} + {%- set intermediate_relation = make_intermediate_relation(target_relation) -%} + -- the intermediate_relation should not already exist in the database; get_relation + -- will return None in that case. Otherwise, we get a relation that we can drop + -- later, before we try to use this name for the current operation + {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%} + /* + See ../view/view.sql for more information about this relation. + */ + {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%} + {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%} + -- as above, the backup_relation should not already exist + {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%} + -- grab current tables grants config for comparision later on + {% set grant_config = config.get('grants') %} + + -- drop the temp relations if they exist already in the database + {{ drop_relation_if_exists(preexisting_intermediate_relation) }} + {{ drop_relation_if_exists(preexisting_backup_relation) }} + + {{ run_hooks(pre_hooks, inside_transaction=False) }} + + -- `BEGIN` happens here: + {{ run_hooks(pre_hooks, inside_transaction=True) }} + + -- build model + {% call statement('main') -%} + {{ get_create_table_as_sql(False, intermediate_relation, sql) }} + {%- endcall %} + + -- cleanup -- this should be the only piece that differs from dbt-core + {% if existing_relation is not none %} + {% if existing_relation.type == 'materialized_view' %} + {{ drop_relation_if_exists(existing_relation) }} + {% else %} + {{ adapter.rename_relation(existing_relation, backup_relation) }} + {% endif %} + {% endif %} + + {{ adapter.rename_relation(intermediate_relation, target_relation) }} + + {% do create_indexes(target_relation) %} + + {{ run_hooks(post_hooks, inside_transaction=True) }} + + {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %} + {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} + + {% do persist_docs(target_relation, model) %} + + -- `COMMIT` happens here + {{ adapter.commit() }} + + -- finally, drop the existing/backup relation after the commit + {{ drop_relation_if_exists(backup_relation) }} + + {{ run_hooks(post_hooks, inside_transaction=False) }} + + {{ return({'relations': [target_relation]}) }} +{% endmaterialization %} diff --git a/dbt/include/redshift/macros/materializations/view.sql b/dbt/include/redshift/macros/materializations/view.sql new file mode 100644 index 000000000..1edd01d5b --- /dev/null +++ b/dbt/include/redshift/macros/materializations/view.sql @@ -0,0 +1,72 @@ +-- see the table materialization for notes +{%- materialization view, adapter='redshift', supported_languages=['sql'] -%} + + {%- set existing_relation = load_cached_relation(this) -%} + {%- set target_relation = this.incorporate(type='view') -%} + {%- set intermediate_relation = make_intermediate_relation(target_relation) -%} + + -- the intermediate_relation should not already exist in the database; get_relation + -- will return None in that case. Otherwise, we get a relation that we can drop + -- later, before we try to use this name for the current operation + {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%} + /* + This relation (probably) doesn't exist yet. If it does exist, it's a leftover from + a previous run, and we're going to try to drop it immediately. At the end of this + materialization, we're going to rename the "existing_relation" to this identifier, + and then we're going to drop it. In order to make sure we run the correct one of: + - drop view ... + - drop table ... + + We need to set the type of this relation to be the type of the existing_relation, if it exists, + or else "view" as a sane default if it does not. Note that if the existing_relation does not + exist, then there is nothing to move out of the way and subsequentally drop. In that case, + this relation will be effectively unused. + */ + {%- set backup_relation_type = 'view' if existing_relation is none else existing_relation.type -%} + {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%} + -- as above, the backup_relation should not already exist + {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%} + -- grab current tables grants config for comparision later on + {% set grant_config = config.get('grants') %} + + {{ run_hooks(pre_hooks, inside_transaction=False) }} + + -- drop the temp relations if they exist already in the database + {{ drop_relation_if_exists(preexisting_intermediate_relation) }} + {{ drop_relation_if_exists(preexisting_backup_relation) }} + + -- `BEGIN` happens here: + {{ run_hooks(pre_hooks, inside_transaction=True) }} + + -- build model + {% call statement('main') -%} + {{ get_create_view_as_sql(intermediate_relation, sql) }} + {%- endcall %} + + -- cleanup + -- move the existing view out of the way - unless it's a materialized view, then drop it + {% if existing_relation is not none %} + {% if existing_relation.type == 'materialized_view' %} + {{ drop_relation_if_exists(existing_relation) }} + {% else %} + {{ adapter.rename_relation(existing_relation, backup_relation) }} + {% endif %} + {% endif %} + {{ adapter.rename_relation(intermediate_relation, target_relation) }} + + {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %} + {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} + + {% do persist_docs(target_relation, model) %} + + {{ run_hooks(post_hooks, inside_transaction=True) }} + + {{ adapter.commit() }} + + {{ drop_relation_if_exists(backup_relation) }} + + {{ run_hooks(post_hooks, inside_transaction=False) }} + + {{ return({'relations': [target_relation]}) }} + +{%- endmaterialization -%} diff --git a/dbt/include/redshift/macros/relations/materialized_view.sql b/dbt/include/redshift/macros/relations/materialized_view.sql index cd20bec9a..987772155 100644 --- a/dbt/include/redshift/macros/relations/materialized_view.sql +++ b/dbt/include/redshift/macros/relations/materialized_view.sql @@ -51,10 +51,10 @@ diststyle {{ materialized_view.dist.diststyle }} {% if materialized_view.dist.distkey %}distkey ({{ materialized_view.dist.distkey }}){% endif %} {% if materialized_view.sort.sortkey %}sortkey ({{ ','.join(materialized_view.sort.sortkey) }}){% endif %} - auto refresh {% if materialized_view.auto_refresh %}yes{% else %}no{% endif %} + auto refresh {% if materialized_view.autorefresh %}yes{% else %}no{% endif %} as ( {{ materialized_view.query }} - ); + ) {% endmacro %} @@ -67,11 +67,11 @@ {%- set _materialized_view_sql -%} select - tb.database, - tb.schema, - tb.table, - tb.diststyle, - tb.sortkey1, + tb.database as database_name, + tb.schema as schema_name, + tb.table as name, + tb.diststyle as dist, + tb.sortkey1 as sortkey, mv.autorefresh from svv_table_info tb left join stv_mv_info mv @@ -86,7 +86,7 @@ {%- set _query_sql -%} select - vw.definition + vw.definition as query from pg_views vw where vw.viewname = '{{ materialized_view.name }}' and vw.schemaname = '{{ materialized_view.schema_name }}' @@ -109,14 +109,8 @@ {% endmacro %} -{%- macro postgres__rename_materialized_view_template(materialized_view, new_name) -%} - - {%- if adapter.is_relation_model(materialized_view) -%} - {%- set fully_qualified_path = materialized_view.fully_qualified_path -%} - {%- else -%} - {%- set fully_qualified_path = materialized_view -%} - {%- endif -%} - - alter materialized view {{ fully_qualified_path }} rename to {{ new_name }} - +{%- macro redshift__rename_materialized_view_template(materialized_view, new_name) -%} + {{- exceptions.raise_compiler_error( + "Redshift does not support the renaming of materialized views. This macro was called by: " ~ materialized_view + ) -}} {%- endmacro -%} diff --git a/dbt/include/redshift/macros/relations/table.sql b/dbt/include/redshift/macros/relations/table.sql index 924592d05..845773faf 100644 --- a/dbt/include/redshift/macros/relations/table.sql +++ b/dbt/include/redshift/macros/relations/table.sql @@ -1,4 +1,4 @@ -{%- macro redshift_drop_table_template(table) -%} +{%- macro redshift__drop_table_template(table) -%} drop table if exists {{ table.fully_qualified_path }} cascade {%- endmacro -%} diff --git a/dbt/include/redshift/macros/relations/view.sql b/dbt/include/redshift/macros/relations/view.sql index 5fe65fe60..efddb63f9 100644 --- a/dbt/include/redshift/macros/relations/view.sql +++ b/dbt/include/redshift/macros/relations/view.sql @@ -3,8 +3,9 @@ {%- endmacro -%} +{# /* Redshift uses `table` here even for a view. Replacing this with `view` will break. */ #} {%- macro redshift__rename_view_template(view, new_name) -%} - alter view {{ view.fully_qualified_path }} rename to {{ new_name }} + alter table {{ view.fully_qualified_path }} rename to {{ new_name }} {%- endmacro -%} diff --git a/tests/functional/adapter/materialized_view_tests/conftest.py b/tests/functional/adapter/materialized_view_tests/conftest.py new file mode 100644 index 000000000..4883ee20f --- /dev/null +++ b/tests/functional/adapter/materialized_view_tests/conftest.py @@ -0,0 +1,59 @@ +import pytest + +from dbt.adapters.relation.models import RelationRef +from dbt.adapters.relation.factory import RelationFactory +from dbt.contracts.relation import RelationType + +from dbt.adapters.redshift.relation import models as relation_models + + +@pytest.fixture(scope="class") +def relation_factory(): + return RelationFactory( + relation_models={ + RelationType.MaterializedView: relation_models.RedshiftMaterializedViewRelation, + }, + relation_can_be_renamed={RelationType.Table, RelationType.View}, + render_policy=relation_models.RedshiftRenderPolicy, + ) + + +@pytest.fixture(scope="class") +def my_materialized_view(project, relation_factory) -> RelationRef: + relation_ref = relation_factory.make_ref( + name="my_materialized_view", + schema_name=project.test_schema, + database_name=project.database, + relation_type=RelationType.MaterializedView, + ) + return relation_ref + + +@pytest.fixture(scope="class") +def my_view(project, relation_factory) -> RelationRef: + return relation_factory.make_ref( + name="my_view", + schema_name=project.test_schema, + database_name=project.database, + relation_type=RelationType.View, + ) + + +@pytest.fixture(scope="class") +def my_table(project, relation_factory) -> RelationRef: + return relation_factory.make_ref( + name="my_table", + schema_name=project.test_schema, + database_name=project.database, + relation_type=RelationType.Table, + ) + + +@pytest.fixture(scope="class") +def my_seed(project, relation_factory) -> RelationRef: + return relation_factory.make_ref( + name="my_seed", + schema_name=project.test_schema, + database_name=project.database, + relation_type=RelationType.Table, + ) diff --git a/tests/functional/adapter/materialized_view_tests/files.py b/tests/functional/adapter/materialized_view_tests/files.py new file mode 100644 index 000000000..c76045084 --- /dev/null +++ b/tests/functional/adapter/materialized_view_tests/files.py @@ -0,0 +1,33 @@ +MY_SEED = """ +id,value +1,100 +2,200 +3,300 +""".strip() + + +MY_TABLE = """ +{{ config( + materialized='table', +) }} +select * from {{ ref('my_seed') }} +""" + + +MY_VIEW = """ +{{ config( + materialized='view', +) }} +select * from {{ ref('my_seed') }} +""" + + +MY_MATERIALIZED_VIEW = """ +{{ config( + materialized='materialized_view', + sort_type='compound', + sort=['id'], + dist='id' +) }} +select * from {{ ref('my_seed') }} +""" diff --git a/tests/functional/adapter/materialized_view_tests/fixtures.py b/tests/functional/adapter/materialized_view_tests/fixtures.py deleted file mode 100644 index 785931c1b..000000000 --- a/tests/functional/adapter/materialized_view_tests/fixtures.py +++ /dev/null @@ -1,85 +0,0 @@ -import pytest - -from dbt.tests.adapter.materialized_view.base import Base -from dbt.tests.adapter.materialized_view.on_configuration_change import ( - OnConfigurationChangeBase, - get_model_file, - set_model_file, -) -from dbt.tests.util import relation_from_name, run_sql_with_adapter - - -def refresh_materialized_view(project, name: str): - sql = f"refresh materialized view {relation_from_name(project.adapter, name)}" - run_sql_with_adapter(project.adapter, sql) - - -class RedshiftBasicBase(Base): - @pytest.fixture(scope="class") - def models(self): - base_table = """ - {{ config(materialized='table') }} - select 1 as base_column - """ - base_materialized_view = """ - {{ config(materialized='materialized_view') }} - select * from {{ ref('base_table') }} - """ - return {"base_table.sql": base_table, "base_materialized_view.sql": base_materialized_view} - - -class RedshiftOnConfigurationChangeBase(OnConfigurationChangeBase): - @pytest.fixture(scope="class") - def models(self): - base_table = """ - {{ config( - materialized='table', - ) }} - select - 1 as id, - 100 as value - """ - base_materialized_view = """ - {{ config( - materialized='materialized_view', - sort='id' - ) }} - select * from {{ ref('base_table') }} - """ - return {"base_table.sql": base_table, "base_materialized_view.sql": base_materialized_view} - - @pytest.fixture(scope="function") - def configuration_changes_apply(self, project): - initial_model = get_model_file(project, "base_materialized_view") - - # turn on auto_refresh - new_model = initial_model.replace( - "materialized='materialized_view',", - "materialized='materialized_view', auto_refresh='yes',", - ) - set_model_file(project, "base_materialized_view", new_model) - - yield - - # set this back for the next test - set_model_file(project, "base_materialized_view", initial_model) - - @pytest.fixture(scope="function") - def configuration_changes_refresh(self, project): - initial_model = get_model_file(project, "base_materialized_view") - - # add a sort_key - new_model = initial_model.replace( - "sort='id'", - "sort='value'", - ) - set_model_file(project, "base_materialized_view", new_model) - - yield - - # set this back for the next test - set_model_file(project, "base_materialized_view", initial_model) - - @pytest.fixture(scope="function") - def update_auto_refresh_message(self, project): - return f"Applying UPDATE AUTOREFRESH to: {relation_from_name(project.adapter, 'base_materialized_view')}" diff --git a/tests/functional/adapter/materialized_view_tests/test_materialized_views.py b/tests/functional/adapter/materialized_view_tests/test_materialized_views.py index ff63f1e01..65c4d9471 100644 --- a/tests/functional/adapter/materialized_view_tests/test_materialized_views.py +++ b/tests/functional/adapter/materialized_view_tests/test_materialized_views.py @@ -1,239 +1,277 @@ import pytest from dbt.contracts.graph.model_config import OnConfigurationChangeOption -from dbt.contracts.relation import RelationType -from dbt.contracts.results import RunStatus -from dbt.tests.adapter.materialized_view.base import ( - run_model, - assert_model_exists_and_is_correct_type, - insert_record, - get_row_count, +from dbt.tests.util import ( + assert_message_in_logs, + get_model_file, + run_dbt, + run_dbt_and_capture, + set_model_file, ) -from dbt.tests.adapter.materialized_view.on_configuration_change import ( - assert_proper_scenario, +from tests.functional.adapter.materialized_view_tests.files import ( + MY_MATERIALIZED_VIEW, + MY_SEED, + MY_TABLE, + MY_VIEW, ) - -from tests.functional.adapter.materialized_view_tests.fixtures import ( - RedshiftBasicBase, - RedshiftOnConfigurationChangeBase, - refresh_materialized_view, +from tests.functional.adapter.materialized_view_tests.utils import ( + query_autorefresh, + query_relation_type, + query_row_count, + query_sort, + swap_autorefresh, + swap_materialized_view_to_table, + swap_materialized_view_to_view, + swap_sortkey, ) -class TestBasic(RedshiftBasicBase): - def test_relation_is_materialized_view_on_initial_creation(self, project): - assert_model_exists_and_is_correct_type( - project, "base_materialized_view", RelationType.MaterializedView - ) - assert_model_exists_and_is_correct_type(project, "base_table", RelationType.Table) +@pytest.fixture(scope="class", autouse=True) +def seeds(): + return {"my_seed.csv": MY_SEED} - def test_relation_is_materialized_view_when_rerun(self, project): - run_model("base_materialized_view") - assert_model_exists_and_is_correct_type( - project, "base_materialized_view", RelationType.MaterializedView - ) - def test_relation_is_materialized_view_on_full_refresh(self, project): - run_model("base_materialized_view", full_refresh=True) - assert_model_exists_and_is_correct_type( - project, "base_materialized_view", RelationType.MaterializedView - ) +@pytest.fixture(scope="class", autouse=True) +def models(): + yield { + "my_table.sql": MY_TABLE, + "my_view.sql": MY_VIEW, + "my_materialized_view.sql": MY_MATERIALIZED_VIEW, + } - def test_relation_is_materialized_view_on_update(self, project): - run_model("base_materialized_view", run_args=["--vars", "quoting: {identifier: True}"]) - assert_model_exists_and_is_correct_type( - project, "base_materialized_view", RelationType.MaterializedView - ) - def test_updated_base_table_data_only_shows_in_materialized_view_after_refresh(self, project): - # poll database - table_start = get_row_count(project, "base_table") - view_start = get_row_count(project, "base_materialized_view") - assert view_start == table_start +@pytest.fixture(scope="class", autouse=True) +def setup(project): + run_dbt(["seed"]) + yield - # insert new record in table - new_record = (2,) - insert_record(project, new_record, "base_table", ["base_column"]) - # poll database - table_mid = get_row_count(project, "base_table") - view_mid = get_row_count(project, "base_materialized_view") +def test_materialized_view_create(project, my_materialized_view): + assert query_relation_type(project, my_materialized_view) is None + run_dbt(["run", "--models", my_materialized_view.name]) + assert query_relation_type(project, my_materialized_view) == "materialized_view" - # refresh the materialized view - refresh_materialized_view(project, "base_materialized_view") - # poll database - table_end = get_row_count(project, "base_table") - view_end = get_row_count(project, "base_materialized_view") - assert view_end == table_end +def test_materialized_view_create_idempotent(project, my_materialized_view): + assert query_relation_type(project, my_materialized_view) is None + run_dbt(["run", "--models", my_materialized_view.name]) + assert query_relation_type(project, my_materialized_view) == "materialized_view" + run_dbt(["run", "--models", my_materialized_view.name]) + assert query_relation_type(project, my_materialized_view) == "materialized_view" - # new records were inserted in the table but didn't show up in the view until it was refreshed - assert table_start < table_mid == table_end - assert view_start == view_mid < view_end +def test_materialized_view_full_refresh(project, my_materialized_view): + run_dbt(["run", "--models", my_materialized_view.name]) + _, logs = run_dbt_and_capture( + ["--debug", "run", "--models", my_materialized_view.name, "--full-refresh"] + ) + assert query_relation_type(project, my_materialized_view) == "materialized_view" + assert_message_in_logs( + f"Applying REPLACE to: {my_materialized_view.fully_qualified_path}", logs + ) -class TestOnConfigurationChangeApply(RedshiftOnConfigurationChangeBase): - def test_full_refresh_takes_precedence_over_any_configuration_changes( - self, - configuration_changes_apply, - configuration_changes_refresh, - replace_message, - configuration_change_message, - ): - results, logs = run_model("base_materialized_view", full_refresh=True) - assert_proper_scenario( - OnConfigurationChangeOption.Apply, - results, - logs, - RunStatus.Success, - messages_in_logs=[replace_message], - messages_not_in_logs=[configuration_change_message], - ) - def test_model_is_refreshed_with_no_configuration_changes( - self, refresh_message, configuration_change_message - ): - results, logs = run_model("base_materialized_view") - assert_proper_scenario( - OnConfigurationChangeOption.Apply, - results, - logs, - RunStatus.Success, - messages_in_logs=[refresh_message, configuration_change_message], - ) +def test_materialized_view_replaces_table(project, my_materialized_view, my_table): + run_dbt(["run", "--models", my_table.name]) + sql = f""" + alter table {my_table.fully_qualified_path} + rename to {my_materialized_view.name} + """ + project.run_sql(sql) + assert query_relation_type(project, my_materialized_view) == "table" + run_dbt(["run", "--models", my_materialized_view.name]) + assert query_relation_type(project, my_materialized_view) == "materialized_view" - def test_model_applies_changes_with_small_configuration_changes( - self, configuration_changes_apply, alter_message, update_auto_refresh_message - ): - results, logs = run_model("base_materialized_view") - assert_proper_scenario( - OnConfigurationChangeOption.Apply, - results, - logs, - RunStatus.Success, - messages_in_logs=[alter_message, update_auto_refresh_message], + +def test_materialized_view_replaces_view(project, my_materialized_view, my_view): + run_dbt(["run", "--models", my_view.name]) + sql = f""" + alter table {my_view.fully_qualified_path} + rename to {my_materialized_view.name} + """ + project.run_sql(sql) + assert query_relation_type(project, my_materialized_view) == "view" + run_dbt(["run", "--models", my_materialized_view.name]) + assert query_relation_type(project, my_materialized_view) == "materialized_view" + + +def test_table_replaces_materialized_view(project, my_materialized_view): + run_dbt(["run", "--models", my_materialized_view.name]) + assert query_relation_type(project, my_materialized_view) == "materialized_view" + swap_materialized_view_to_table(project, my_materialized_view) + run_dbt(["run", "--models", my_materialized_view.name]) + assert query_relation_type(project, my_materialized_view) == "table" + + +def test_view_replaces_materialized_view(project, my_materialized_view, my_view): + run_dbt(["run", "--models", my_materialized_view.name]) + assert query_relation_type(project, my_materialized_view) == "materialized_view" + swap_materialized_view_to_view(project, my_materialized_view) + run_dbt(["run", "--models", my_materialized_view.name]) + assert query_relation_type(project, my_materialized_view) == "view" + + +def test_materialized_view_only_updates_after_refresh(project, my_materialized_view, my_seed): + run_dbt(["run", "--models", my_materialized_view.name]) + + # poll database + table_start = query_row_count(project, my_seed) + view_start = query_row_count(project, my_materialized_view) + + # insert new record in table + project.run_sql(f"insert into {my_seed.fully_qualified_path} (id, value) values (4, 400);") + + # poll database + table_mid = query_row_count(project, my_seed) + view_mid = query_row_count(project, my_materialized_view) + + # refresh the materialized view + project.run_sql(f"refresh materialized view {my_materialized_view.fully_qualified_path};") + + # poll database + table_end = query_row_count(project, my_seed) + view_end = query_row_count(project, my_materialized_view) + + # new records were inserted in the table but didn't show up in the view until it was refreshed + assert table_start < table_mid == table_end + assert view_start == view_mid < view_end + + +class OnConfigurationChangeBase: + @pytest.fixture(scope="class", autouse=True) + def models(self): + yield {"my_materialized_view.sql": MY_MATERIALIZED_VIEW} + + @pytest.fixture(scope="function", autouse=True) + def setup(self, project, my_materialized_view): + run_dbt(["seed"]) + + # make sure the model in the data reflects the files each time + run_dbt(["run", "--models", my_materialized_view.name, "--full-refresh"]) + + # the tests touch these files, store their contents in memory + initial_model = get_model_file(project, my_materialized_view) + + yield + + # and then reset them after the test runs + set_model_file(project, my_materialized_view, initial_model) + + +class TestOnConfigurationChangeApply(OnConfigurationChangeBase): + @pytest.fixture(scope="class") + def project_config_update(self): + return {"models": {"on_configuration_change": OnConfigurationChangeOption.Apply.value}} + + def test_autorefresh_change_is_applied_with_alter(self, project, my_materialized_view): + assert query_autorefresh(project, my_materialized_view) is False + swap_autorefresh(project, my_materialized_view) + _, logs = run_dbt_and_capture(["--debug", "run", "--models", my_materialized_view.name]) + assert query_autorefresh(project, my_materialized_view) is True + assert_message_in_logs( + f"Applying ALTER to: {my_materialized_view.fully_qualified_path}", logs + ) + assert_message_in_logs( + f"Applying UPDATE AUTOREFRESH to: {my_materialized_view.fully_qualified_path}", logs + ) + assert_message_in_logs( + f"Applying REPLACE to: {my_materialized_view.fully_qualified_path}", logs, False ) - def test_model_rebuilds_with_large_configuration_changes( - self, configuration_changes_refresh, alter_message, replace_message - ): - results, logs = run_model("base_materialized_view") - assert_proper_scenario( - OnConfigurationChangeOption.Apply, - results, - logs, - RunStatus.Success, - messages_in_logs=[alter_message, replace_message], + def test_sort_change_is_applied_with_replace(self, project, my_materialized_view): + assert query_sort(project, my_materialized_view) == "id" + swap_sortkey(project, my_materialized_view) + _, logs = run_dbt_and_capture(["--debug", "run", "--models", my_materialized_view.name]) + assert query_sort(project, my_materialized_view) == "value" + assert_message_in_logs( + f"Applying ALTER to: {my_materialized_view.fully_qualified_path}", logs + ) + assert_message_in_logs( + f"Applying REPLACE to: {my_materialized_view.fully_qualified_path}", logs ) - def test_model_only_rebuilds_with_large_configuration_changes( - self, - configuration_changes_apply, - configuration_changes_refresh, - alter_message, - replace_message, - update_auto_refresh_message, + def test_autorefresh_change_is_applied_with_replace_when_run_with_sort_change( + self, project, my_materialized_view ): - results, logs = run_model("base_materialized_view") - assert_proper_scenario( - OnConfigurationChangeOption.Apply, - results, + assert query_autorefresh(project, my_materialized_view) is False + swap_autorefresh(project, my_materialized_view) + swap_sortkey(project, my_materialized_view) + _, logs = run_dbt_and_capture(["--debug", "run", "--models", my_materialized_view.name]) + assert query_autorefresh(project, my_materialized_view) is True + assert_message_in_logs( + f"Applying ALTER to: {my_materialized_view.fully_qualified_path}", logs + ) + assert_message_in_logs( + f"Applying REPLACE to: {my_materialized_view.fully_qualified_path}", logs + ) + assert_message_in_logs( + f"Applying UPDATE AUTOREFRESH to: {my_materialized_view.fully_qualified_path}", logs, - RunStatus.Success, - messages_in_logs=[alter_message, replace_message], - messages_not_in_logs=[update_auto_refresh_message], + False, ) -class TestOnConfigurationChangeContinue(RedshiftOnConfigurationChangeBase): +class TestOnConfigurationChangeContinue(OnConfigurationChangeBase): @pytest.fixture(scope="class") def project_config_update(self): return {"models": {"on_configuration_change": OnConfigurationChangeOption.Continue.value}} - def test_full_refresh_takes_precedence_over_any_configuration_changes( - self, - configuration_changes_apply, - configuration_changes_refresh, - replace_message, - configuration_change_message, - ): - results, logs = run_model("base_materialized_view", full_refresh=True) - assert_proper_scenario( - OnConfigurationChangeOption.Continue, - results, + def test_autorefresh_change_is_not_applied(self, project, my_materialized_view): + assert query_autorefresh(project, my_materialized_view) is False + swap_autorefresh(project, my_materialized_view) + _, logs = run_dbt_and_capture(["--debug", "run", "--models", my_materialized_view.name]) + assert query_autorefresh(project, my_materialized_view) is False + assert_message_in_logs( + f"Configuration changes were identified and `on_configuration_change` was set" + f" to `continue` for `{my_materialized_view.fully_qualified_path}`", logs, - RunStatus.Success, - messages_in_logs=[replace_message], - messages_not_in_logs=[configuration_change_message], ) - - def test_model_is_refreshed_with_no_configuration_changes( - self, refresh_message, configuration_change_message - ): - results, logs = run_model("base_materialized_view") - assert_proper_scenario( - OnConfigurationChangeOption.Continue, - results, - logs, - RunStatus.Success, - messages_in_logs=[refresh_message, configuration_change_message], + assert_message_in_logs( + f"Applying ALTER to: {my_materialized_view.fully_qualified_path}", logs, False ) - - def test_model_is_skipped_with_configuration_changes( - self, configuration_changes_apply, configuration_change_continue_message - ): - results, logs = run_model("base_materialized_view") - assert_proper_scenario( - OnConfigurationChangeOption.Continue, - results, + assert_message_in_logs( + f"Applying UPDATE AUTOREFRESH to: {my_materialized_view.fully_qualified_path}", logs, - RunStatus.Success, - messages_in_logs=[configuration_change_continue_message], + False, ) + assert_message_in_logs( + f"Applying REPLACE to: {my_materialized_view.fully_qualified_path}", logs, False + ) + + def test_full_refresh_still_occurs_with_changes(self, project, my_materialized_view): + run_dbt(["run", "--models", my_materialized_view.name, "--full-refresh"]) + assert query_relation_type(project, my_materialized_view) == "materialized_view" -class TestOnConfigurationChangeFail(RedshiftOnConfigurationChangeBase): +class TestOnConfigurationChangeFail(OnConfigurationChangeBase): @pytest.fixture(scope="class") def project_config_update(self): return {"models": {"on_configuration_change": OnConfigurationChangeOption.Fail.value}} - def test_full_refresh_takes_precedence_over_any_configuration_changes( - self, - configuration_changes_apply, - configuration_changes_refresh, - replace_message, - configuration_change_message, - ): - results, logs = run_model("base_materialized_view", full_refresh=True) - assert_proper_scenario( - OnConfigurationChangeOption.Fail, - results, - logs, - RunStatus.Success, - messages_in_logs=[replace_message], - messages_not_in_logs=[configuration_change_message], + def test_autorefresh_change_is_not_applied(self, project, my_materialized_view): + assert query_autorefresh(project, my_materialized_view) is False + swap_autorefresh(project, my_materialized_view) + _, logs = run_dbt_and_capture( + ["--debug", "run", "--models", my_materialized_view.name], expect_pass=False ) - - def test_model_is_refreshed_with_no_configuration_changes( - self, refresh_message, configuration_change_message - ): - results, logs = run_model("base_materialized_view") - assert_proper_scenario( - OnConfigurationChangeOption.Fail, - results, + assert query_autorefresh(project, my_materialized_view) is False + assert_message_in_logs( + f"Configuration changes were identified and `on_configuration_change` was set" + f" to `fail` for `{my_materialized_view.fully_qualified_path}`", logs, - RunStatus.Success, - messages_in_logs=[refresh_message, configuration_change_message], ) - - def test_run_fails_with_configuration_changes( - self, configuration_changes_apply, configuration_change_fail_message - ): - results, logs = run_model("base_materialized_view", expect_pass=False) - assert_proper_scenario( - OnConfigurationChangeOption.Fail, - results, + assert_message_in_logs( + f"Applying ALTER to: {my_materialized_view.fully_qualified_path}", logs, False + ) + assert_message_in_logs( + f"Applying UPDATE AUTOREFRESH to: {my_materialized_view.fully_qualified_path}", logs, - RunStatus.Error, - messages_in_logs=[configuration_change_fail_message], + False, + ) + assert_message_in_logs( + f"Applying REPLACE to: {my_materialized_view.fully_qualified_path}", logs, False ) + + def test_full_refresh_still_occurs_with_changes(self, project, my_materialized_view): + run_dbt(["run", "--models", my_materialized_view.name, "--full-refresh"]) + assert query_relation_type(project, my_materialized_view) == "materialized_view" diff --git a/tests/functional/adapter/materialized_view_tests/utils.py b/tests/functional/adapter/materialized_view_tests/utils.py new file mode 100644 index 000000000..5f7980cbe --- /dev/null +++ b/tests/functional/adapter/materialized_view_tests/utils.py @@ -0,0 +1,84 @@ +from typing import Optional + +from dbt.adapters.relation.models import Relation +from dbt.tests.util import get_model_file, set_model_file + + +def swap_sortkey(project, my_materialized_view): + initial_model = get_model_file(project, my_materialized_view) + new_model = initial_model.replace("sort=['id']", "sort=['value']") + set_model_file(project, my_materialized_view, new_model) + + +def swap_autorefresh(project, my_materialized_view): + initial_model = get_model_file(project, my_materialized_view) + new_model = initial_model.replace("dist='id'", "dist='id', autorefresh=True") + set_model_file(project, my_materialized_view, new_model) + + +def swap_materialized_view_to_table(project, my_materialized_view): + initial_model = get_model_file(project, my_materialized_view) + new_model = initial_model.replace("materialized='materialized_view'", "materialized='table'") + set_model_file(project, my_materialized_view, new_model) + + +def swap_materialized_view_to_view(project, my_materialized_view): + initial_model = get_model_file(project, my_materialized_view) + new_model = initial_model.replace("materialized='materialized_view'", "materialized='view'") + set_model_file(project, my_materialized_view, new_model) + + +def query_relation_type(project, relation: Relation) -> Optional[str]: + sql = f""" + select + 'table' as relation_type + from pg_tables + where schemaname = '{relation.schema_name}' + and tablename = '{relation.name}' + union all + select + case + when definition ilike '%create materialized view%' + then 'materialized_view' + else 'view' + end as relation_type + from pg_views + where schemaname = '{relation.schema_name}' + and viewname = '{relation.name}' + """ + results = project.run_sql(sql, fetch="all") + if len(results) == 0: + return None + elif len(results) > 1: + raise ValueError(f"More than one instance of {relation.name} found!") + else: + return results[0][0] + + +def query_row_count(project, relation: Relation) -> int: + sql = f"select count(*) from {relation.fully_qualified_path};" + return project.run_sql(sql, fetch="one")[0] + + +def query_sort(project, relation: Relation) -> bool: + sql = f""" + select + tb.sortkey1 as sortkey + from svv_table_info tb + where tb.table ilike '{ relation.name }' + and tb.schema ilike '{ relation.schema_name }' + and tb.database ilike '{ relation.database_name }' + """ + return project.run_sql(sql, fetch="one")[0] + + +def query_autorefresh(project, relation: Relation) -> bool: + sql = f""" + select + case mv.autorefresh when 't' then True when 'f' then False end as autorefresh + from stv_mv_info mv + where trim(mv.name) ilike '{ relation.name }' + and trim(mv.schema) ilike '{ relation.schema_name }' + and trim(mv.db_name) ilike '{ relation.database_name }' + """ + return project.run_sql(sql, fetch="one")[0] diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 5833cfa4a..6b0f8a487 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -43,8 +43,8 @@ def materialization_factory(relation_factory): @pytest.fixture -def materialized_view_stub(relation_factory): - return relation_factory.make_stub( +def materialized_view_ref(relation_factory): + return relation_factory.make_ref( name="my_materialized_view", schema_name="my_schema", database_name="my_database", @@ -53,8 +53,8 @@ def materialized_view_stub(relation_factory): @pytest.fixture -def view_stub(relation_factory): - return relation_factory.make_stub( +def view_ref(relation_factory): + return relation_factory.make_ref( name="my_view", schema_name="my_schema", database_name="my_database", @@ -71,11 +71,22 @@ def materialized_view_describe_relation_results(): "name": "my_materialized_view", "schema_name": "my_schema", "database_name": "my_database", - "query": "select 42 from meaning_of_life", + "dist": """KEY("id")""", + "sortkey": "other_id", + "autorefresh": "t", } ] ) - return {"materialized_view": materialized_view_agate} + + query_agate = agate.Table.from_object( + [ + { + "query": "select 4 as id, 2 as other_id from meaning_of_life", + } + ] + ) + + return {"materialized_view": materialized_view_agate, "query": query_agate} @pytest.fixture @@ -104,11 +115,11 @@ def materialized_view_model_node(): "quoting": {}, "column_types": {}, "tags": [], - # TODO replace this with sort/dist info - "indexes": [ - {"columns": ["id", "value"], "type": "hash"}, - {"columns": ["id"], "unique": True}, - ], + "autorefresh": True, + "dist": "id", + "sort": ["other_id"], + "sort_type": "compound", + "backup": False, } ), tags=[], @@ -175,8 +186,8 @@ def test_materialization_factory(materialization_factory): assert redshift_parser == models.RedshiftMaterializedViewRelation -def test_materialized_view_stub(materialized_view_stub): - assert materialized_view_stub.name == "my_materialized_view" +def test_materialized_view_ref(materialized_view_ref): + assert materialized_view_ref.name == "my_materialized_view" def test_materialized_view_model_node(materialized_view_model_node): diff --git a/tests/unit/materialization_tests/test_materialization.py b/tests/unit/materialization_tests/test_materialization.py index f8c51675c..8f2a01441 100644 --- a/tests/unit/materialization_tests/test_materialization.py +++ b/tests/unit/materialization_tests/test_materialization.py @@ -5,6 +5,8 @@ MaterializationBuildStrategy, ) +from dbt.adapters.redshift.relation import models + def test_materialized_view_create(materialized_view_runtime_config, relation_factory): materialization = MaterializedViewMaterialization.from_runtime_config( @@ -14,9 +16,9 @@ def test_materialized_view_create(materialized_view_runtime_config, relation_fac assert materialization.should_revoke_grants is False -def test_materialized_view_replace(materialized_view_runtime_config, relation_factory, view_stub): +def test_materialized_view_replace(materialized_view_runtime_config, relation_factory, view_ref): materialization = MaterializedViewMaterialization.from_runtime_config( - materialized_view_runtime_config, relation_factory, view_stub + materialized_view_runtime_config, relation_factory, view_ref ) assert materialization.build_strategy == MaterializationBuildStrategy.Replace assert materialization.should_revoke_grants is True @@ -25,7 +27,9 @@ def test_materialized_view_replace(materialized_view_runtime_config, relation_fa def test_materialized_view_alter( materialized_view_runtime_config, relation_factory, materialized_view_relation ): - altered_materialized_view = replace(materialized_view_relation, indexes={}) + altered_materialized_view = replace( + materialized_view_relation, sort=models.RedshiftSortRelation.from_dict({}) + ) materialization = MaterializedViewMaterialization.from_runtime_config( materialized_view_runtime_config, relation_factory, altered_materialized_view diff --git a/tests/unit/materialization_tests/test_materialization_factory.py b/tests/unit/materialization_tests/test_materialization_factory.py index f176dca4a..f2bb0a93a 100644 --- a/tests/unit/materialization_tests/test_materialization_factory.py +++ b/tests/unit/materialization_tests/test_materialization_factory.py @@ -1,14 +1,14 @@ from dbt.adapters.materialization.models import MaterializationType from dbt.contracts.relation import RelationType -from dbt.adapters.redshift.relation import models as relation_models +from dbt.adapters.redshift.relation import models def test_make_from_runtime_config(materialization_factory, materialized_view_runtime_config): materialization = materialization_factory.make_from_runtime_config( runtime_config=materialized_view_runtime_config, materialization_type=MaterializationType.MaterializedView, - existing_relation_stub=None, + existing_relation_ref=None, ) assert materialization.type == MaterializationType.MaterializedView @@ -19,18 +19,16 @@ def test_make_from_runtime_config(materialization_factory, materialized_view_run assert materialized_view.schema_name == "my_schema" assert materialized_view.database_name == "my_database" assert materialized_view.query == "select 42 from meaning_of_life" - - index_1 = relation_models.RedshiftIndexRelation( - column_names=frozenset({"id", "value"}), - method=relation_models.RedshiftIndexMethod.hash, - unique=False, - render=relation_models.RedshiftRenderPolicy, + sort = models.RedshiftSortRelation( + sortstyle=models.RedshiftSortStyle.compound, + sortkey=frozenset({"other_id"}), + render=models.RedshiftRenderPolicy, ) - index_2 = relation_models.RedshiftIndexRelation( - column_names=frozenset({"id"}), - method=relation_models.RedshiftIndexMethod.btree, - unique=True, - render=relation_models.RedshiftRenderPolicy, + assert materialized_view.sort == sort + dist = models.RedshiftDistRelation( + diststyle=models.RedshiftDistStyle.key, + distkey="id", + render=models.RedshiftRenderPolicy, ) - assert index_1 in materialized_view.indexes - assert index_2 in materialized_view.indexes + assert materialized_view.dist == dist + assert materialized_view.autorefresh is True diff --git a/tests/unit/relation_tests/model_tests/test_dist.py b/tests/unit/relation_tests/model_tests/test_dist.py new file mode 100644 index 000000000..2297bf033 --- /dev/null +++ b/tests/unit/relation_tests/model_tests/test_dist.py @@ -0,0 +1,30 @@ +from typing import Type + +import pytest +from dbt.exceptions import DbtRuntimeError + +from dbt.adapters.redshift.relation.models import RedshiftDistRelation + + +@pytest.mark.parametrize( + "config_dict,exception", + [ + ({"diststyle": "auto"}, None), + ({"diststyle": "auto", "distkey": "id"}, DbtRuntimeError), + ({"diststyle": "even"}, None), + ({"diststyle": "even", "distkey": "id"}, DbtRuntimeError), + ({"diststyle": "all"}, None), + ({"diststyle": "all", "distkey": "id"}, DbtRuntimeError), + ({"diststyle": "key"}, DbtRuntimeError), + ({"diststyle": "key", "distkey": "id"}, None), + ({}, None), + ], +) +def test_create_index(config_dict: dict, exception: Type[Exception]): + if exception: + with pytest.raises(exception): + RedshiftDistRelation.from_dict(config_dict) + else: + my_dist = RedshiftDistRelation.from_dict(config_dict) + assert my_dist.diststyle == config_dict.get("diststyle", "auto") + assert my_dist.distkey == config_dict.get("distkey") diff --git a/tests/unit/relation_tests/model_tests/test_index.py b/tests/unit/relation_tests/model_tests/test_index.py deleted file mode 100644 index 11cd355b2..000000000 --- a/tests/unit/relation_tests/model_tests/test_index.py +++ /dev/null @@ -1,25 +0,0 @@ -from typing import Type - -import pytest -from dbt.exceptions import DbtRuntimeError - -from dbt.adapters.redshift.relation.models import RedshiftIndexRelation - - -@pytest.mark.parametrize( - "config_dict,exception", - [ - ({"column_names": frozenset({"id", "value"}), "method": "hash", "unique": False}, None), - ({"column_names": frozenset("id"), "method": "btree", "unique": True}, None), - ({}, DbtRuntimeError), - ({"method": "btree", "unique": True}, DbtRuntimeError), - ], -) -# TODO replace this with sort/dist stuff -def test_create_index(config_dict: dict, exception: Type[Exception]): - if exception: - with pytest.raises(exception): - RedshiftIndexRelation.from_dict(config_dict) - else: - my_index = RedshiftIndexRelation.from_dict(config_dict) - assert my_index.column_names == config_dict.get("column_names") diff --git a/tests/unit/relation_tests/model_tests/test_materialized_view.py b/tests/unit/relation_tests/model_tests/test_materialized_view.py index 7166154f8..fd2e5c5d8 100644 --- a/tests/unit/relation_tests/model_tests/test_materialized_view.py +++ b/tests/unit/relation_tests/model_tests/test_materialized_view.py @@ -6,8 +6,10 @@ from dbt.exceptions import DbtRuntimeError from dbt.adapters.redshift.relation.models import ( + RedshiftDistRelation, RedshiftMaterializedViewRelation, RedshiftMaterializedViewRelationChangeset, + RedshiftSortRelation, ) @@ -33,14 +35,9 @@ "database": {"name": "my_database"}, }, "query": "select 42 from meaning_of_life", - "indexes": [ - { - "column_names": frozenset({"id", "value"}), - "method": "hash", - "unique": False, - }, - {"column_names": frozenset({"id"}), "method": "btree", "unique": True}, - ], + "dist": {"diststyle": "key", "distkey": "id"}, + "sort": {"sortstyle": "compound", "sortkey": ["id", "value"]}, + "autorefresh": True, }, None, ), @@ -51,6 +48,31 @@ "name": "my_schema", "database": {"name": "my_database"}, }, + # missing "query" + }, + DbtRuntimeError, + ), + ( + { + "name": "my_materialized_view", + "schema": { + "name": "my_schema", + "database": {"name": "my_database"}, + }, + "query": "select 1 from my_favoriate_table", + "dist": {"diststyle": "auto"}, # "auto" not supported for Redshift MVs + }, + DbtRuntimeError, + ), + ( + { + "name": "my_super_long_named_materialized_view" + * 10, # names must be <= 127 characters + "schema": { + "name": "my_schema", + "database": {"name": "my_database"}, + }, + "query": "select 1 from my_favoriate_table", }, DbtRuntimeError, ), @@ -62,27 +84,62 @@ def test_create_materialized_view(config_dict: dict, exception: Type[Exception]) RedshiftMaterializedViewRelation.from_dict(config_dict) else: my_materialized_view = RedshiftMaterializedViewRelation.from_dict(config_dict) + assert my_materialized_view.name == config_dict.get("name") - assert my_materialized_view.schema_name == config_dict.get("schema").get("name") - assert my_materialized_view.database_name == config_dict.get("schema").get("database").get( - "name" - ) + assert my_materialized_view.schema_name == config_dict.get("schema", {}).get("name") + assert my_materialized_view.database_name == config_dict.get("schema", {}).get( + "database", {} + ).get("name") assert my_materialized_view.query == config_dict.get("query") - if indexes := config_dict.get("indexes"): - parsed = {(index.method, index.unique) for index in my_materialized_view.indexes} - raw = {(index.get("method"), index.get("unique")) for index in indexes} - assert parsed == raw + assert my_materialized_view.backup == config_dict.get("backup", True) + + default_dist = RedshiftDistRelation.from_dict({"diststyle": "even"}) + default_diststyle = default_dist.diststyle + default_distkey = default_dist.distkey + assert my_materialized_view.dist.diststyle == config_dict.get("dist", {}).get( + "diststyle", default_diststyle + ) + assert my_materialized_view.dist.distkey == config_dict.get("dist", {}).get( + "distkey", default_distkey + ) + + default_sort = RedshiftSortRelation.from_dict({}) + default_sortstyle = default_sort.sortstyle + default_sortkey = default_sort.sortkey + assert my_materialized_view.sort.sortstyle == config_dict.get("sort", {}).get( + "sortstyle", default_sortstyle + ) + assert my_materialized_view.sort.sortkey == frozenset( + config_dict.get("sort", {}).get("sortkey", default_sortkey) + ) + + assert my_materialized_view.autorefresh == config_dict.get("autorefresh", False) + assert my_materialized_view.can_be_renamed is False -def test_create_materialized_view_changeset(materialized_view_relation): +@pytest.mark.parametrize( + "changes,is_empty,requires_full_refresh", + [ + ( + {"autorefresh": "f"}, + False, + False, + ), + ({"sort": RedshiftSortRelation.from_dict({"sortkey": "id"})}, False, True), + ({}, True, False), + ], +) +def test_create_materialized_view_changeset( + materialized_view_relation, changes, is_empty, requires_full_refresh +): existing_materialized_view = replace(materialized_view_relation) # pulled from `./dbt_postgres_tests/conftest.py` # TODO update with sort/dist/autorefresh/backup stuff - target_materialized_view = replace(existing_materialized_view, indexes=frozenset({})) + target_materialized_view = replace(existing_materialized_view, **changes) changeset = RedshiftMaterializedViewRelationChangeset.from_relations( existing_materialized_view, target_materialized_view ) - assert changeset.is_empty is False - assert changeset.requires_full_refresh is False + assert changeset.is_empty is is_empty + assert changeset.requires_full_refresh is requires_full_refresh diff --git a/tests/unit/relation_tests/model_tests/test_sort.py b/tests/unit/relation_tests/model_tests/test_sort.py new file mode 100644 index 000000000..1aa016815 --- /dev/null +++ b/tests/unit/relation_tests/model_tests/test_sort.py @@ -0,0 +1,30 @@ +from typing import Type + +import pytest +from dbt.exceptions import DbtRuntimeError + +from dbt.adapters.redshift.relation.models import RedshiftSortRelation + + +@pytest.mark.parametrize( + "config_dict,exception", + [ + ({}, None), + ({"sortstyle": "auto", "sortkey": "id"}, DbtRuntimeError), + ({"sortstyle": "compound", "sortkey": ["id"]}, None), + ({"sortstyle": "interleaved", "sortkey": ["id"]}, None), + ({"sortstyle": "auto"}, None), + ({"sortstyle": "compound"}, DbtRuntimeError), + ({"sortstyle": "interleaved"}, DbtRuntimeError), + ({"sortstyle": "compound", "sortkey": ["id", "value"]}, None), + ], +) +def test_create_sort(config_dict: dict, exception: Type[Exception]): + if exception: + with pytest.raises(exception): + RedshiftSortRelation.from_dict(config_dict) + else: + my_sortkey = RedshiftSortRelation.from_dict(config_dict) + default_sortstyle = "compound" if "sortkey" in config_dict else "auto" + assert my_sortkey.sortstyle == config_dict.get("sortstyle", default_sortstyle) + assert my_sortkey.sortkey == frozenset(config_dict.get("sortkey", {})) diff --git a/tests/unit/relation_tests/test_relation_factory.py b/tests/unit/relation_tests/test_relation_factory.py index 3a35152f1..009f14343 100644 --- a/tests/unit/relation_tests/test_relation_factory.py +++ b/tests/unit/relation_tests/test_relation_factory.py @@ -1,58 +1,47 @@ """ Uses the following fixtures in `unit/dbt_redshift_tests/conftest.py`: - `relation_factory` -- `materialized_view_stub` +- `materialized_view_ref` """ +from dbt.adapters.redshift.relation import models -from dbt.contracts.relation import RelationType -from dbt.adapters.postgres.relation import models +def test_make_ref(materialized_view_ref): + assert materialized_view_ref.name == "my_materialized_view" + assert materialized_view_ref.schema_name == "my_schema" + assert materialized_view_ref.database_name == "my_database" + assert materialized_view_ref.type == "materialized_view" + assert materialized_view_ref.can_be_renamed is True -def test_make_stub(materialized_view_stub): - assert materialized_view_stub.name == "my_materialized_view" - assert materialized_view_stub.schema_name == "my_schema" - assert materialized_view_stub.database_name == "my_database" - assert materialized_view_stub.type == "materialized_view" - assert materialized_view_stub.can_be_renamed is True +def test_make_backup_ref(relation_factory, materialized_view_ref): + backup_ref = relation_factory.make_backup_ref(materialized_view_ref) + assert backup_ref.name == '"my_materialized_view__dbt_backup"' -def test_make_backup_stub(relation_factory, materialized_view_stub): - backup_stub = relation_factory.make_backup_stub(materialized_view_stub) - assert backup_stub.name == '"my_materialized_view__dbt_backup"' - - -def test_make_intermediate(relation_factory, materialized_view_stub): - intermediate_relation = relation_factory.make_intermediate(materialized_view_stub) +def test_make_intermediate(relation_factory, materialized_view_ref): + intermediate_relation = relation_factory.make_intermediate(materialized_view_ref) assert intermediate_relation.name == '"my_materialized_view__dbt_tmp"' -def test_make_from_describe_relation_results( - relation_factory, materialized_view_describe_relation_results -): - materialized_view = relation_factory.make_from_describe_relation_results( - materialized_view_describe_relation_results, RelationType.MaterializedView - ) - - assert materialized_view.name == "my_materialized_view" - assert materialized_view.schema_name == "my_schema" - assert materialized_view.database_name == "my_database" - assert materialized_view.query == "select 42 from meaning_of_life" - - index_1 = models.RedshiftIndexRelation( - column_names=frozenset({"id", "value"}), - method=models.RedshiftIndexMethod.hash, - unique=False, +def test_make_from_describe_relation_results(relation_factory, materialized_view_relation): + assert materialized_view_relation.name == "my_materialized_view" + assert materialized_view_relation.schema_name == "my_schema" + assert materialized_view_relation.database_name == "my_database" + assert materialized_view_relation.query == "select 4 as id, 2 as other_id from meaning_of_life" + sort = models.RedshiftSortRelation( + sortstyle=models.RedshiftSortStyle.compound, + sortkey=frozenset({"other_id"}), render=models.RedshiftRenderPolicy, ) - index_2 = models.RedshiftIndexRelation( - column_names=frozenset({"id"}), - method=models.RedshiftIndexMethod.btree, - unique=True, + assert materialized_view_relation.sort == sort + dist = models.RedshiftDistRelation( + diststyle=models.RedshiftDistStyle.key, + distkey='"id"', render=models.RedshiftRenderPolicy, ) - assert index_1 in materialized_view.indexes - assert index_2 in materialized_view.indexes + assert materialized_view_relation.dist == dist + assert materialized_view_relation.autorefresh is True def test_make_from_model_node(relation_factory, materialized_view_model_node): @@ -62,18 +51,16 @@ def test_make_from_model_node(relation_factory, materialized_view_model_node): assert materialized_view.schema_name == "my_schema" assert materialized_view.database_name == "my_database" assert materialized_view.query == "select 42 from meaning_of_life" - - index_1 = models.RedshiftIndexRelation( - column_names=frozenset({"id", "value"}), - method=models.RedshiftIndexMethod.hash, - unique=False, + sort = models.RedshiftSortRelation( + sortstyle=models.RedshiftSortStyle.compound, + sortkey=frozenset({"other_id"}), render=models.RedshiftRenderPolicy, ) - index_2 = models.RedshiftIndexRelation( - column_names=frozenset({"id"}), - method=models.RedshiftIndexMethod.btree, - unique=True, + assert materialized_view.sort == sort + dist = models.RedshiftDistRelation( + diststyle=models.RedshiftDistStyle.key, + distkey="id", render=models.RedshiftRenderPolicy, ) - assert index_1 in materialized_view.indexes - assert index_2 in materialized_view.indexes + assert materialized_view.dist == dist + assert materialized_view.autorefresh is True From 541e9bd27a1facad1285913795cb5b3e48f4f94d Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 12 Jul 2023 03:00:36 -0400 Subject: [PATCH 07/12] downstream results from moving functionality in dbt-core from `MaterializedViewRelation` up into `Relation` --- dbt/adapters/redshift/relation/models/materialized_view.py | 6 +++--- dbt/include/redshift/macros/relations/materialized_view.sql | 2 +- tests/unit/conftest.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dbt/adapters/redshift/relation/models/materialized_view.py b/dbt/adapters/redshift/relation/models/materialized_view.py index f0345ff40..a11882628 100644 --- a/dbt/adapters/redshift/relation/models/materialized_view.py +++ b/dbt/adapters/redshift/relation/models/materialized_view.py @@ -167,7 +167,7 @@ def parse_describe_relation_results( ) config_dict = super().parse_describe_relation_results(describe_relation_results) - materialized_view: agate.Row = describe_relation_results["materialized_view"].rows[0] + materialized_view: agate.Row = describe_relation_results["relation"].rows[0] config_dict.update( { "autorefresh": materialized_view.get("autorefresh"), @@ -194,10 +194,10 @@ def parse_describe_relation_results( def _combine_describe_relation_results_tables( cls, describe_relation_results: Dict[str, agate.Table] ) -> Dict[str, agate.Table]: - materialized_view_table: agate.Table = describe_relation_results["materialized_view"] + materialized_view_table: agate.Table = describe_relation_results["relation"] query_table: agate.Table = describe_relation_results["query"] combined_table: agate.Table = materialized_view_table.join(query_table, full_outer=True) - return {"materialized_view": combined_table} + return {"relation": combined_table} @classmethod def _parse_query(cls, query: str) -> str: diff --git a/dbt/include/redshift/macros/relations/materialized_view.sql b/dbt/include/redshift/macros/relations/materialized_view.sql index 987772155..e46643a89 100644 --- a/dbt/include/redshift/macros/relations/materialized_view.sql +++ b/dbt/include/redshift/macros/relations/materialized_view.sql @@ -94,7 +94,7 @@ {%- endset %} {% set _query = run_query(_query_sql) %} - {% do return({'materialized_view': _materialized_view, 'query': _query}) %} + {% do return({'relation': _materialized_view, 'query': _query}) %} {% endmacro %} diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 6b0f8a487..2a3157b9f 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -86,7 +86,7 @@ def materialized_view_describe_relation_results(): ] ) - return {"materialized_view": materialized_view_agate, "query": query_agate} + return {"relation": materialized_view_agate, "query": query_agate} @pytest.fixture From 78eb9eff6942b1629df3c86b5f22eb57298b3a51 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 12 Jul 2023 04:27:25 -0400 Subject: [PATCH 08/12] downstream results from removing unnecessary abstraction of `MaterializedViewRelation` --- .../relation/models/materialized_view.py | 25 +++++++++++++------ .../model_tests/test_materialized_view.py | 9 +------ 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/dbt/adapters/redshift/relation/models/materialized_view.py b/dbt/adapters/redshift/relation/models/materialized_view.py index a11882628..a282c38a4 100644 --- a/dbt/adapters/redshift/relation/models/materialized_view.py +++ b/dbt/adapters/redshift/relation/models/materialized_view.py @@ -1,17 +1,17 @@ from copy import deepcopy from dataclasses import dataclass -from typing import Dict, Optional, Set +from typing import Dict, Optional, Set, Union import agate from dbt.adapters.relation.models import ( - MaterializedViewRelation, - MaterializedViewRelationChangeset, Relation, RelationChange, + RelationChangeset, RelationChangeAction, ) from dbt.adapters.validation import ValidationMixin, ValidationRule from dbt.contracts.graph.nodes import ModelNode +from dbt.contracts.relation import RelationType from dbt.exceptions import DbtRuntimeError from dbt.adapters.redshift.relation.models.dist import ( @@ -31,7 +31,7 @@ @dataclass(frozen=True, eq=True, unsafe_hash=True) -class RedshiftMaterializedViewRelation(MaterializedViewRelation, ValidationMixin): +class RedshiftMaterializedViewRelation(Relation, ValidationMixin): """ This config follow the specs found here: https://docs.aws.amazon.com/redshift/latest/dg/materialized-view-create-sql-command.html @@ -66,9 +66,10 @@ class RedshiftMaterializedViewRelation(MaterializedViewRelation, ValidationMixin autorefresh: Optional[bool] = False # configuration - render = RedshiftRenderPolicy - SchemaParser = RedshiftSchemaRelation # type: ignore + type = RelationType.MaterializedView can_be_renamed = False + SchemaParser = RedshiftSchemaRelation # type: ignore + render = RedshiftRenderPolicy @property def validation_rules(self) -> Set[ValidationRule]: @@ -246,7 +247,7 @@ def requires_full_refresh(self) -> bool: @dataclass -class RedshiftMaterializedViewRelationChangeset(MaterializedViewRelationChangeset): +class RedshiftMaterializedViewRelationChangeset(RelationChangeset): backup: Optional[RedshiftBackupRelationChange] = None dist: Optional[RedshiftDistRelationChange] = None sort: Optional[RedshiftSortRelationChange] = None @@ -264,7 +265,15 @@ def parse_relations(cls, existing_relation: Relation, target_relation: Relation) f" new: {target_relation}\n" ) - config_dict = super().parse_relations(existing_relation, target_relation) + config_dict: Dict[ + str, + Union[ + RedshiftAutoRefreshRelationChange, + RedshiftBackupRelationChange, + RedshiftDistRelationChange, + RedshiftSortRelationChange, + ], + ] = {} if target_relation.autorefresh != existing_relation.autorefresh: config_dict.update( diff --git a/tests/unit/relation_tests/model_tests/test_materialized_view.py b/tests/unit/relation_tests/model_tests/test_materialized_view.py index fd2e5c5d8..e09509949 100644 --- a/tests/unit/relation_tests/model_tests/test_materialized_view.py +++ b/tests/unit/relation_tests/model_tests/test_materialized_view.py @@ -120,11 +120,7 @@ def test_create_materialized_view(config_dict: dict, exception: Type[Exception]) @pytest.mark.parametrize( "changes,is_empty,requires_full_refresh", [ - ( - {"autorefresh": "f"}, - False, - False, - ), + ({"autorefresh": "f"}, False, False), ({"sort": RedshiftSortRelation.from_dict({"sortkey": "id"})}, False, True), ({}, True, False), ], @@ -133,9 +129,6 @@ def test_create_materialized_view_changeset( materialized_view_relation, changes, is_empty, requires_full_refresh ): existing_materialized_view = replace(materialized_view_relation) - - # pulled from `./dbt_postgres_tests/conftest.py` - # TODO update with sort/dist/autorefresh/backup stuff target_materialized_view = replace(existing_materialized_view, **changes) changeset = RedshiftMaterializedViewRelationChangeset.from_relations( From 721849a5a73c817f94cffd007fc4b0e33c32bc07 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 12 Jul 2023 16:16:15 -0400 Subject: [PATCH 09/12] fixed expected results of two tests --- tests/unit/conftest.py | 6 ++++-- .../unit/relation_tests/test_relation_factory.py | 15 ++++++++++----- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 2a3157b9f..034791c85 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -27,7 +27,10 @@ def relation_factory(): relation_changesets={ RelationType.MaterializedView: models.RedshiftMaterializedViewRelationChangeset, }, - relation_can_be_renamed={RelationType.MaterializedView}, + relation_can_be_renamed={ + RelationType.Table, + RelationType.View, + }, render_policy=models.RedshiftRenderPolicy, ) @@ -64,7 +67,6 @@ def view_ref(relation_factory): @pytest.fixture def materialized_view_describe_relation_results(): - # TODO separate query and add in sort/dist info materialized_view_agate = agate.Table.from_object( [ { diff --git a/tests/unit/relation_tests/test_relation_factory.py b/tests/unit/relation_tests/test_relation_factory.py index 009f14343..c9a7bd666 100644 --- a/tests/unit/relation_tests/test_relation_factory.py +++ b/tests/unit/relation_tests/test_relation_factory.py @@ -3,6 +3,9 @@ - `relation_factory` - `materialized_view_ref` """ +import pytest +from dbt.exceptions import DbtRuntimeError + from dbt.adapters.redshift.relation import models @@ -11,17 +14,19 @@ def test_make_ref(materialized_view_ref): assert materialized_view_ref.schema_name == "my_schema" assert materialized_view_ref.database_name == "my_database" assert materialized_view_ref.type == "materialized_view" - assert materialized_view_ref.can_be_renamed is True + assert materialized_view_ref.can_be_renamed is False def test_make_backup_ref(relation_factory, materialized_view_ref): - backup_ref = relation_factory.make_backup_ref(materialized_view_ref) - assert backup_ref.name == '"my_materialized_view__dbt_backup"' + # materialized views cannot be renamed in redshift + with pytest.raises(DbtRuntimeError): + relation_factory.make_backup_ref(materialized_view_ref) def test_make_intermediate(relation_factory, materialized_view_ref): - intermediate_relation = relation_factory.make_intermediate(materialized_view_ref) - assert intermediate_relation.name == '"my_materialized_view__dbt_tmp"' + # materialized views cannot be renamed in redshift + with pytest.raises(DbtRuntimeError): + relation_factory.make_intermediate(materialized_view_ref) def test_make_from_describe_relation_results(relation_factory, materialized_view_relation): From 369388db7b2033f6c0bc6fbd8ce61fde148bdb93 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 13 Jul 2023 00:13:32 -0400 Subject: [PATCH 10/12] propagated feedback from core regarding naming, typing, and moving from runtime config to compiled node --- dbt/adapters/redshift/impl.py | 14 ++- .../redshift/relation/models/database.py | 4 +- dbt/adapters/redshift/relation/models/dist.py | 16 +-- .../relation/models/materialized_view.py | 28 +++-- .../redshift/relation/models/schema.py | 4 +- dbt/adapters/redshift/relation/models/sort.py | 18 +-- .../materialized_view_tests/conftest.py | 20 +++- tests/unit/conftest.py | 111 +++++++----------- .../test_materialization.py | 18 +-- .../test_materialization_factory.py | 6 +- .../relation_tests/test_relation_factory.py | 2 +- 11 files changed, 115 insertions(+), 126 deletions(-) diff --git a/dbt/adapters/redshift/impl.py b/dbt/adapters/redshift/impl.py index 507085931..b224177cc 100644 --- a/dbt/adapters/redshift/impl.py +++ b/dbt/adapters/redshift/impl.py @@ -14,7 +14,11 @@ import dbt.exceptions from dbt.adapters.redshift import RedshiftConnectionManager, RedshiftRelation -from dbt.adapters.redshift.relation import models as relation_models +from dbt.adapters.redshift.relation.models import ( + RedshiftMaterializedViewRelation, + RedshiftMaterializedViewRelationChangeset, + RedshiftRenderPolicy, +) logger = AdapterLogger("Redshift") @@ -49,19 +53,19 @@ class RedshiftAdapter(SQLAdapter): } @property - def relation_factory(self): + def relation_factory(self) -> RelationFactory: return RelationFactory( relation_models={ - RelationType.MaterializedView: relation_models.RedshiftMaterializedViewRelation, + RelationType.MaterializedView: RedshiftMaterializedViewRelation, }, relation_changesets={ - RelationType.MaterializedView: relation_models.RedshiftMaterializedViewRelationChangeset, + RelationType.MaterializedView: RedshiftMaterializedViewRelationChangeset, }, relation_can_be_renamed={ RelationType.Table, RelationType.View, }, - render_policy=relation_models.RedshiftRenderPolicy, + render_policy=RedshiftRenderPolicy, ) @classmethod diff --git a/dbt/adapters/redshift/relation/models/database.py b/dbt/adapters/redshift/relation/models/database.py index 84f756549..54da20f6b 100644 --- a/dbt/adapters/redshift/relation/models/database.py +++ b/dbt/adapters/redshift/relation/models/database.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import Set +from typing import Any, Dict, Set from dbt.adapters.relation.models import DatabaseRelation from dbt.adapters.validation import ValidationMixin, ValidationRule @@ -36,7 +36,7 @@ def validation_rules(self) -> Set[ValidationRule]: } @classmethod - def from_dict(cls, config_dict) -> "RedshiftDatabaseRelation": + def from_dict(cls, config_dict: Dict[str, Any]) -> "RedshiftDatabaseRelation": database = super().from_dict(config_dict) assert isinstance(database, RedshiftDatabaseRelation) return database diff --git a/dbt/adapters/redshift/relation/models/dist.py b/dbt/adapters/redshift/relation/models/dist.py index 0f85f6004..1860fb1bc 100644 --- a/dbt/adapters/redshift/relation/models/dist.py +++ b/dbt/adapters/redshift/relation/models/dist.py @@ -1,6 +1,6 @@ from copy import deepcopy from dataclasses import dataclass -from typing import Optional, Set +from typing import Any, Dict, Optional, Set import agate from dbt.adapters.relation.models import ( @@ -9,7 +9,7 @@ RelationComponent, ) from dbt.adapters.validation import ValidationMixin, ValidationRule -from dbt.contracts.graph.nodes import ModelNode +from dbt.contracts.graph.nodes import ParsedNode from dbt.dataclass_schema import StrEnum from dbt.exceptions import DbtRuntimeError @@ -70,7 +70,7 @@ def validation_rules(self) -> Set[ValidationRule]: } @classmethod - def from_dict(cls, config_dict) -> "RedshiftDistRelation": + def from_dict(cls, config_dict: Dict[str, Any]) -> "RedshiftDistRelation": # don't alter the incoming config kwargs_dict = deepcopy(config_dict) @@ -82,12 +82,12 @@ def from_dict(cls, config_dict) -> "RedshiftDistRelation": return dist @classmethod - def parse_model_node(cls, model_node: ModelNode) -> dict: + def parse_node(cls, node: ParsedNode) -> Dict[str, Any]: """ Translate ModelNode objects from the user-provided config into a standard dictionary. Args: - model_node: the description of the distkey and diststyle from the user in this format: + node: the description of the distkey and diststyle from the user in this format: { "dist": any("auto", "even", "all") or "" @@ -95,7 +95,7 @@ def parse_model_node(cls, model_node: ModelNode) -> dict: Returns: a standard dictionary describing this `RedshiftDistConfig` instance """ - dist = model_node.config.extra.get("dist", "") + dist = node.config.extra.get("dist", "") diststyle = dist.lower() @@ -115,7 +115,9 @@ def parse_model_node(cls, model_node: ModelNode) -> dict: return config @classmethod - def parse_describe_relation_results(cls, describe_relation_results: agate.Row) -> dict: + def parse_describe_relation_results( + cls, describe_relation_results: agate.Row + ) -> Dict[str, Any]: """ Translate agate objects from the database into a standard dictionary. diff --git a/dbt/adapters/redshift/relation/models/materialized_view.py b/dbt/adapters/redshift/relation/models/materialized_view.py index a282c38a4..8c58904ec 100644 --- a/dbt/adapters/redshift/relation/models/materialized_view.py +++ b/dbt/adapters/redshift/relation/models/materialized_view.py @@ -1,6 +1,6 @@ from copy import deepcopy from dataclasses import dataclass -from typing import Dict, Optional, Set, Union +from typing import Any, Dict, Optional, Set, Union import agate from dbt.adapters.relation.models import ( @@ -10,7 +10,7 @@ RelationChangeAction, ) from dbt.adapters.validation import ValidationMixin, ValidationRule -from dbt.contracts.graph.nodes import ModelNode +from dbt.contracts.graph.nodes import CompiledNode from dbt.contracts.relation import RelationType from dbt.exceptions import DbtRuntimeError @@ -97,7 +97,7 @@ def validation_rules(self) -> Set[ValidationRule]: } @classmethod - def from_dict(cls, config_dict) -> "RedshiftMaterializedViewRelation": + def from_dict(cls, config_dict: Dict[str, Any]) -> "RedshiftMaterializedViewRelation": # don't alter the incoming config kwargs_dict = deepcopy(config_dict) @@ -113,28 +113,28 @@ def from_dict(cls, config_dict) -> "RedshiftMaterializedViewRelation": return materialized_view @classmethod - def parse_model_node(cls, model_node: ModelNode) -> dict: - config_dict = super().parse_model_node(model_node) + def parse_node(cls, node: CompiledNode) -> Dict[str, Any]: # type: ignore + config_dict = super().parse_node(node) config_dict.update( { - "backup": model_node.config.extra.get("backup"), - "autorefresh": model_node.config.extra.get("autorefresh"), + "backup": node.config.extra.get("backup"), + "autorefresh": node.config.extra.get("autorefresh"), } ) - if model_node.config.get("dist"): - config_dict.update({"dist": RedshiftDistRelation.parse_model_node(model_node)}) + if node.config.get("dist"): + config_dict.update({"dist": RedshiftDistRelation.parse_node(node)}) - if model_node.config.get("sort"): - config_dict.update({"sort": RedshiftSortRelation.parse_model_node(model_node)}) + if node.config.get("sort"): + config_dict.update({"sort": RedshiftSortRelation.parse_node(node)}) return config_dict @classmethod def parse_describe_relation_results( cls, describe_relation_results: Dict[str, agate.Table] - ) -> dict: + ) -> Dict[str, Any]: """ Translate agate objects from the database into a standard dictionary. @@ -254,7 +254,9 @@ class RedshiftMaterializedViewRelationChangeset(RelationChangeset): autorefresh: Optional[RedshiftAutoRefreshRelationChange] = None @classmethod - def parse_relations(cls, existing_relation: Relation, target_relation: Relation) -> dict: + def parse_relations( + cls, existing_relation: Relation, target_relation: Relation + ) -> Dict[str, Any]: try: assert isinstance(existing_relation, RedshiftMaterializedViewRelation) assert isinstance(target_relation, RedshiftMaterializedViewRelation) diff --git a/dbt/adapters/redshift/relation/models/schema.py b/dbt/adapters/redshift/relation/models/schema.py index 4bb252b0b..cda55b72c 100644 --- a/dbt/adapters/redshift/relation/models/schema.py +++ b/dbt/adapters/redshift/relation/models/schema.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import Set +from typing import Any, Dict, Set from dbt.adapters.relation.models import SchemaRelation from dbt.adapters.validation import ValidationMixin, ValidationRule @@ -28,7 +28,7 @@ class RedshiftSchemaRelation(SchemaRelation, ValidationMixin): DatabaseParser = RedshiftDatabaseRelation # type: ignore @classmethod - def from_dict(cls, config_dict) -> "RedshiftSchemaRelation": + def from_dict(cls, config_dict: Dict[str, Any]) -> "RedshiftSchemaRelation": schema = super().from_dict(config_dict) assert isinstance(schema, RedshiftSchemaRelation) return schema diff --git a/dbt/adapters/redshift/relation/models/sort.py b/dbt/adapters/redshift/relation/models/sort.py index 26939d4fd..dc2016575 100644 --- a/dbt/adapters/redshift/relation/models/sort.py +++ b/dbt/adapters/redshift/relation/models/sort.py @@ -1,6 +1,6 @@ from copy import deepcopy from dataclasses import dataclass, field -from typing import Optional, FrozenSet, Set +from typing import Any, Dict, Optional, FrozenSet, Set import agate from dbt.adapters.relation.models import ( @@ -9,7 +9,7 @@ RelationComponent, ) from dbt.adapters.validation import ValidationMixin, ValidationRule -from dbt.contracts.graph.nodes import ModelNode +from dbt.contracts.graph.nodes import ParsedNode from dbt.dataclass_schema import StrEnum from dbt.exceptions import DbtRuntimeError @@ -102,7 +102,7 @@ def validation_rules(self) -> Set[ValidationRule]: } @classmethod - def from_dict(cls, config_dict) -> "RedshiftSortRelation": + def from_dict(cls, config_dict: Dict[str, Any]) -> "RedshiftSortRelation": # don't alter the incoming config kwargs_dict = deepcopy(config_dict) @@ -117,12 +117,12 @@ def from_dict(cls, config_dict) -> "RedshiftSortRelation": return sort @classmethod - def parse_model_node(cls, model_node: ModelNode) -> dict: + def parse_node(cls, node: ParsedNode) -> Dict[str, Any]: """ Translate ModelNode objects from the user-provided config into a standard dictionary. Args: - model_node: the description of the sortkey and sortstyle from the user in this format: + node: the description of the sortkey and sortstyle from the user in this format: { "sort_key": "" or [""] or ["",...] @@ -133,10 +133,10 @@ def parse_model_node(cls, model_node: ModelNode) -> dict: """ config_dict = {} - if sortstyle := model_node.config.extra.get("sort_type"): + if sortstyle := node.config.extra.get("sort_type"): config_dict.update({"sortstyle": sortstyle.lower()}) - if sortkey := model_node.config.extra.get("sort"): + if sortkey := node.config.extra.get("sort"): # we allow users to specify the `sort_key` as a string if it's a single column if isinstance(sortkey, str): sortkey = [sortkey] @@ -146,7 +146,9 @@ def parse_model_node(cls, model_node: ModelNode) -> dict: return config_dict @classmethod - def parse_describe_relation_results(cls, describe_relation_results: agate.Row) -> dict: + def parse_describe_relation_results( + cls, describe_relation_results: agate.Row + ) -> Dict[str, Any]: """ Translate agate objects from the database into a standard dictionary. diff --git a/tests/functional/adapter/materialized_view_tests/conftest.py b/tests/functional/adapter/materialized_view_tests/conftest.py index 4883ee20f..617431d89 100644 --- a/tests/functional/adapter/materialized_view_tests/conftest.py +++ b/tests/functional/adapter/materialized_view_tests/conftest.py @@ -1,20 +1,30 @@ import pytest from dbt.adapters.relation.models import RelationRef -from dbt.adapters.relation.factory import RelationFactory +from dbt.adapters.relation import RelationFactory from dbt.contracts.relation import RelationType -from dbt.adapters.redshift.relation import models as relation_models +from dbt.adapters.redshift.relation.models import ( + RedshiftMaterializedViewRelation, + RedshiftMaterializedViewRelationChangeset, + RedshiftRenderPolicy, +) @pytest.fixture(scope="class") def relation_factory(): return RelationFactory( relation_models={ - RelationType.MaterializedView: relation_models.RedshiftMaterializedViewRelation, + RelationType.MaterializedView: RedshiftMaterializedViewRelation, }, - relation_can_be_renamed={RelationType.Table, RelationType.View}, - render_policy=relation_models.RedshiftRenderPolicy, + relation_changesets={ + RelationType.MaterializedView: RedshiftMaterializedViewRelationChangeset, + }, + relation_can_be_renamed={ + RelationType.Table, + RelationType.View, + }, + render_policy=RedshiftRenderPolicy, ) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 034791c85..d69dff8da 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -1,37 +1,38 @@ -from dataclasses import dataclass - import agate import pytest -from dbt.adapters.materialization.factory import MaterializationFactory +from dbt.adapters.materialization import MaterializationFactory from dbt.adapters.materialization.models import ( MaterializationType, MaterializedViewMaterialization, ) -from dbt.adapters.relation.factory import RelationFactory +from dbt.adapters.relation import RelationFactory from dbt.contracts.files import FileHash -from dbt.contracts.graph.model_config import OnConfigurationChangeOption from dbt.contracts.graph.nodes import DependsOn, ModelNode, NodeConfig from dbt.contracts.relation import RelationType from dbt.node_types import NodeType -from dbt.adapters.redshift.relation import models +from dbt.adapters.redshift.relation.models import ( + RedshiftMaterializedViewRelation, + RedshiftMaterializedViewRelationChangeset, + RedshiftRenderPolicy, +) @pytest.fixture def relation_factory(): return RelationFactory( relation_models={ - RelationType.MaterializedView: models.RedshiftMaterializedViewRelation, + RelationType.MaterializedView: RedshiftMaterializedViewRelation, }, relation_changesets={ - RelationType.MaterializedView: models.RedshiftMaterializedViewRelationChangeset, + RelationType.MaterializedView: RedshiftMaterializedViewRelationChangeset, }, relation_can_be_renamed={ RelationType.Table, RelationType.View, }, - render_policy=models.RedshiftRenderPolicy, + render_policy=RedshiftRenderPolicy, ) @@ -65,32 +66,6 @@ def view_ref(relation_factory): ) -@pytest.fixture -def materialized_view_describe_relation_results(): - materialized_view_agate = agate.Table.from_object( - [ - { - "name": "my_materialized_view", - "schema_name": "my_schema", - "database_name": "my_database", - "dist": """KEY("id")""", - "sortkey": "other_id", - "autorefresh": "t", - } - ] - ) - - query_agate = agate.Table.from_object( - [ - { - "query": "select 4 as id, 2 as other_id from meaning_of_life", - } - ] - ) - - return {"relation": materialized_view_agate, "query": query_agate} - - @pytest.fixture def materialized_view_model_node(): return ModelNode( @@ -122,6 +97,7 @@ def materialized_view_model_node(): "sort": ["other_id"], "sort_type": "compound", "backup": False, + "on_configuration_change": "continue", } ), tags=[], @@ -136,36 +112,35 @@ def materialized_view_model_node(): @pytest.fixture -def materialized_view_relation(relation_factory, materialized_view_describe_relation_results): - return relation_factory.make_from_describe_relation_results( - materialized_view_describe_relation_results, RelationType.MaterializedView +def materialized_view_describe_relation_results(): + materialized_view_agate = agate.Table.from_object( + [ + { + "name": "my_materialized_view", + "schema_name": "my_schema", + "database_name": "my_database", + "dist": """KEY("id")""", + "sortkey": "other_id", + "autorefresh": "t", + } + ] ) + query_agate = agate.Table.from_object( + [ + { + "query": "select 4 as id, 2 as other_id from meaning_of_life", + } + ] + ) + + return {"relation": materialized_view_agate, "query": query_agate} + @pytest.fixture -def materialized_view_runtime_config(materialized_view_model_node): - """ - This is not actually a `RuntimeConfigObject`. It's an object that has attribution that looks like - a boiled down version of a RuntimeConfigObject. - - TODO: replace this with an actual `RuntimeConfigObject` - """ - - @dataclass() - class RuntimeConfigObject: - model: ModelNode - full_refresh: bool - grants: dict - on_configuration_change: OnConfigurationChangeOption - - def get(self, attribute: str, default=None): - return getattr(self, attribute, default) - - return RuntimeConfigObject( - model=materialized_view_model_node, - full_refresh=False, - grants={}, - on_configuration_change=OnConfigurationChangeOption.Continue, +def materialized_view_relation(relation_factory, materialized_view_describe_relation_results): + return relation_factory.make_from_describe_relation_results( + materialized_view_describe_relation_results, RelationType.MaterializedView ) @@ -176,16 +151,16 @@ def get(self, attribute: str, default=None): def test_relation_factory(relation_factory): assert ( - relation_factory._get_parser(RelationType.MaterializedView) - == models.RedshiftMaterializedViewRelation + relation_factory._get_relation_class(RelationType.MaterializedView) + == RedshiftMaterializedViewRelation ) def test_materialization_factory(materialization_factory): - redshift_parser = materialization_factory.relation_factory._get_parser( + redshift_parser = materialization_factory.relation_factory._get_relation_class( RelationType.MaterializedView ) - assert redshift_parser == models.RedshiftMaterializedViewRelation + assert redshift_parser == RedshiftMaterializedViewRelation def test_materialized_view_ref(materialized_view_ref): @@ -196,12 +171,6 @@ def test_materialized_view_model_node(materialized_view_model_node): assert materialized_view_model_node.name == "my_materialized_view" -def test_materialized_view_runtime_config(materialized_view_runtime_config): - assert materialized_view_runtime_config.get("full_refresh", False) is False - assert materialized_view_runtime_config.get("on_configuration_change", "apply") == "continue" - assert materialized_view_runtime_config.model.name == "my_materialized_view" - - def test_materialized_view_relation(materialized_view_relation): assert materialized_view_relation.type == RelationType.MaterializedView assert materialized_view_relation.name == "my_materialized_view" diff --git a/tests/unit/materialization_tests/test_materialization.py b/tests/unit/materialization_tests/test_materialization.py index 8f2a01441..0dedce491 100644 --- a/tests/unit/materialization_tests/test_materialization.py +++ b/tests/unit/materialization_tests/test_materialization.py @@ -8,31 +8,31 @@ from dbt.adapters.redshift.relation import models -def test_materialized_view_create(materialized_view_runtime_config, relation_factory): - materialization = MaterializedViewMaterialization.from_runtime_config( - materialized_view_runtime_config, relation_factory +def test_materialized_view_create(materialized_view_model_node, relation_factory): + materialization = MaterializedViewMaterialization.from_node( + materialized_view_model_node, relation_factory ) assert materialization.build_strategy == MaterializationBuildStrategy.Create assert materialization.should_revoke_grants is False -def test_materialized_view_replace(materialized_view_runtime_config, relation_factory, view_ref): - materialization = MaterializedViewMaterialization.from_runtime_config( - materialized_view_runtime_config, relation_factory, view_ref +def test_materialized_view_replace(materialized_view_model_node, relation_factory, view_ref): + materialization = MaterializedViewMaterialization.from_node( + materialized_view_model_node, relation_factory, view_ref ) assert materialization.build_strategy == MaterializationBuildStrategy.Replace assert materialization.should_revoke_grants is True def test_materialized_view_alter( - materialized_view_runtime_config, relation_factory, materialized_view_relation + materialized_view_model_node, relation_factory, materialized_view_relation ): altered_materialized_view = replace( materialized_view_relation, sort=models.RedshiftSortRelation.from_dict({}) ) - materialization = MaterializedViewMaterialization.from_runtime_config( - materialized_view_runtime_config, relation_factory, altered_materialized_view + materialization = MaterializedViewMaterialization.from_node( + materialized_view_model_node, relation_factory, altered_materialized_view ) assert materialization.build_strategy == MaterializationBuildStrategy.Alter assert materialization.should_revoke_grants is True diff --git a/tests/unit/materialization_tests/test_materialization_factory.py b/tests/unit/materialization_tests/test_materialization_factory.py index f2bb0a93a..dad958f1b 100644 --- a/tests/unit/materialization_tests/test_materialization_factory.py +++ b/tests/unit/materialization_tests/test_materialization_factory.py @@ -4,9 +4,9 @@ from dbt.adapters.redshift.relation import models -def test_make_from_runtime_config(materialization_factory, materialized_view_runtime_config): - materialization = materialization_factory.make_from_runtime_config( - runtime_config=materialized_view_runtime_config, +def test_make_from_runtime_config(materialization_factory, materialized_view_model_node): + materialization = materialization_factory.make_from_node( + node=materialized_view_model_node, materialization_type=MaterializationType.MaterializedView, existing_relation_ref=None, ) diff --git a/tests/unit/relation_tests/test_relation_factory.py b/tests/unit/relation_tests/test_relation_factory.py index c9a7bd666..f08b7a422 100644 --- a/tests/unit/relation_tests/test_relation_factory.py +++ b/tests/unit/relation_tests/test_relation_factory.py @@ -50,7 +50,7 @@ def test_make_from_describe_relation_results(relation_factory, materialized_view def test_make_from_model_node(relation_factory, materialized_view_model_node): - materialized_view = relation_factory.make_from_model_node(materialized_view_model_node) + materialized_view = relation_factory.make_from_node(materialized_view_model_node) assert materialized_view.name == "my_materialized_view" assert materialized_view.schema_name == "my_schema" From c4a094f3fb59a37998482bba4fa49cf5337d1732 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 13 Jul 2023 22:49:34 -0400 Subject: [PATCH 11/12] minor updates from recent changes on core --- dbt/adapters/redshift/relation/models/database.py | 8 +------- dbt/adapters/redshift/relation/models/dist.py | 9 ++++++--- .../redshift/relation/models/materialized_view.py | 14 +++++++++----- dbt/adapters/redshift/relation/models/schema.py | 10 ++-------- dbt/adapters/redshift/relation/models/sort.py | 9 ++++++--- .../adapter/materialized_view_tests/conftest.py | 1 + .../test_materialized_views.py | 1 + tests/unit/conftest.py | 5 +++-- .../test_materialization_factory.py | 1 - 9 files changed, 29 insertions(+), 29 deletions(-) diff --git a/dbt/adapters/redshift/relation/models/database.py b/dbt/adapters/redshift/relation/models/database.py index 54da20f6b..a270553ef 100644 --- a/dbt/adapters/redshift/relation/models/database.py +++ b/dbt/adapters/redshift/relation/models/database.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import Any, Dict, Set +from typing import Set from dbt.adapters.relation.models import DatabaseRelation from dbt.adapters.validation import ValidationMixin, ValidationRule @@ -34,9 +34,3 @@ def validation_rules(self) -> Set[ValidationRule]: ), ) } - - @classmethod - def from_dict(cls, config_dict: Dict[str, Any]) -> "RedshiftDatabaseRelation": - database = super().from_dict(config_dict) - assert isinstance(database, RedshiftDatabaseRelation) - return database diff --git a/dbt/adapters/redshift/relation/models/dist.py b/dbt/adapters/redshift/relation/models/dist.py index 1860fb1bc..b8f692a3a 100644 --- a/dbt/adapters/redshift/relation/models/dist.py +++ b/dbt/adapters/redshift/relation/models/dist.py @@ -2,8 +2,8 @@ from dataclasses import dataclass from typing import Any, Dict, Optional, Set -import agate from dbt.adapters.relation.models import ( + DescribeRelationResults, RelationChange, RelationChangeAction, RelationComponent, @@ -116,7 +116,7 @@ def parse_node(cls, node: ParsedNode) -> Dict[str, Any]: @classmethod def parse_describe_relation_results( - cls, describe_relation_results: agate.Row + cls, describe_relation_results: DescribeRelationResults ) -> Dict[str, Any]: """ Translate agate objects from the database into a standard dictionary. @@ -130,7 +130,10 @@ def parse_describe_relation_results( Returns: a standard dictionary describing this `RedshiftDistConfig` instance """ - dist: str = describe_relation_results.get("dist") + describe_relation_results_entry = cls._parse_single_record_from_describe_relation_results( + describe_relation_results, "dist" + ) + dist: str = describe_relation_results_entry.get("dist") try: # covers `AUTO`, `ALL`, `EVEN`, `KEY`, '', diff --git a/dbt/adapters/redshift/relation/models/materialized_view.py b/dbt/adapters/redshift/relation/models/materialized_view.py index 8c58904ec..6b2e6c15d 100644 --- a/dbt/adapters/redshift/relation/models/materialized_view.py +++ b/dbt/adapters/redshift/relation/models/materialized_view.py @@ -4,13 +4,14 @@ import agate from dbt.adapters.relation.models import ( + DescribeRelationResults, Relation, RelationChange, RelationChangeset, RelationChangeAction, ) from dbt.adapters.validation import ValidationMixin, ValidationRule -from dbt.contracts.graph.nodes import CompiledNode +from dbt.contracts.graph.nodes import ParsedNode from dbt.contracts.relation import RelationType from dbt.exceptions import DbtRuntimeError @@ -68,7 +69,7 @@ class RedshiftMaterializedViewRelation(Relation, ValidationMixin): # configuration type = RelationType.MaterializedView can_be_renamed = False - SchemaParser = RedshiftSchemaRelation # type: ignore + SchemaParser = RedshiftSchemaRelation render = RedshiftRenderPolicy @property @@ -113,7 +114,7 @@ def from_dict(cls, config_dict: Dict[str, Any]) -> "RedshiftMaterializedViewRela return materialized_view @classmethod - def parse_node(cls, node: CompiledNode) -> Dict[str, Any]: # type: ignore + def parse_node(cls, node: ParsedNode) -> Dict[str, Any]: # type: ignore config_dict = super().parse_node(node) config_dict.update( @@ -133,7 +134,7 @@ def parse_node(cls, node: CompiledNode) -> Dict[str, Any]: # type: ignore @classmethod def parse_describe_relation_results( - cls, describe_relation_results: Dict[str, agate.Table] + cls, describe_relation_results: DescribeRelationResults ) -> Dict[str, Any]: """ Translate agate objects from the database into a standard dictionary. @@ -163,12 +164,15 @@ def parse_describe_relation_results( """ # merge these because the base class assumes `query` is on the same record as `name`, `schema_name` and # `database_name` + assert isinstance(describe_relation_results, Dict) describe_relation_results = cls._combine_describe_relation_results_tables( describe_relation_results ) config_dict = super().parse_describe_relation_results(describe_relation_results) - materialized_view: agate.Row = describe_relation_results["relation"].rows[0] + materialized_view = cls._parse_single_record_from_describe_relation_results( + describe_relation_results, "relation" + ) config_dict.update( { "autorefresh": materialized_view.get("autorefresh"), diff --git a/dbt/adapters/redshift/relation/models/schema.py b/dbt/adapters/redshift/relation/models/schema.py index cda55b72c..2d1eec464 100644 --- a/dbt/adapters/redshift/relation/models/schema.py +++ b/dbt/adapters/redshift/relation/models/schema.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import Any, Dict, Set +from typing import Set from dbt.adapters.relation.models import SchemaRelation from dbt.adapters.validation import ValidationMixin, ValidationRule @@ -25,13 +25,7 @@ class RedshiftSchemaRelation(SchemaRelation, ValidationMixin): # configuration render = RedshiftRenderPolicy - DatabaseParser = RedshiftDatabaseRelation # type: ignore - - @classmethod - def from_dict(cls, config_dict: Dict[str, Any]) -> "RedshiftSchemaRelation": - schema = super().from_dict(config_dict) - assert isinstance(schema, RedshiftSchemaRelation) - return schema + DatabaseParser = RedshiftDatabaseRelation @property def validation_rules(self) -> Set[ValidationRule]: diff --git a/dbt/adapters/redshift/relation/models/sort.py b/dbt/adapters/redshift/relation/models/sort.py index dc2016575..441905041 100644 --- a/dbt/adapters/redshift/relation/models/sort.py +++ b/dbt/adapters/redshift/relation/models/sort.py @@ -2,8 +2,8 @@ from dataclasses import dataclass, field from typing import Any, Dict, Optional, FrozenSet, Set -import agate from dbt.adapters.relation.models import ( + DescribeRelationResults, RelationChange, RelationChangeAction, RelationComponent, @@ -147,7 +147,7 @@ def parse_node(cls, node: ParsedNode) -> Dict[str, Any]: @classmethod def parse_describe_relation_results( - cls, describe_relation_results: agate.Row + cls, describe_relation_results: DescribeRelationResults ) -> Dict[str, Any]: """ Translate agate objects from the database into a standard dictionary. @@ -167,7 +167,10 @@ def parse_describe_relation_results( Returns: a standard dictionary describing this `RedshiftSortConfig` instance """ - if sortkey := describe_relation_results.get("sortkey"): + describe_relation_results_entry = cls._parse_single_record_from_describe_relation_results( + describe_relation_results, "sort" + ) + if sortkey := describe_relation_results_entry.get("sortkey"): return {"sortkey": {sortkey}} return {} diff --git a/tests/functional/adapter/materialized_view_tests/conftest.py b/tests/functional/adapter/materialized_view_tests/conftest.py index 617431d89..066b51a34 100644 --- a/tests/functional/adapter/materialized_view_tests/conftest.py +++ b/tests/functional/adapter/materialized_view_tests/conftest.py @@ -14,6 +14,7 @@ @pytest.fixture(scope="class") def relation_factory(): return RelationFactory( + relation_types=RelationType, relation_models={ RelationType.MaterializedView: RedshiftMaterializedViewRelation, }, diff --git a/tests/functional/adapter/materialized_view_tests/test_materialized_views.py b/tests/functional/adapter/materialized_view_tests/test_materialized_views.py index 65c4d9471..2162ed4f5 100644 --- a/tests/functional/adapter/materialized_view_tests/test_materialized_views.py +++ b/tests/functional/adapter/materialized_view_tests/test_materialized_views.py @@ -251,6 +251,7 @@ def project_config_update(self): def test_autorefresh_change_is_not_applied(self, project, my_materialized_view): assert query_autorefresh(project, my_materialized_view) is False swap_autorefresh(project, my_materialized_view) + # note the expected fail, versus the pass with the `continue` setting _, logs = run_dbt_and_capture( ["--debug", "run", "--models", my_materialized_view.name], expect_pass=False ) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index d69dff8da..a3a86cc35 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -22,6 +22,7 @@ @pytest.fixture def relation_factory(): return RelationFactory( + relation_types=RelationType, relation_models={ RelationType.MaterializedView: RedshiftMaterializedViewRelation, }, @@ -151,13 +152,13 @@ def materialized_view_relation(relation_factory, materialized_view_describe_rela def test_relation_factory(relation_factory): assert ( - relation_factory._get_relation_class(RelationType.MaterializedView) + relation_factory._get_relation_model(RelationType.MaterializedView) == RedshiftMaterializedViewRelation ) def test_materialization_factory(materialization_factory): - redshift_parser = materialization_factory.relation_factory._get_relation_class( + redshift_parser = materialization_factory.relation_factory._get_relation_model( RelationType.MaterializedView ) assert redshift_parser == RedshiftMaterializedViewRelation diff --git a/tests/unit/materialization_tests/test_materialization_factory.py b/tests/unit/materialization_tests/test_materialization_factory.py index dad958f1b..06be7d4a2 100644 --- a/tests/unit/materialization_tests/test_materialization_factory.py +++ b/tests/unit/materialization_tests/test_materialization_factory.py @@ -7,7 +7,6 @@ def test_make_from_runtime_config(materialization_factory, materialized_view_model_node): materialization = materialization_factory.make_from_node( node=materialized_view_model_node, - materialization_type=MaterializationType.MaterializedView, existing_relation_ref=None, ) assert materialization.type == MaterializationType.MaterializedView From fe964b784ef06784c1aa74dda3456a132a2d11ef Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 13 Jul 2023 23:10:44 -0400 Subject: [PATCH 12/12] removed unnecessary query parser --- .../relation/models/materialized_view.py | 34 +------------------ 1 file changed, 1 insertion(+), 33 deletions(-) diff --git a/dbt/adapters/redshift/relation/models/materialized_view.py b/dbt/adapters/redshift/relation/models/materialized_view.py index 6b2e6c15d..61ed904b5 100644 --- a/dbt/adapters/redshift/relation/models/materialized_view.py +++ b/dbt/adapters/redshift/relation/models/materialized_view.py @@ -173,12 +173,7 @@ def parse_describe_relation_results( materialized_view = cls._parse_single_record_from_describe_relation_results( describe_relation_results, "relation" ) - config_dict.update( - { - "autorefresh": materialized_view.get("autorefresh"), - "query": cls._parse_query(materialized_view.get("query")), - } - ) + config_dict.update({"autorefresh": materialized_view.get("autorefresh")}) # the default for materialized views differs from the default for diststyle in general # only set it if we got a value @@ -204,33 +199,6 @@ def _combine_describe_relation_results_tables( combined_table: agate.Table = materialized_view_table.join(query_table, full_outer=True) return {"relation": combined_table} - @classmethod - def _parse_query(cls, query: str) -> str: - """ - Get the select statement from the materialized view definition in Redshift. - - Args: - query: the `create materialized view` statement from `pg_views`, for example: - - create materialized view my_materialized_view - backup yes - diststyle even - sortkey (id) - auto refresh no - as ( - select * from my_base_table - ); - - Returns: the `select ...` statement, for example: - - select * from my_base_table - - """ - return query - # open_paren = query.find("as (") - # close_paren = query.find(");") - # return query[open_paren:close_paren].strip() - @dataclass(frozen=True, eq=True, unsafe_hash=True) class RedshiftAutoRefreshRelationChange(RelationChange):