From fd02a3446dcc6eaf3a90d67a0a91f459d6f96157 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 27 Jun 2023 00:48:08 -0400 Subject: [PATCH 01/28] added updates from snowflake to core, pushed down to postgres, draft --- core/dbt/adapters/base/relation.py | 70 ++++++- core/dbt/adapters/relation_configs/README.md | 26 ++- .../dbt/adapters/relation_configs/__init__.py | 9 +- core/dbt/adapters/relation_configs/base.py | 161 ++++++++++++++ .../adapters/relation_configs/config_base.py | 44 ---- .../{config_validation.py => validation.py} | 13 ++ .../macros/adapters/drop_relation.sql | 14 +- .../models/materialized_view/alter.sql | 9 + .../alter_materialized_view.sql | 30 --- .../models/materialized_view/create.sql | 9 + .../create_materialized_view.sql | 9 - .../models/materialized_view/describe.sql | 9 + .../models/materialized_view/drop.sql | 20 ++ ...aterialized_view_configuration_changes.sql | 23 -- .../materialized_view/materialized_view.sql | 70 ++++--- .../models/materialized_view/refresh.sql | 9 + .../refresh_materialized_view.sql | 9 - .../models/materialized_view/rename.sql | 9 + .../models/materialized_view/replace.sql | 9 + .../replace_materialized_view.sql | 9 - .../dbt/adapters/postgres/relation.py | 73 +++---- .../postgres/relation_configs/__init__.py | 10 +- .../postgres/relation_configs/base.py | 15 ++ .../postgres/relation_configs/constants.py | 1 - .../postgres/relation_configs/database.py | 64 ++++++ .../postgres/relation_configs/index.py | 21 +- .../relation_configs/materialized_view.py | 196 +++++++++++++++--- .../postgres/relation_configs/policies.py | 20 ++ .../postgres/relation_configs/schema.py | 80 +++++++ .../dbt/include/postgres/macros/adapters.sql | 4 +- .../materializations/materialized_view.sql | 177 +++++++++++----- 31 files changed, 914 insertions(+), 308 deletions(-) create mode 100644 core/dbt/adapters/relation_configs/base.py delete mode 100644 core/dbt/adapters/relation_configs/config_base.py rename core/dbt/adapters/relation_configs/{config_validation.py => validation.py} (79%) create mode 100644 core/dbt/include/global_project/macros/materializations/models/materialized_view/alter.sql delete mode 100644 core/dbt/include/global_project/macros/materializations/models/materialized_view/alter_materialized_view.sql create mode 100644 core/dbt/include/global_project/macros/materializations/models/materialized_view/create.sql delete mode 100644 core/dbt/include/global_project/macros/materializations/models/materialized_view/create_materialized_view.sql create mode 100644 core/dbt/include/global_project/macros/materializations/models/materialized_view/describe.sql create mode 100644 core/dbt/include/global_project/macros/materializations/models/materialized_view/drop.sql delete mode 100644 core/dbt/include/global_project/macros/materializations/models/materialized_view/get_materialized_view_configuration_changes.sql create mode 100644 core/dbt/include/global_project/macros/materializations/models/materialized_view/refresh.sql delete mode 100644 core/dbt/include/global_project/macros/materializations/models/materialized_view/refresh_materialized_view.sql create mode 100644 core/dbt/include/global_project/macros/materializations/models/materialized_view/rename.sql create mode 100644 core/dbt/include/global_project/macros/materializations/models/materialized_view/replace.sql delete mode 100644 core/dbt/include/global_project/macros/materializations/models/materialized_view/replace_materialized_view.sql create mode 100644 plugins/postgres/dbt/adapters/postgres/relation_configs/base.py delete mode 100644 plugins/postgres/dbt/adapters/postgres/relation_configs/constants.py create mode 100644 plugins/postgres/dbt/adapters/postgres/relation_configs/database.py create mode 100644 plugins/postgres/dbt/adapters/postgres/relation_configs/policies.py create mode 100644 plugins/postgres/dbt/adapters/postgres/relation_configs/schema.py diff --git a/core/dbt/adapters/base/relation.py b/core/dbt/adapters/base/relation.py index ae4e585d524..daa27df993c 100644 --- a/core/dbt/adapters/base/relation.py +++ b/core/dbt/adapters/base/relation.py @@ -2,7 +2,18 @@ from dataclasses import dataclass, field from typing import Optional, TypeVar, Any, Type, Dict, Iterator, Tuple, Set -from dbt.contracts.graph.nodes import SourceDefinition, ManifestNode, ResultNode, ParsedNode +from dbt.adapters.relation_configs import ( + RelationConfigBase, + DescribeRelationResults, +) +from dbt.context.providers import RuntimeConfigObject +from dbt.contracts.graph.nodes import ( + SourceDefinition, + ManifestNode, + ResultNode, + ParsedNode, + ModelNode, +) from dbt.contracts.relation import ( RelationType, ComponentName, @@ -35,6 +46,8 @@ class BaseRelation(FakeAPIObject, Hashable): include_policy: Policy = field(default_factory=lambda: Policy()) quote_policy: Policy = field(default_factory=lambda: Policy()) dbt_created: bool = False + # registers RelationConfigBases to RelationTypes + relation_configs: Dict[RelationType, RelationConfigBase] = field(default_factory=dict) def _is_exactish_match(self, field: ComponentName, value: str) -> bool: if self.dbt_created and self.quote_policy.get_part(field) is False: @@ -286,6 +299,61 @@ def create( ) return cls.from_dict(kwargs) + @classmethod + def from_runtime_config(cls, runtime_config: RuntimeConfigObject) -> RelationConfigBase: + """ + Produce a validated relation config from the config available in the global jinja context. + + The intention is to remove validation from the jinja context and put it in python. This method gets + called in a jinja template and it's results are used in the jinja template. For an example, please + refer to `dbt/include/global_project/macros/materializations/models/materialized_view/materialization.sql`. + In this file, the relation config is retrieved right away, to ensure that the config is validated before + any sql is executed against the database. + + Args: + runtime_config: the `config` RuntimeConfigObject instance that's in the global jinja context + + Returns: a validated adapter-specific, relation_type-specific RelationConfigBase instance + """ + model_node: ModelNode = runtime_config.model + relation_type = cls.get_relation_type()(model_node.config.materialized) + + if relation_config := cls.relation_configs.get(relation_type): + relation = relation_config.from_model_node(model_node) + else: + raise dbt.exceptions.DbtRuntimeError( + f"from_runtime_config() is not supported for the provided relation type: {relation_type}" + ) + + return relation + + @classmethod + def from_describe_relation_results( + cls, describe_relation_results: DescribeRelationResults, relation_type: RelationType + ) -> RelationConfigBase: + """ + Produce a validated relation config from a series of "describe "-type queries. + + The intention is to remove validation from the jinja context and put it in python. This method gets + called in a jinja template and it's results are used in the jinja template. For an example, please + refer to `dbt/include/global_project/macros/materializations/models/materialized_view/materialization.sql`. + + Args: + describe_relation_results: the results of one or more queries run against the database + to describe this relation + relation_type: the type of relation associated with the relation results + + Returns: a validated adapter-specific, relation_type-specific RelationConfigBase instance + """ + if relation_config := cls.relation_configs.get(relation_type): + relation = relation_config.from_describe_relation_results(describe_relation_results) + else: + raise dbt.exceptions.DbtRuntimeError( + f"from_relation_results() is not supported for the provided relation type: {relation_type}" + ) + + return relation + def __repr__(self) -> str: return "<{} {}>".format(self.__class__.__name__, self.render()) diff --git a/core/dbt/adapters/relation_configs/README.md b/core/dbt/adapters/relation_configs/README.md index 6be3bc59d12..c61f04b3d05 100644 --- a/core/dbt/adapters/relation_configs/README.md +++ b/core/dbt/adapters/relation_configs/README.md @@ -1,21 +1,25 @@ -# RelationConfig +# Relation Configs This package serves as an initial abstraction for managing the inspection of existing relations and determining changes on those relations. It arose from the materialized view work and is currently only supporting -materialized views for Postgres and Redshift as well as dynamic tables for Snowflake. There are three main +materialized views for Postgres, Redshift, and BigQuery as well as dynamic tables for Snowflake. There are three main classes in this package. ## RelationConfigBase -This is a very small class that only has a `from_dict()` method and a default `NotImplementedError()`. At some -point this could be replaced by a more robust framework, like `mashumaro` or `pydantic`. +This is a very small class that only has a handful of methods. It's effectively a parser, but with two sources. +The objective is to provide a stopping point between dbt-specific config and database-specific config for two +primary reasons: + +- apply validation rules in the parlance of the database +- articular what changes are monitored, and how those changes are applied in the database + +At some point this could be theoretically be replaced by a more robust framework, like `mashumaro` or `pydantic`. ## RelationConfigChange -This class inherits from `RelationConfigBase` ; however, this can be thought of as a separate class. The subclassing -merely points to the idea that both classes would likely inherit from the same class in a `mashumaro` or -`pydantic` implementation. This class is much more restricted in attribution. It should really only -ever need an `action` and a `context`. This can be though of as being analogous to a web request. You need to -know what you're doing (`action`: 'create' = GET, 'drop' = DELETE, etc.) and the information (`context`) needed -to make the change. In our scenarios, the context tends to be an instance of `RelationConfigBase` corresponding -to the new state. +A `RelationConfigChange` can be thought of as being analogous to a web request on a `RelationConfigBase`. +You need to know what you're doing (`action`: 'create' = GET, 'drop' = DELETE, etc.) +and the information (`context`) needed to make the change. +In our scenarios, the context tends to be an instance of `RelationConfigBase` corresponding to the new state +or a single value if the change is simple. ## RelationConfigValidationMixin This mixin provides optional validation mechanics that can be applied to either `RelationConfigBase` or diff --git a/core/dbt/adapters/relation_configs/__init__.py b/core/dbt/adapters/relation_configs/__init__.py index b8c73447a68..7e850d8509a 100644 --- a/core/dbt/adapters/relation_configs/__init__.py +++ b/core/dbt/adapters/relation_configs/__init__.py @@ -1,12 +1,13 @@ -from dbt.adapters.relation_configs.config_base import ( # noqa: F401 +from dbt.adapters.relation_configs.base import ( # noqa: F401 RelationConfigBase, - RelationResults, + DescribeRelationResults, ) -from dbt.adapters.relation_configs.config_change import ( # noqa: F401 +from dbt.adapters.relation_configs.change import ( # noqa: F401 RelationConfigChangeAction, RelationConfigChange, + RelationConfigChangeset, ) -from dbt.adapters.relation_configs.config_validation import ( # noqa: F401 +from dbt.adapters.relation_configs.validation import ( # noqa: F401 RelationConfigValidationMixin, RelationConfigValidationRule, ) diff --git a/core/dbt/adapters/relation_configs/base.py b/core/dbt/adapters/relation_configs/base.py new file mode 100644 index 00000000000..283c97b95d8 --- /dev/null +++ b/core/dbt/adapters/relation_configs/base.py @@ -0,0 +1,161 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Union, Optional, Dict + +import agate +from dbt.utils import filter_null_values + +from dbt.contracts.graph.nodes import ModelNode +from dbt.contracts.relation import Policy, ComponentName, RelationType + + +""" +Relation metadata from the database comes in the form of a collection of one or more `agate.Table`s. In order to +reference the tables, they are added to a dictionary. There can be more than one table because there can be +multiple grains of data for a single object. For example, a materialized view in Postgres has base level information, +like name. But it also can have multiple indexes, which needs to be a separate query. It might look like this: + +{ + "materialized_view": agate.Table( + agate.Row({"table_name": "table_abc", "query": "select * from table_def"}) + ), + "indexes": agate.Table("rows": [ + agate.Row({"name": "index_a", "columns": ["column_a"], "type": "hash", "unique": False}), + agate.Row({"name": "index_b", "columns": ["time_dim_a"], "type": "btree", "unique": False}), + ]), +} + +Generally speaking, "primary" `RelationConfigBase` instances (e.g. materialized view) will be described with +an `agate.Table` and "dependent" `RelationConfigBase` instances (e.g. index) will be described with an `agate.Row`. +This happens simply because the primary instance is the first step in processing the metadata, but the dependent +instance can be looped when dispatching to it in `parse_describe_relation_results()`. +""" +DescribeRelationResults = Dict[str, Union[agate.Row, agate.Table]] + + +@dataclass(frozen=True) +class RelationConfigBase(ABC): + relation_type: RelationType + include_policy: Policy + quote_policy: Policy + quote_character: str + + @classmethod + def from_dict(cls, kwargs_dict) -> "RelationConfigBase": + """ + This assumes the subclass of `RelationConfigBase` is flat, in the sense that no attribute is + itself another subclass of `RelationConfigBase`. If that's not the case, this should be overriden + to manually manage that complexity. But remember to either call `super().from_dict()` at the end, + or at least use `filter_null_values()` so that defaults get applied properly for the dataclass. + + Args: + kwargs_dict: the dict representation of this instance + + Returns: the `RelationConfigBase` representation associated with the provided dict + """ + return cls(**filter_null_values(kwargs_dict)) # type: ignore + + @classmethod + def from_model_node(cls, model_node: ModelNode) -> "RelationConfigBase": + """ + A wrapper around `cls.parse_model_node()` and `cls.from_dict()` that pipes the results of the first into + the second. This shouldn't really need to be overridden; instead, the component methods should be overridden. + + Args: + model_node: the `model` (`ModelNode`) attribute off of `config` (`RuntimeConfigObject`) in the global + jinja context of a materialization + + Returns: + a validated `RelationConfigBase` instance specific to the adapter and relation type + """ + relation_config = cls.parse_model_node(model_node) + relation = cls.from_dict(relation_config) + return relation + + @classmethod + @abstractmethod + def parse_model_node(cls, model_node: ModelNode) -> dict: + """ + The purpose of this method is to translate the dbt/user generic parlance into the database parlance and + format it for `RelationConfigBase` consumption. + + In many cases this may be a one-to-one mapping; e.g. dbt calls it "schema_name" and the database calls it + "schema_name". This could also be a renaming, calculation, or dispatch to a lower grain object. + + See `dbt/adapters/postgres/relation_configs/materialized_view.py` to see an example implementation. + + Args: + model_node: the `model` (`ModelNode`) attribute off of `config` (`RuntimeConfigObject`) in the global + jinja context of a materialization + + Returns: + a non-validated dictionary version of a `RelationConfigBase` instance specific to the adapter and + relation type + """ + raise NotImplementedError("`parse_model_node()` needs to be implemented for this adapter.") + + @classmethod + def from_describe_relation_results( + cls, describe_relation_results: DescribeRelationResults + ) -> "RelationConfigBase": + """ + A wrapper around `cls.parse_describe_relation_results()` and `cls.from_dict()` that pipes the results of the + first into the second. This shouldn't really need to be overridden; instead, the component methods should + be overridden. + + Args: + describe_relation_results: the results of one or more queries run against the database to gather the + requisite metadata to describe this relation + + Returns: + a validated `RelationConfigBase` instance specific to the adapter and relation type + """ + relation_config = cls.parse_describe_relation_results(describe_relation_results) + relation = cls.from_dict(relation_config) + return relation + + @classmethod + @abstractmethod + def parse_describe_relation_results( + cls, describe_relation_results: DescribeRelationResults + ) -> dict: + """ + The purpose of this method is to format the database parlance for `RelationConfigBase` consumption. + + This tends to be one-to-one except for combining grains of data. For example, a single materialized + view could have multiple indexes which would result in multiple queries to the database to build one + materialized view config object. All of these pieces get knit together here. + + See `dbt/adapters/postgres/relation_configs/materialized_view.py` to see an example implementation. + + Args: + describe_relation_results: the results of one or more queries run against the database to gather the + requisite metadata to describe this relation + + Returns: + a non-validated dictionary version of a `RelationConfigBase` instance specific to the adapter and + relation type + """ + raise NotImplementedError( + "`parse_describe_relation_results()` needs to be implemented for this adapter." + ) + + @classmethod + def _render_part(cls, component: ComponentName, value: str) -> Optional[str]: + """ + Apply the include and quote policy to the value. + + *Note: The quote character is removed and then re-added to support backup and intermediate relation names + + Args: + component: the component of the policy to apply + value: the value to which the policies should be applied + + Returns: + a policy-compliant value + """ + if cls.include_policy.get_part(component): + if cls.quote_policy.get_part(component): + return f"{cls.quote_character}{value.replace(cls.quote_character, '')}{cls.quote_character}" + return value.lower() + return None diff --git a/core/dbt/adapters/relation_configs/config_base.py b/core/dbt/adapters/relation_configs/config_base.py deleted file mode 100644 index 9d0cddb0d21..00000000000 --- a/core/dbt/adapters/relation_configs/config_base.py +++ /dev/null @@ -1,44 +0,0 @@ -from dataclasses import dataclass -from typing import Union, Dict - -import agate -from dbt.utils import filter_null_values - - -""" -This is what relation metadata from the database looks like. It's a dictionary because there will be -multiple grains of data for a single object. For example, a materialized view in Postgres has base level information, -like name. But it also can have multiple indexes, which needs to be a separate query. It might look like this: - -{ - "base": agate.Row({"table_name": "table_abc", "query": "select * from table_def"}) - "indexes": agate.Table("rows": [ - agate.Row({"name": "index_a", "columns": ["column_a"], "type": "hash", "unique": False}), - agate.Row({"name": "index_b", "columns": ["time_dim_a"], "type": "btree", "unique": False}), - ]) -} -""" -RelationResults = Dict[str, Union[agate.Row, agate.Table]] - - -@dataclass(frozen=True) -class RelationConfigBase: - @classmethod - def from_dict(cls, kwargs_dict) -> "RelationConfigBase": - """ - This assumes the subclass of `RelationConfigBase` is flat, in the sense that no attribute is - itself another subclass of `RelationConfigBase`. If that's not the case, this should be overriden - to manually manage that complexity. - - Args: - kwargs_dict: the dict representation of this instance - - Returns: the `RelationConfigBase` representation associated with the provided dict - """ - return cls(**filter_null_values(kwargs_dict)) # type: ignore - - @classmethod - def _not_implemented_error(cls) -> NotImplementedError: - return NotImplementedError( - "This relation type has not been fully configured for this adapter." - ) diff --git a/core/dbt/adapters/relation_configs/config_validation.py b/core/dbt/adapters/relation_configs/validation.py similarity index 79% rename from core/dbt/adapters/relation_configs/config_validation.py rename to core/dbt/adapters/relation_configs/validation.py index 17bf74bf3e7..9164a0e94cb 100644 --- a/core/dbt/adapters/relation_configs/config_validation.py +++ b/core/dbt/adapters/relation_configs/validation.py @@ -6,11 +6,24 @@ @dataclass(frozen=True, eq=True, unsafe_hash=True) class RelationConfigValidationRule: + """ + A validation rule consists of two parts: + - validation_check: the thing that should be True + - validation_error: the error to raise in the event the validation check is False + """ + validation_check: bool validation_error: Optional[DbtRuntimeError] @property def default_error(self): + """ + This is a built-in stock error message. It may suffice in that it will raise an error for you, but + you should likely supply one in the rule that is more descriptive. This is akin to raising `Exception`. + + Returns: + a stock error message + """ return DbtRuntimeError( "There was a validation error in preparing this relation config." "No additional context was provided by this adapter." diff --git a/core/dbt/include/global_project/macros/adapters/drop_relation.sql b/core/dbt/include/global_project/macros/adapters/drop_relation.sql index bd254c78d51..5fc3ec79f70 100644 --- a/core/dbt/include/global_project/macros/adapters/drop_relation.sql +++ b/core/dbt/include/global_project/macros/adapters/drop_relation.sql @@ -35,10 +35,14 @@ {%- endmacro %} -{% macro drop_materialized_view(relation) -%} - {{ return(adapter.dispatch('drop_materialized_view', 'dbt')(relation)) }} +{% macro drop_relation_sql(relation_config) -%} + {{ return(adapter.dispatch('drop_relation_sql', 'dbt')(relation_config)) }} {%- endmacro %} -{% macro default__drop_materialized_view(relation) -%} - drop materialized view if exists {{ relation }} cascade -{%- endmacro %} +{%- macro default__drop_relation_sql(relation_config) -%} + {%- if relation_config.relation_type == adapter.Relation.MaterializedView -%} + {{- drop_materialized_view_sql(relation_config) -}} + {%- else -%} + drop {{ relation_config.relation_type }} if exists {{ relation_config.fully_qualified_path }} cascade + {%- endif -%} +{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/alter.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view/alter.sql new file mode 100644 index 00000000000..f7450314094 --- /dev/null +++ b/core/dbt/include/global_project/macros/materializations/models/materialized_view/alter.sql @@ -0,0 +1,9 @@ +{% macro alter_materialized_view_sql(new_materialized_view, existing_materialized_view) %} + {{- log('Applying ALTER to: ' ~ new_materialized_view.fully_qualified_path) -}} + {{- adapter.dispatch('alter_materialized_view_sql', 'dbt')(new_materialized_view, existing_materialized_view) -}} +{% endmacro %} + + +{% macro default__alter_materialized_view_sql(new_materialized_view, existing_materialized_view) %} + {{ exceptions.raise_compiler_error("Materialized views have not been implemented for this adapter.") }} +{% endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/alter_materialized_view.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view/alter_materialized_view.sql deleted file mode 100644 index b9ccdc2f141..00000000000 --- a/core/dbt/include/global_project/macros/materializations/models/materialized_view/alter_materialized_view.sql +++ /dev/null @@ -1,30 +0,0 @@ -{% macro get_alter_materialized_view_as_sql( - relation, - configuration_changes, - sql, - existing_relation, - backup_relation, - intermediate_relation -) %} - {{- log('Applying ALTER to: ' ~ relation) -}} - {{- adapter.dispatch('get_alter_materialized_view_as_sql', 'dbt')( - relation, - configuration_changes, - sql, - existing_relation, - backup_relation, - intermediate_relation - ) -}} -{% endmacro %} - - -{% macro default__get_alter_materialized_view_as_sql( - relation, - configuration_changes, - sql, - existing_relation, - backup_relation, - intermediate_relation -) %} - {{ exceptions.raise_compiler_error("Materialized views have not been implemented for this adapter.") }} -{% endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/create.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view/create.sql new file mode 100644 index 00000000000..7ac572c3b49 --- /dev/null +++ b/core/dbt/include/global_project/macros/materializations/models/materialized_view/create.sql @@ -0,0 +1,9 @@ +{% macro create_materialized_view_sql(materialized_view, intermediate=False) -%} + {{- log('Applying CREATE to: ' ~ materialized_view.fully_qualified_path) -}} + {{- adapter.dispatch('create_materialized_view_sql', 'dbt')(materialized_view, intermediate) -}} +{%- endmacro %} + + +{% macro default__create_materialized_view_sql(materialized_view, intermediate=False) -%} + {{ exceptions.raise_compiler_error("Materialized views have not been implemented for this adapter.") }} +{% endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/create_materialized_view.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view/create_materialized_view.sql deleted file mode 100644 index 4b2ebeb3aa1..00000000000 --- a/core/dbt/include/global_project/macros/materializations/models/materialized_view/create_materialized_view.sql +++ /dev/null @@ -1,9 +0,0 @@ -{% macro get_create_materialized_view_as_sql(relation, sql) -%} - {{- log('Applying CREATE to: ' ~ relation) -}} - {{- adapter.dispatch('get_create_materialized_view_as_sql', 'dbt')(relation, sql) -}} -{%- endmacro %} - - -{% macro default__get_create_materialized_view_as_sql(relation, sql) -%} - {{ exceptions.raise_compiler_error("Materialized views have not been implemented for this adapter.") }} -{% endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/describe.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view/describe.sql new file mode 100644 index 00000000000..e55f1c409a9 --- /dev/null +++ b/core/dbt/include/global_project/macros/materializations/models/materialized_view/describe.sql @@ -0,0 +1,9 @@ +{% macro describe_materialized_view(materialized_view) %} + {{- log('Applying DESCRIBE to: ' ~ materialized_view.fully_qualified_path) -}} + {{- adapter.dispatch('describe_materialized_view', 'dbt')(materialized_view) -}} +{% endmacro %} + + +{% macro default__describe_materialized_view(materialized_view) %} + {{ exceptions.raise_compiler_error("Materialized views have not been implemented for this adapter.") }} +{% endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/drop.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view/drop.sql new file mode 100644 index 00000000000..325bf224549 --- /dev/null +++ b/core/dbt/include/global_project/macros/materializations/models/materialized_view/drop.sql @@ -0,0 +1,20 @@ +-- This is the new one that gets used with RelationConfigBase instances +{% macro drop_materialized_view_sql(materialized_view) -%} + {{ return(adapter.dispatch('drop_materialized_view_sql', 'dbt')(materialized_view)) }} +{%- endmacro %} + + +{% macro default__drop_materialized_view_sql(materialized_view) -%} + {{ exceptions.raise_compiler_error("Materialized views have not been implemented for this adapter.") }} +{%- endmacro %} + + +-- This is the old one that gets used with BaseRelation instances +{% macro drop_materialized_view(relation) -%} + {{ return(adapter.dispatch('drop_materialized_view', 'dbt')(relation)) }} +{%- endmacro %} + + +{% macro default__drop_materialized_view(relation) -%} + {{ exceptions.raise_compiler_error("Materialized views have not been implemented for this adapter.") }} +{%- endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/get_materialized_view_configuration_changes.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view/get_materialized_view_configuration_changes.sql deleted file mode 100644 index b1639b1631e..00000000000 --- a/core/dbt/include/global_project/macros/materializations/models/materialized_view/get_materialized_view_configuration_changes.sql +++ /dev/null @@ -1,23 +0,0 @@ -{% macro get_materialized_view_configuration_changes(existing_relation, new_config) %} - /* {# - It's recommended that configuration changes be formatted as follows: - {"": [{"action": "", "context": ...}]} - - For example: - { - "indexes": [ - {"action": "drop", "context": "index_abc"}, - {"action": "create", "context": {"columns": ["column_1", "column_2"], "type": "hash", "unique": True}}, - ], - } - - Either way, `get_materialized_view_configuration_changes` needs to align with `get_alter_materialized_view_as_sql`. - #} */ - {{- log('Determining configuration changes on: ' ~ existing_relation) -}} - {%- do return(adapter.dispatch('get_materialized_view_configuration_changes', 'dbt')(existing_relation, new_config)) -%} -{% endmacro %} - - -{% macro default__get_materialized_view_configuration_changes(existing_relation, new_config) %} - {{ exceptions.raise_compiler_error("Materialized views have not been implemented for this adapter.") }} -{% endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/materialized_view.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view/materialized_view.sql index 015f6cb8585..62a58bc6c61 100644 --- a/core/dbt/include/global_project/macros/materializations/models/materialized_view/materialized_view.sql +++ b/core/dbt/include/global_project/macros/materializations/models/materialized_view/materialized_view.sql @@ -1,4 +1,9 @@ {% materialization materialized_view, default %} + + -- Try to create a valid materialized view from the config before doing anything else + {% set new_materialized_view = adapter.Relation.from_runtime_config(config) %} + + -- We still need these because they tie into the existing process (e.g. RelationBase vs. RelationConfigBase) {% set existing_relation = load_cached_relation(this) %} {% set target_relation = this.incorporate(type=this.MaterializedView) %} {% set intermediate_relation = make_intermediate_relation(target_relation) %} @@ -7,12 +12,12 @@ {{ materialized_view_setup(backup_relation, intermediate_relation, pre_hooks) }} - {% set build_sql = materialized_view_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %} + {% set build_sql = materialized_view_build_sql(new_materialized_view, existing_relation, backup_relation, intermediate_relation) %} {% if build_sql == '' %} - {{ materialized_view_execute_no_op(target_relation) }} + {{ materialized_view_execute_no_op(new_materialized_view) }} {% else %} - {{ materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) }} + {{ materialized_view_execute_build_sql(build_sql, new_materialized_view, post_hooks) }} {% endif %} {{ materialized_view_teardown(backup_relation, intermediate_relation, post_hooks) }} @@ -49,37 +54,44 @@ {% endmacro %} -{% macro materialized_view_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %} +{% macro materialized_view_build_sql(new_materialized_view, existing_relation, backup_relation, intermediate_relation) %} {% set full_refresh_mode = should_full_refresh() %} - -- determine the scenario we're in: create, full_refresh, alter, refresh data + -- determine the scenario we're in: create, full_refresh, alter {% if existing_relation is none %} - {% set build_sql = get_create_materialized_view_as_sql(target_relation, sql) %} + {% set build_sql = create_materialized_view_sql(new_materialized_view) %} {% elif full_refresh_mode or not existing_relation.is_materialized_view %} - {% set build_sql = get_replace_materialized_view_as_sql(target_relation, sql, existing_relation, backup_relation, intermediate_relation) %} + {% set build_sql = replace_materialized_view_sql(new_materialized_view, existing_relation, backup_relation, intermediate_relation) %} {% else %} + {% set build_sql = alter_materialized_view_with_on_configuration_option_sql(new_materialized_view) %} + {% endif %} - -- get config options - {% set on_configuration_change = config.get('on_configuration_change') %} - {% set configuration_changes = get_materialized_view_configuration_changes(existing_relation, config) %} + {% do return(build_sql) %} - {% if configuration_changes is none %} - {% set build_sql = refresh_materialized_view(target_relation) %} +{% endmacro %} - {% elif on_configuration_change == 'apply' %} - {% set build_sql = get_alter_materialized_view_as_sql(target_relation, configuration_changes, sql, existing_relation, backup_relation, intermediate_relation) %} - {% elif on_configuration_change == 'continue' %} - {% set build_sql = '' %} - {{ exceptions.warn("Configuration changes were identified and `on_configuration_change` was set to `continue` for `" ~ target_relation ~ "`") }} - {% elif on_configuration_change == 'fail' %} - {{ exceptions.raise_fail_fast_error("Configuration changes were identified and `on_configuration_change` was set to `fail` for `" ~ target_relation ~ "`") }} - {% else %} - -- this only happens if the user provides a value other than `apply`, 'skip', 'fail' - {{ exceptions.raise_compiler_error("Unexpected configuration scenario") }} +{% macro alter_materialized_view_with_on_configuration_option_sql(new_materialized_view) %} - {% endif %} + {% set describe_relation_results = describe_materialized_view(new_materialized_view) %} + {% set existing_materialized_view = adapter.Relation.from_describe_relation_results(describe_relation_results, adapter.Relation.MaterializedView) %} + {% set on_configuration_change = config.get('on_configuration_change') %} + + {% if new_materialized_view == existing_materialized_view %} + {% set build_sql = refresh_materialized_view_sql(new_materialized_view) %} + + {% elif on_configuration_change == 'apply' %} + {% set build_sql = alter_materialized_view_sql(new_materialized_view, existing_materialized_view) %} + {% elif on_configuration_change == 'continue' %} + {% set build_sql = '' %} + {{ exceptions.warn("Configuration changes were identified and `on_configuration_change` was set to `continue` for `" ~ new_materialized_view.fully_qualified_path ~ "`") }} + {% elif on_configuration_change == 'fail' %} + {{ exceptions.raise_fail_fast_error("Configuration changes were identified and `on_configuration_change` was set to `fail` for `" ~ new_materialized_view.fully_qualified_path ~ "`") }} + + {% else %} + -- this only happens if the user provides a value other than `apply`, 'continue', 'fail', which should have already raised an exception + {{ exceptions.raise_compiler_error("Unexpected configuration scenario: `" ~ on_configuration_change ~ "`") }} {% endif %} @@ -88,17 +100,17 @@ {% endmacro %} -{% macro materialized_view_execute_no_op(target_relation) %} +{% macro materialized_view_execute_no_op(new_materialized_view) %} {% do store_raw_result( name="main", - message="skip " ~ target_relation, + message="skip " ~ new_materialized_view.fully_qualified_path, code="skip", rows_affected="-1" ) %} {% endmacro %} -{% macro materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) %} +{% macro materialized_view_execute_build_sql(build_sql, new_materialized_view, post_hooks) %} -- `BEGIN` happens here: {{ run_hooks(pre_hooks, inside_transaction=True) }} @@ -109,10 +121,10 @@ {{ build_sql }} {% endcall %} - {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %} - {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} + {% set should_revoke = should_revoke(new_materialized_view.fully_qualified_path, full_refresh_mode=True) %} + {% do apply_grants(new_materialized_view.fully_qualified_path, grant_config, should_revoke=should_revoke) %} - {% do persist_docs(target_relation, model) %} + {% do persist_docs(new_materialized_view.fully_qualified_path, model) %} {{ run_hooks(post_hooks, inside_transaction=True) }} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/refresh.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view/refresh.sql new file mode 100644 index 00000000000..6f8605dbacc --- /dev/null +++ b/core/dbt/include/global_project/macros/materializations/models/materialized_view/refresh.sql @@ -0,0 +1,9 @@ +{% macro refresh_materialized_view(materialized_view) %} + {{- log('Applying REFRESH to: ' ~ materialized_view.fully_qualified_path) -}} + {{- adapter.dispatch('refresh_materialized_view', 'dbt')(materialized_view) -}} +{% endmacro %} + + +{% macro default__refresh_materialized_view(materialized_view) %} + {{ exceptions.raise_compiler_error("Materialized views have not been implemented for this adapter.") }} +{% endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/refresh_materialized_view.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view/refresh_materialized_view.sql deleted file mode 100644 index 16345138593..00000000000 --- a/core/dbt/include/global_project/macros/materializations/models/materialized_view/refresh_materialized_view.sql +++ /dev/null @@ -1,9 +0,0 @@ -{% macro refresh_materialized_view(relation) %} - {{- log('Applying REFRESH to: ' ~ relation) -}} - {{- adapter.dispatch('refresh_materialized_view', 'dbt')(relation) -}} -{% endmacro %} - - -{% macro default__refresh_materialized_view(relation) %} - {{ exceptions.raise_compiler_error("Materialized views have not been implemented for this adapter.") }} -{% endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/rename.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view/rename.sql new file mode 100644 index 00000000000..53087873d86 --- /dev/null +++ b/core/dbt/include/global_project/macros/materializations/models/materialized_view/rename.sql @@ -0,0 +1,9 @@ +{% macro rename_materialized_view(materialized_view, name, intermediate=False) %} + {{- log('Applying RENAME to: ' ~ materialized_view.fully_qualified_path) -}} + {{- adapter.dispatch('rename_materialized_view', 'dbt')(materialized_view, name, intermediate) -}} +{% endmacro %} + + +{% macro default__rename_materialized_view(materialized_view, name, intermediate=False) %} + {{ exceptions.raise_compiler_error("Materialized views have not been implemented for this adapter.") }} +{% endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/replace.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view/replace.sql new file mode 100644 index 00000000000..2d68f04884c --- /dev/null +++ b/core/dbt/include/global_project/macros/materializations/models/materialized_view/replace.sql @@ -0,0 +1,9 @@ +{% macro replace_materialized_view_sql(new_materialized_view, existing_relation) %} + {{- log('Applying REPLACE to: ' ~ new_materialized_view.fully_qualified_path) -}} + {{- adapter.dispatch('replace_materialized_view_sql', 'dbt')(new_materialized_view, existing_relation) -}} +{% endmacro %} + + +{% macro default__replace_materialized_view_sql(new_materialized_view, existing_relation) %} + {{ exceptions.raise_compiler_error("Materialized views have not been implemented for this adapter.") }} +{% endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/replace_materialized_view.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view/replace_materialized_view.sql deleted file mode 100644 index 43319c5cc1b..00000000000 --- a/core/dbt/include/global_project/macros/materializations/models/materialized_view/replace_materialized_view.sql +++ /dev/null @@ -1,9 +0,0 @@ -{% macro get_replace_materialized_view_as_sql(relation, sql, existing_relation, backup_relation, intermediate_relation) %} - {{- log('Applying REPLACE to: ' ~ relation) -}} - {{- adapter.dispatch('get_replace_materialized_view_as_sql', 'dbt')(relation, sql, existing_relation, backup_relation, intermediate_relation) -}} -{% endmacro %} - - -{% macro default__get_replace_materialized_view_as_sql(relation, sql, existing_relation, backup_relation, intermediate_relation) %} - {{ exceptions.raise_compiler_error("Materialized views have not been implemented for this adapter.") }} -{% endmacro %} diff --git a/plugins/postgres/dbt/adapters/postgres/relation.py b/plugins/postgres/dbt/adapters/postgres/relation.py index 43822efb11f..986114b83d2 100644 --- a/plugins/postgres/dbt/adapters/postgres/relation.py +++ b/plugins/postgres/dbt/adapters/postgres/relation.py @@ -1,25 +1,26 @@ -from dataclasses import dataclass -from typing import Optional, Set, FrozenSet +from dataclasses import dataclass, field +from typing import Set, FrozenSet from dbt.adapters.base.relation import BaseRelation -from dbt.adapters.relation_configs import ( - RelationConfigChangeAction, - RelationResults, -) -from dbt.context.providers import RuntimeConfigObject +from dbt.adapters.relation_configs import RelationConfigChangeAction from dbt.exceptions import DbtRuntimeError from dbt.adapters.postgres.relation_configs import ( PostgresIndexConfig, PostgresIndexConfigChange, PostgresMaterializedViewConfig, - PostgresMaterializedViewConfigChangeCollection, + PostgresMaterializedViewConfigChangeset, MAX_CHARACTERS_IN_IDENTIFIER, + PostgresIncludePolicy, + PostgresQuotePolicy, ) @dataclass(frozen=True, eq=False, repr=False) class PostgresRelation(BaseRelation): + include_policy: PostgresIncludePolicy = field(default_factory=PostgresIncludePolicy) + quote_policy: PostgresQuotePolicy = field(default_factory=PostgresQuotePolicy) + def __post_init__(self): # Check for length of Postgres table/view names. # Check self.type to exclude test relation identifiers @@ -36,33 +37,39 @@ def __post_init__(self): def relation_max_name_length(self): return MAX_CHARACTERS_IN_IDENTIFIER - def get_materialized_view_config_change_collection( - self, relation_results: RelationResults, runtime_config: RuntimeConfigObject - ) -> Optional[PostgresMaterializedViewConfigChangeCollection]: - config_change_collection = PostgresMaterializedViewConfigChangeCollection() + @classmethod + def materialized_view_config_changeset( + cls, + new_materialized_view: PostgresMaterializedViewConfig, + existing_materialized_view: PostgresMaterializedViewConfig, + ) -> PostgresMaterializedViewConfigChangeset: + try: + assert isinstance(new_materialized_view, PostgresMaterializedViewConfig) + assert isinstance(existing_materialized_view, PostgresMaterializedViewConfig) + except AssertionError: + raise DbtRuntimeError( + f"Two materialized view configs were expected, but received:" + f"/n {new_materialized_view}" + f"/n {existing_materialized_view}" + ) - existing_materialized_view = PostgresMaterializedViewConfig.from_relation_results( - relation_results - ) - new_materialized_view = PostgresMaterializedViewConfig.from_model_node( - runtime_config.model - ) + config_changeset = PostgresMaterializedViewConfigChangeset() - config_change_collection.indexes = self._get_index_config_changes( - existing_materialized_view.indexes, new_materialized_view.indexes + config_changeset.indexes = cls.index_config_changeset( + new_materialized_view.indexes, existing_materialized_view.indexes ) - # we return `None` instead of an empty `PostgresMaterializedViewConfigChangeCollection` object - # so that it's easier and more extensible to check in the materialization: - # `core/../materializations/materialized_view.sql` : - # {% if configuration_changes is none %} - if config_change_collection.has_changes: - return config_change_collection + if config_changeset.is_empty and new_materialized_view != existing_materialized_view: + # we need to force a full refresh if we didn't detect any changes but the objects are not the same + config_changeset.force_full_refresh() - def _get_index_config_changes( - self, - existing_indexes: FrozenSet[PostgresIndexConfig], + return config_changeset + + @classmethod + def index_config_changeset( + cls, new_indexes: FrozenSet[PostgresIndexConfig], + existing_indexes: FrozenSet[PostgresIndexConfig], ) -> Set[PostgresIndexConfigChange]: """ Get the index updates that will occur as a result of a new run @@ -77,15 +84,11 @@ def _get_index_config_changes( Returns: a set of index updates in the form {"action": "drop/create", "context": } """ drop_changes = set( - PostgresIndexConfigChange.from_dict( - {"action": RelationConfigChangeAction.drop, "context": index} - ) + PostgresIndexConfigChange(action=RelationConfigChangeAction.drop, context=index) for index in existing_indexes.difference(new_indexes) ) create_changes = set( - PostgresIndexConfigChange.from_dict( - {"action": RelationConfigChangeAction.create, "context": index} - ) + PostgresIndexConfigChange(action=RelationConfigChangeAction.create, context=index) for index in new_indexes.difference(existing_indexes) ) return set().union(drop_changes, create_changes) diff --git a/plugins/postgres/dbt/adapters/postgres/relation_configs/__init__.py b/plugins/postgres/dbt/adapters/postgres/relation_configs/__init__.py index 9fdb942bfa5..ac2def44060 100644 --- a/plugins/postgres/dbt/adapters/postgres/relation_configs/__init__.py +++ b/plugins/postgres/dbt/adapters/postgres/relation_configs/__init__.py @@ -1,11 +1,13 @@ -from dbt.adapters.postgres.relation_configs.constants import ( # noqa: F401 - MAX_CHARACTERS_IN_IDENTIFIER, -) from dbt.adapters.postgres.relation_configs.index import ( # noqa: F401 PostgresIndexConfig, PostgresIndexConfigChange, ) +from dbt.adapters.postgres.relation_configs.policies import ( # noqa: F401 + PostgresIncludePolicy, + PostgresQuotePolicy, + MAX_CHARACTERS_IN_IDENTIFIER, +) from dbt.adapters.postgres.relation_configs.materialized_view import ( # noqa: F401 PostgresMaterializedViewConfig, - PostgresMaterializedViewConfigChangeCollection, + PostgresMaterializedViewConfigChangeset, ) diff --git a/plugins/postgres/dbt/adapters/postgres/relation_configs/base.py b/plugins/postgres/dbt/adapters/postgres/relation_configs/base.py new file mode 100644 index 00000000000..40a966140d2 --- /dev/null +++ b/plugins/postgres/dbt/adapters/postgres/relation_configs/base.py @@ -0,0 +1,15 @@ +from abc import ABC +from dataclasses import dataclass + +from dbt.adapters.relation_configs import RelationConfigBase + +from dbt.adapters.postgres.relation_configs.policies import ( + PostgresIncludePolicy, + PostgresQuotePolicy, +) + + +@dataclass(frozen=True, eq=True, unsafe_hash=True) +class PostgresRelationConfigBase(RelationConfigBase, ABC): + include_policy = PostgresIncludePolicy() + quote_policy = PostgresQuotePolicy() diff --git a/plugins/postgres/dbt/adapters/postgres/relation_configs/constants.py b/plugins/postgres/dbt/adapters/postgres/relation_configs/constants.py deleted file mode 100644 index 9228df23043..00000000000 --- a/plugins/postgres/dbt/adapters/postgres/relation_configs/constants.py +++ /dev/null @@ -1 +0,0 @@ -MAX_CHARACTERS_IN_IDENTIFIER = 63 diff --git a/plugins/postgres/dbt/adapters/postgres/relation_configs/database.py b/plugins/postgres/dbt/adapters/postgres/relation_configs/database.py new file mode 100644 index 00000000000..ff399f15716 --- /dev/null +++ b/plugins/postgres/dbt/adapters/postgres/relation_configs/database.py @@ -0,0 +1,64 @@ +from dataclasses import dataclass +from typing import Set + +import agate +from dbt.adapters.relation_configs import ( + RelationConfigValidationMixin, + RelationConfigValidationRule, +) +from dbt.contracts.graph.nodes import ModelNode +from dbt.contracts.relation import ComponentName +from dbt.exceptions import DbtRuntimeError + +from dbt.adapters.postgres.relation_configs.base import PostgresRelationConfigBase + + +@dataclass(frozen=True, eq=True, unsafe_hash=True) +class PostgresDatabaseConfig(PostgresRelationConfigBase, RelationConfigValidationMixin): + """ + This config follow the specs found here: + https://www.postgresql.org/docs/current/sql-createdatabase.html + + The following parameters are configurable by dbt: + - name: name of the database + """ + + name: str + + @property + def fully_qualified_path(self) -> str: + return self.name + + @property + def validation_rules(self) -> Set[RelationConfigValidationRule]: + return { + RelationConfigValidationRule( + validation_check=len(self.name or "") > 0, + validation_error=DbtRuntimeError( + f"dbt-postgres requires a name for a database, received: {self.name}" + ), + ) + } + + @classmethod + def from_dict(cls, config_dict: dict) -> "PostgresDatabaseConfig": + kwargs_dict = { + "name": cls._render_part(ComponentName.Database, config_dict["name"]), + } + + database: "PostgresDatabaseConfig" = super().from_dict(kwargs_dict) # type: ignore + return database + + @classmethod + def parse_model_node(cls, model_node: ModelNode) -> dict: + config_dict = { + "name": model_node.database, + } + return config_dict + + @classmethod + def parse_describe_relation_results(cls, describe_relation_results: agate.Row) -> dict: + config_dict = { + "name": describe_relation_results["database_name"], + } + return config_dict diff --git a/plugins/postgres/dbt/adapters/postgres/relation_configs/index.py b/plugins/postgres/dbt/adapters/postgres/relation_configs/index.py index 3a072ea4307..1602d0e802b 100644 --- a/plugins/postgres/dbt/adapters/postgres/relation_configs/index.py +++ b/plugins/postgres/dbt/adapters/postgres/relation_configs/index.py @@ -2,16 +2,18 @@ from typing import Set, FrozenSet import agate +from dbt.contracts.relation import ComponentName from dbt.dataclass_schema import StrEnum from dbt.exceptions import DbtRuntimeError from dbt.adapters.relation_configs import ( - RelationConfigBase, RelationConfigValidationMixin, RelationConfigValidationRule, RelationConfigChangeAction, RelationConfigChange, ) +from dbt.adapters.postgres.relation_configs.base import PostgresRelationConfigBase + class PostgresIndexMethod(StrEnum): btree = "btree" @@ -27,7 +29,7 @@ def default(cls) -> "PostgresIndexMethod": @dataclass(frozen=True, eq=True, unsafe_hash=True) -class PostgresIndexConfig(RelationConfigBase, RelationConfigValidationMixin): +class PostgresIndexConfig(PostgresRelationConfigBase, RelationConfigValidationMixin): """ This config fallows the specs found here: https://www.postgresql.org/docs/current/sql-createindex.html @@ -61,11 +63,11 @@ def validation_rules(self) -> Set[RelationConfigValidationRule]: @classmethod def from_dict(cls, config_dict) -> "PostgresIndexConfig": - # TODO: include the QuotePolicy instead of defaulting to lower() kwargs_dict = { "name": config_dict.get("name"), "column_names": frozenset( - column.lower() for column in config_dict.get("column_names", set()) + cls._render_part(ComponentName.Identifier, column) + for column in config_dict.get("column_names", set()) ), "unique": config_dict.get("unique"), "method": config_dict.get("method"), @@ -83,12 +85,12 @@ def parse_model_node(cls, model_node_entry: dict) -> dict: return config_dict @classmethod - def parse_relation_results(cls, relation_results_entry: agate.Row) -> dict: + def parse_describe_relation_results(cls, describe_relation_results: agate.Row) -> dict: config_dict = { - "name": relation_results_entry.get("name"), - "column_names": set(relation_results_entry.get("column_names", "").split(",")), - "unique": relation_results_entry.get("unique"), - "method": relation_results_entry.get("method"), + "name": describe_relation_results.get("name"), + "column_names": set(describe_relation_results.get("column_names", "").split(",")), + "unique": describe_relation_results.get("unique"), + "method": describe_relation_results.get("method"), } return config_dict @@ -129,6 +131,7 @@ class PostgresIndexConfigChange(RelationConfigChange, RelationConfigValidationMi } """ + action: RelationConfigChangeAction context: PostgresIndexConfig @property diff --git a/plugins/postgres/dbt/adapters/postgres/relation_configs/materialized_view.py b/plugins/postgres/dbt/adapters/postgres/relation_configs/materialized_view.py index 15e700e777a..9a6ded32af7 100644 --- a/plugins/postgres/dbt/adapters/postgres/relation_configs/materialized_view.py +++ b/plugins/postgres/dbt/adapters/postgres/relation_configs/materialized_view.py @@ -1,25 +1,28 @@ from dataclasses import dataclass, field -from typing import Set, FrozenSet, List +from typing import Set, FrozenSet, List, Dict import agate from dbt.adapters.relation_configs import ( - RelationConfigBase, - RelationResults, + RelationConfigChangeset, RelationConfigValidationMixin, RelationConfigValidationRule, ) from dbt.contracts.graph.nodes import ModelNode +from dbt.contracts.relation import ComponentName, RelationType from dbt.exceptions import DbtRuntimeError -from dbt.adapters.postgres.relation_configs.constants import MAX_CHARACTERS_IN_IDENTIFIER +from dbt.adapters.postgres.relation_configs.base import PostgresRelationConfigBase +from dbt.adapters.postgres.relation_configs.database import PostgresDatabaseConfig from dbt.adapters.postgres.relation_configs.index import ( PostgresIndexConfig, PostgresIndexConfigChange, ) +from dbt.adapters.postgres.relation_configs.policies import MAX_CHARACTERS_IN_IDENTIFIER +from dbt.adapters.postgres.relation_configs.schema import PostgresSchemaConfig @dataclass(frozen=True, eq=True, unsafe_hash=True) -class PostgresMaterializedViewConfig(RelationConfigBase, RelationConfigValidationMixin): +class PostgresMaterializedViewConfig(PostgresRelationConfigBase, RelationConfigValidationMixin): """ This config follows the specs found here: https://www.postgresql.org/docs/current/sql-creatematerializedview.html @@ -35,13 +38,69 @@ class PostgresMaterializedViewConfig(RelationConfigBase, RelationConfigValidatio - with_data: `True` """ - table_name: str = "" - query: str = "" + table_name: str + schema: PostgresSchemaConfig + query: str indexes: FrozenSet[PostgresIndexConfig] = field(default_factory=frozenset) + relation_type = RelationType.MaterializedView + + @property + def schema_name(self) -> str: + return self.schema.schema_name + + @property + def database(self) -> PostgresDatabaseConfig: + return self.schema.database + + @property + def database_name(self) -> str: + return self.database.name + + @property + def backup_name(self) -> str: + """ + Used for hot-swapping during replacement + + Returns: + a name unique to this materialized view + """ + return self._render_part(ComponentName.Identifier, f"{self.table_name}__dbt_backup") + + @property + def intermediate_name(self) -> str: + """ + Used for hot-swapping during replacement + + Returns: + a name unique to this materialized view + """ + return self._render_part(ComponentName.Identifier, f"{self.table_name}__dbt_tmp") + + @property + def fully_qualified_path(self) -> str: + return self._fully_qualified_path(self.table_name) + + @property + def fully_qualified_path_backup(self) -> str: + return self._fully_qualified_path(self.backup_name) + + @property + def fully_qualified_path_intermediate(self) -> str: + return self._fully_qualified_path(self.intermediate_name) + + def _fully_qualified_path(self, table_name) -> str: + return ".".join( + part for part in [self.schema.fully_qualified_path, table_name] if part is not None + ) @property def validation_rules(self) -> Set[RelationConfigValidationRule]: - # index rules get run by default with the mixin + """ + Validation rules at the materialized view level. All attribute level rules get run as a result of + `RelationConfigValidationMixin`. + + Returns: a set of rules that should evaluate to `True` (i.e. False == validation failure) + """ return { RelationConfigValidationRule( validation_check=self.table_name is None @@ -51,13 +110,33 @@ def validation_rules(self) -> Set[RelationConfigValidationRule]: f"characters: {self.table_name}" ), ), + RelationConfigValidationRule( + validation_check=all({self.database_name, self.schema_name, self.table_name}), + validation_error=DbtRuntimeError( + f"dbt-snowflake requires all three parts of an object's path, received:/n" + f" database: {self.database_name}/n" + f" schema: {self.schema_name}/n" + f" identifier: {self.table_name}/n" + ), + ), } @classmethod def from_dict(cls, config_dict: dict) -> "PostgresMaterializedViewConfig": + """ + Creates an instance of this class given the dict representation + + This is generally used indirectly by calling either `from_model_node()` or `from_relation_results()` + + Args: + config_dict: a dict that aligns with the structure of this class, and it's attribute classes (e.g. indexes) + + Returns: an instance of this class + """ kwargs_dict = { - "table_name": config_dict.get("table_name"), - "query": config_dict.get("query"), + "table_name": cls._render_part(ComponentName.Identifier, config_dict["table_name"]), + "schema": PostgresSchemaConfig.from_dict(config_dict["schema"]), + "query": config_dict["query"], "indexes": frozenset( PostgresIndexConfig.from_dict(index) for index in config_dict.get("indexes", {}) ), @@ -65,49 +144,104 @@ def from_dict(cls, config_dict: dict) -> "PostgresMaterializedViewConfig": materialized_view: "PostgresMaterializedViewConfig" = super().from_dict(kwargs_dict) # type: ignore return materialized_view - @classmethod - def from_model_node(cls, model_node: ModelNode) -> "PostgresMaterializedViewConfig": - materialized_view_config = cls.parse_model_node(model_node) - materialized_view = cls.from_dict(materialized_view_config) - return materialized_view - @classmethod def parse_model_node(cls, model_node: ModelNode) -> dict: + """ + Parse `RuntimeConfigObject.model` into a dict representation of a `PostgresMaterializedViewConfig` instance + + This is generally used indirectly by calling `from_model_node()`, but there are times when the dict + version is more appropriate. + + Args: + model_node: the `model` attribute from `config` in the jinja context + + Example `model_node`: + + ModelNode({ + ..., + "compiled_code": "create materialized view my_materialized_view as select * from my_ref_table;", + ..., + "config" { + ..., + "extra": { + ..., + "indexes": [ + {"columns": ["id"], "type": "hash", "unique": True}, + ], + ..., + }, + ..., + }, + ..., + "identifier": "my_materialized_view", + ..., + }) + + Returns: a dict representation of an instance of this class that can be passed into `from_dict()` + """ indexes: List[dict] = model_node.config.extra.get("indexes", []) config_dict = { "table_name": model_node.identifier, - "query": model_node.compiled_code, + "schema": PostgresSchemaConfig.parse_model_node(model_node), + "query": (model_node.compiled_code or "").strip(), "indexes": [PostgresIndexConfig.parse_model_node(index) for index in indexes], } return config_dict @classmethod - def from_relation_results( - cls, relation_results: RelationResults - ) -> "PostgresMaterializedViewConfig": - materialized_view_config = cls.parse_relation_results(relation_results) - materialized_view = cls.from_dict(materialized_view_config) - return materialized_view + def parse_describe_relation_results( + cls, describe_relation_results: Dict[str, agate.Table] + ) -> dict: + """ + Parse `RelationResults` into a dict representation of a `PostgresMaterializedViewConfig` instance + + This is generally used indirectly by calling `from_relation_results()`, but there are times when the dict + version is more appropriate. + + Args: + describe_relation_results: the results of a set of queries that fully describe an instance of this class + + Example of `relation_results`: + + { + "materialized_view": agate.Table(agate.Row({ + "table_name": "my_materialized_view", + "query": "create materialized view my_materialized_view as select * from my_ref_table;", + })), + "indexes": agate.Table([ + agate.Row({"columns": ["id"], "type": "hash", "unique": True}), + ..., + ], + } + + Returns: a dict representation of an instance of this class that can be passed into `from_dict()` + """ + materialized_view: agate.Row = describe_relation_results["materialized_view"].rows[0] + indexes: agate.Table = describe_relation_results["indexes"] - @classmethod - def parse_relation_results(cls, relation_results: RelationResults) -> dict: - indexes: agate.Table = relation_results.get("indexes", agate.Table(rows={})) config_dict = { + "table_name": materialized_view["table_name"], + "schema": PostgresSchemaConfig.parse_describe_relation_results(materialized_view), + "query": materialized_view["query"].strip(), "indexes": [ - PostgresIndexConfig.parse_relation_results(index) for index in indexes.rows + PostgresIndexConfig.parse_describe_relation_results(index) + for index in indexes.rows ], } return config_dict @dataclass -class PostgresMaterializedViewConfigChangeCollection: +class PostgresMaterializedViewConfigChangeset(RelationConfigChangeset): indexes: Set[PostgresIndexConfigChange] = field(default_factory=set) @property def requires_full_refresh(self) -> bool: - return any(index.requires_full_refresh for index in self.indexes) + return ( + any(index.requires_full_refresh for index in self.indexes) + or super().requires_full_refresh + ) @property - def has_changes(self) -> bool: - return self.indexes != set() + def is_empty(self) -> bool: + return self.indexes == set() and super().is_empty diff --git a/plugins/postgres/dbt/adapters/postgres/relation_configs/policies.py b/plugins/postgres/dbt/adapters/postgres/relation_configs/policies.py new file mode 100644 index 00000000000..2b7b3f78d7c --- /dev/null +++ b/plugins/postgres/dbt/adapters/postgres/relation_configs/policies.py @@ -0,0 +1,20 @@ +from dataclasses import dataclass + +from dbt.adapters.base.relation import Policy + + +QUOTE_CHAR = '"' +MAX_CHARACTERS_IN_IDENTIFIER = 63 + + +class PostgresIncludePolicy(Policy): + database: bool = True + schema: bool = True + identifier: bool = True + + +@dataclass +class PostgresQuotePolicy(Policy): + database: bool = True + schema: bool = True + identifier: bool = True diff --git a/plugins/postgres/dbt/adapters/postgres/relation_configs/schema.py b/plugins/postgres/dbt/adapters/postgres/relation_configs/schema.py new file mode 100644 index 00000000000..349772a745c --- /dev/null +++ b/plugins/postgres/dbt/adapters/postgres/relation_configs/schema.py @@ -0,0 +1,80 @@ +from dataclasses import dataclass +from typing import Set + +import agate +from dbt.adapters.relation_configs import ( + RelationConfigValidationMixin, + RelationConfigValidationRule, +) +from dbt.contracts.graph.nodes import ModelNode +from dbt.contracts.relation import ComponentName +from dbt.exceptions import DbtRuntimeError + +from dbt.adapters.postgres.relation_configs.base import PostgresRelationConfigBase +from dbt.adapters.postgres.relation_configs.database import PostgresDatabaseConfig + + +@dataclass(frozen=True, eq=True, unsafe_hash=True) +class PostgresSchemaConfig(PostgresRelationConfigBase, RelationConfigValidationMixin): + """ + This config follow the specs found here: + https://www.postgresql.org/docs/15/sql-createschema.html + + The following parameters are configurable by dbt: + - name: name of the schema + - database_name: name of the database + """ + + schema_name: str + database: PostgresDatabaseConfig + + @property + def database_name(self) -> str: + return self.database.name + + @property + def fully_qualified_path(self) -> str: + return ".".join( + part + for part in [self.database.fully_qualified_path, self.schema_name] + if part is not None + ) + + @property + def validation_rules(self) -> Set[RelationConfigValidationRule]: + return { + RelationConfigValidationRule( + validation_check=len(self.schema_name or "") > 0, + validation_error=DbtRuntimeError( + f"dbt-postgres requires a name for a schema, received: {self.schema_name}" + ), + ) + } + + @classmethod + def from_dict(cls, config_dict: dict) -> "PostgresSchemaConfig": + kwargs_dict = { + "schema_name": cls._render_part(ComponentName.Schema, config_dict["schema_name"]), + "database": PostgresDatabaseConfig.from_dict(config_dict["database"]), + } + + schema: "PostgresSchemaConfig" = super().from_dict(kwargs_dict) # type: ignore + return schema + + @classmethod + def parse_model_node(cls, model_node: ModelNode) -> dict: + config_dict = { + "schema_name": model_node.schema, + "database": PostgresDatabaseConfig.parse_model_node(model_node), + } + return config_dict + + @classmethod + def parse_describe_relation_results(cls, describe_relation_results: agate.Row) -> dict: + config_dict = { + "schema_name": describe_relation_results["schema_name"], + "database": PostgresDatabaseConfig.parse_describe_relation_results( + describe_relation_results + ), + } + return config_dict diff --git a/plugins/postgres/dbt/include/postgres/macros/adapters.sql b/plugins/postgres/dbt/include/postgres/macros/adapters.sql index c8bdab6eccb..64520ffb945 100644 --- a/plugins/postgres/dbt/include/postgres/macros/adapters.sql +++ b/plugins/postgres/dbt/include/postgres/macros/adapters.sql @@ -239,8 +239,8 @@ join pg_attribute a on a.attrelid = t.oid and a.attnum = ANY(ix.indkey) - where t.relname = '{{ relation.identifier }}' - and n.nspname = '{{ relation.schema }}' + where t.relname ilike '{{ relation.identifier }}' + and n.nspname ilike '{{ relation.schema }}' and t.relkind in ('r', 'm') group by 1, 2, 3 order by 1, 2, 3 diff --git a/plugins/postgres/dbt/include/postgres/macros/materializations/materialized_view.sql b/plugins/postgres/dbt/include/postgres/macros/materializations/materialized_view.sql index 1fc7d864b5b..da0551c8f01 100644 --- a/plugins/postgres/dbt/include/postgres/macros/materializations/materialized_view.sql +++ b/plugins/postgres/dbt/include/postgres/macros/materializations/materialized_view.sql @@ -1,84 +1,157 @@ -{% macro postgres__get_alter_materialized_view_as_sql( - relation, - configuration_changes, - sql, - existing_relation, - backup_relation, - intermediate_relation -) %} - - -- apply a full refresh immediately if needed +{#- /* + This file contains DDL that gets consumed in the default materialized view materialization in `dbt-core`. + These macros could be used elsewhere as they do not care that they are being called by a materialization; + but the original intention was to support the materialization of materialized views. These macros represent + the basic interactions dbt-postgres requires of materialized views in Postgres: + - ALTER + - CREATE + - DESCRIBE + - DROP + - REFRESH + - RENAME + - REPLACE + These macros all take a PostgresMaterializedViewConfig instance as an input. This class can be found in: + `dbt/adapters/postgres/relation_configs/materialized_view.py` + + Used in: + `dbt/include/global_project/macros/materializations/models/materialized_view/materialized_view.sql` + Uses: + `dbt/adapters/postgres/relation.py` + `dbt/adapters/postgres/relation_configs/` +*/ -#} + + +{% macro postgres__alter_materialized_view_sql(new_materialized_view, existing_materialized_view) %} + + {#- /* + We need to get the config changeset to determine if we require a full refresh (happens if any change + in the changeset requires a full refresh or if an unmonitored change was detected) + or if we can get away with altering the dynamic table in place. + */ -#} + {% set config_changeset = adapter.Relation.materialized_view_config_changeset(new_materialized_view, existing_materialized_view) %} + {% if configuration_changes.requires_full_refresh %} - {{ get_replace_materialized_view_as_sql(relation, sql, existing_relation, backup_relation, intermediate_relation) }} + {{ replace_materialized_view_sql(new_materialized_view) }} - -- otherwise apply individual changes as needed {% else %} +-- TODO: make sure this works with old/new + {%- for _index_change in configuration_changes.index_changes -%} + {%- set _index = _index_change.context -%} - {{ postgres__update_indexes_on_materialized_view(relation, configuration_changes.indexes) }} + {%- if _index_change.action == "drop" -%} - {%- endif -%} + {{ postgres__get_drop_index_sql(relation, _index.name) }}; -{% endmacro %} + {%- elif _index_change.action == "create" -%} + {{ postgres__get_create_index_sql(relation, _index.as_node_config) }} -{% macro postgres__get_create_materialized_view_as_sql(relation, sql) %} - create materialized view if not exists {{ relation }} as {{ sql }}; + {%- endif -%} - {% for _index_dict in config.get('indexes', []) -%} - {{- get_create_index_sql(relation, _index_dict) -}} - {%- endfor -%} + {%- endfor -%} -{% endmacro %} - - -{% macro postgres__get_replace_materialized_view_as_sql(relation, sql, existing_relation, backup_relation, intermediate_relation) %} - {{- get_create_materialized_view_as_sql(intermediate_relation, sql) -}} + {%- endif -%} - {% if existing_relation is not none %} - alter materialized view {{ existing_relation }} rename to {{ backup_relation.include(database=False, schema=False) }}; - {% endif %} +{% endmacro %} - alter materialized view {{ intermediate_relation }} rename to {{ relation.include(database=False, schema=False) }}; -{% endmacro %} +{% macro postgres__create_materialized_view_sql(materialized_view, intermediate=False) %} + {%- if intermediate -%} + {%- set materialized_view_path = materialized_view.fully_qualified_path_intermediate -%} + {%- else -%} + {%- set materialized_view_path = materialized_view.fully_qualified_path -%} + {%- endif -%} + create materialized view {{ materialized_view_path }} as + {{ materialized_view.query }} + ; +--TODO: replace this + {% for _index_dict in config.get('indexes', []) -%} + {{- get_create_index_sql(relation, _index_dict) -}} + {%- endfor -%} -{% macro postgres__get_materialized_view_configuration_changes(existing_relation, new_config) %} - {% set _existing_materialized_view = postgres__describe_materialized_view(existing_relation) %} - {% set _configuration_changes = existing_relation.get_materialized_view_config_change_collection(_existing_materialized_view, new_config) %} - {% do return(_configuration_changes) %} {% endmacro %} -{% macro postgres__refresh_materialized_view(relation) %} - refresh materialized view {{ relation }}; +{% macro postgres__describe_materialized_view_sql(materialized_view) %} + + {% set _materialized_view_sql -%} + select + t.tablename, + t.schemaname, + {{ this.database }} as databasename, + v.definition + from pg_tables t + join pg_views v + on v.viewname = t.tablename + and v.schemaname = t.schemaname + where t.tablename ilike '{{ materialized_view.table_name }}' + and t.schemaname ilike '{{ materialized_view.schema_name }}' + {%- endset -%} + {% set _materialized_view = run_query(_materialized_view_sql) %} + + {%- set _indexes_sql -%} + select + i.relname as name, + m.amname as method, + ix.indisunique as "unique", + array_to_string(array_agg(a.attname), ',') as column_names + from pg_index ix + join pg_class i + on i.oid = ix.indexrelid + join pg_am m + on m.oid=i.relam + join pg_class t + on t.oid = ix.indrelid + join pg_namespace n + on n.oid = t.relnamespace + join pg_attribute a + on a.attrelid = t.oid + and a.attnum = ANY(ix.indkey) + where t.relname ilike '{{ materialized_view.table_name }}' + and n.nspname ilike '{{ materialized_view.schema_name }}' + and t.relkind = 'm' + group by 1, 2, 3 + order by 1, 2, 3 + {%- endset -%} + {% set _indexes = run_query(_indexes_sql) %} + + {% do return({'materialized_view': _materialized_view, 'indexes': _indexes}) %} {% endmacro %} -{%- macro postgres__update_indexes_on_materialized_view(relation, index_changes) -%} - {{- log("Applying UPDATE INDEXES to: " ~ relation) -}} +-- This is the new one that gets used with RelationConfigBase instances +{% macro postgres__drop_materialized_view_sql(materialized_view) -%} + drop materialized view if exists {{ materialized_view.fully_qualified_path }} cascade +{%- endmacro %} - {%- for _index_change in index_changes -%} - {%- set _index = _index_change.context -%} - {%- if _index_change.action == "drop" -%} +-- This is the old one that gets used with BaseRelation instances +{% macro postgres__drop_materialized_view(relation) -%} + drop materialized view if exists {{ relation }} cascade +{%- endmacro %} - {{ postgres__get_drop_index_sql(relation, _index.name) }}; - {%- elif _index_change.action == "create" -%} - - {{ postgres__get_create_index_sql(relation, _index.as_node_config) }} +{% macro postgres__refresh_materialized_view_sql(materialized_view) %} + refresh materialized view {{ materialized_view.fully_qualified_path }}; +{% endmacro %} - {%- endif -%} - {%- endfor -%} +{% macro postgres__rename_materialized_view_sql(materialized_view, name, intermediate=False) %} + {%- if intermediate -%} + {%- set materialized_view_path = materialized_view.fully_qualified_path_intermediate -%} + {%- else -%} + {%- set materialized_view_path = materialized_view.fully_qualified_path -%} + {%- endif -%} -{%- endmacro -%} + alter materialized view {{ materialized_view_path }} rename to {{ name }}; +{% endmacro %} -{% macro postgres__describe_materialized_view(relation) %} - -- for now just get the indexes, we don't need the name or the query yet - {% set _indexes = run_query(get_show_indexes_sql(relation)) %} - {% do return({'indexes': _indexes}) %} +{% macro postgres__replace_materialized_view_sql(new_materialized_view, existing_relation) %} + {{- create_materialized_view_sql(new_materialized_view, True) -}} +--TODO: this only works for existing materialized views + {{ rename_materialized_view(new_materialized_view, new_materialized_view.backup_name) }}; + {{ rename_materialized_view(new_materialized_view, new_materialized_view.name, True) }} {% endmacro %} From 0146343324a90abca5b307023dfe785d2c742350 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 27 Jun 2023 00:48:13 -0400 Subject: [PATCH 02/28] added updates from snowflake to core, pushed down to postgres, draft --- core/dbt/adapters/relation_configs/change.py | 74 +++++++++++++++++++ .../relation_configs/config_change.py | 23 ------ 2 files changed, 74 insertions(+), 23 deletions(-) create mode 100644 core/dbt/adapters/relation_configs/change.py delete mode 100644 core/dbt/adapters/relation_configs/config_change.py diff --git a/core/dbt/adapters/relation_configs/change.py b/core/dbt/adapters/relation_configs/change.py new file mode 100644 index 00000000000..1bfdf942b98 --- /dev/null +++ b/core/dbt/adapters/relation_configs/change.py @@ -0,0 +1,74 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Hashable + +from dbt.dataclass_schema import StrEnum + + +class RelationConfigChangeAction(StrEnum): + alter = "alter" + create = "create" + drop = "drop" + + +@dataclass(frozen=True, eq=True, unsafe_hash=True) +class RelationConfigChange(ABC): + """ + Changes are generally "alter the thing in place" or "drop the old one in favor of the new one". In other words, + you will either wind up with a single `alter` or a pair of `drop` and `create`. In the `alter` scenario, + `context` tends to be a single value, like a setting. In the `drop` and `create` scenario, + `context` tends to be the whole object, in particular for `create`. + """ + + action: RelationConfigChangeAction + context: Hashable # this is usually a RelationConfig, e.g. `IndexConfig`, or single value, e.g. `str` + + @property + @abstractmethod + def requires_full_refresh(self) -> bool: + """ + Indicates if this change can be performed via alter logic (hence `False`), or will require a full refresh + (hence `True`). While this is generally determined by the type of change being made, which could be a + static property, this is purposely being left as a dynamic property to allow for edge cases. + + Returns: + `True` if the change requires a full refresh, `False` if the change can be applied to the object + """ + raise NotImplementedError( + "Configuration change management has not been fully configured for this adapter and/or relation type." + ) + + +@dataclass +class RelationConfigChangeset(ABC): + _requires_full_refresh_override: bool = False + + @property + def requires_full_refresh(self) -> bool: + """ + This should be a calculation based on the changes that you stack on this class. + Remember to call `super().requires_full_refresh()` in your conditions, or at least reference + `self._requires_full_refresh_override` + + Returns: + `True` if any change requires a full refresh or if the override has been triggered + `False` if all changes can be made without requiring a full refresh + """ + return self._requires_full_refresh_override + + @property + def is_empty(self) -> bool: + """ + Indicates if there are any changes in this changeset. + + Returns: + `True` if there is any change or if the override has been triggered + `False` if there are no changes + """ + return not self._requires_full_refresh_override + + def force_full_refresh(self): + """ + Activates the full refresh override. + """ + self._requires_full_refresh_override = True diff --git a/core/dbt/adapters/relation_configs/config_change.py b/core/dbt/adapters/relation_configs/config_change.py deleted file mode 100644 index ac653fa5210..00000000000 --- a/core/dbt/adapters/relation_configs/config_change.py +++ /dev/null @@ -1,23 +0,0 @@ -from abc import ABC, abstractmethod -from dataclasses import dataclass -from typing import Hashable - -from dbt.adapters.relation_configs.config_base import RelationConfigBase -from dbt.dataclass_schema import StrEnum - - -class RelationConfigChangeAction(StrEnum): - alter = "alter" - create = "create" - drop = "drop" - - -@dataclass(frozen=True, eq=True, unsafe_hash=True) -class RelationConfigChange(RelationConfigBase, ABC): - action: RelationConfigChangeAction - context: Hashable # this is usually a RelationConfig, e.g. IndexConfig, but shouldn't be limited - - @property - @abstractmethod - def requires_full_refresh(self) -> bool: - raise self._not_implemented_error() From 96cac81a91cacc5ccd00fe2b44ebd59129b01a0e Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 28 Jun 2023 19:13:01 -0400 Subject: [PATCH 03/28] added updates from snowflake to core, pushed down to postgres, working --- .flake8 | 2 + core/dbt/adapters/base/relation.py | 49 ++++--- core/dbt/adapters/relation_configs/README.md | 83 ++++++++++-- .../dbt/adapters/relation_configs/__init__.py | 19 ++- .../relation_configs/{base.py => _base.py} | 66 +++------- .../{change.py => _change.py} | 0 .../adapters/relation_configs/_database.py | 23 ++++ .../relation_configs/_materialization.py | 79 ++++++++++++ core/dbt/adapters/relation_configs/_policy.py | 119 +++++++++++++++++ core/dbt/adapters/relation_configs/_schema.py | 29 +++++ .../{validation.py => _validation.py} | 2 +- core/dbt/contracts/relation.py | 2 +- .../macros/adapters/relation.sql | 30 +++++ .../materializations/materialization.sql | 69 ++++++++++ .../models/materialized_view/create.sql | 6 +- .../models/materialized_view/describe.sql | 10 +- .../materialized_view/materialized_view.sql | 88 ++----------- .../models/materialized_view/refresh.sql | 6 +- .../models/materialized_view/rename.sql | 6 +- .../dbt/adapters/postgres/relation.py | 43 +++++- .../postgres/relation_configs/__init__.py | 17 ++- .../postgres/relation_configs/base.py | 15 --- .../postgres/relation_configs/database.py | 34 ++--- .../postgres/relation_configs/index.py | 51 ++++---- .../relation_configs/materialized_view.py | 122 +++++++++--------- .../postgres/relation_configs/policies.py | 20 --- .../postgres/relation_configs/policy.py | 48 +++++++ .../postgres/relation_configs/schema.py | 35 +++-- .../dbt/include/postgres/macros/index.sql | 71 ++++++++++ .../materializations/materialized_view.sql | 63 ++++----- .../on_configuration_change.py | 7 - .../test_materialized_view.py | 27 ++-- 32 files changed, 833 insertions(+), 408 deletions(-) rename core/dbt/adapters/relation_configs/{base.py => _base.py} (70%) rename core/dbt/adapters/relation_configs/{change.py => _change.py} (100%) create mode 100644 core/dbt/adapters/relation_configs/_database.py create mode 100644 core/dbt/adapters/relation_configs/_materialization.py create mode 100644 core/dbt/adapters/relation_configs/_policy.py create mode 100644 core/dbt/adapters/relation_configs/_schema.py rename core/dbt/adapters/relation_configs/{validation.py => _validation.py} (98%) create mode 100644 core/dbt/include/global_project/macros/materializations/materialization.sql delete mode 100644 plugins/postgres/dbt/adapters/postgres/relation_configs/base.py delete mode 100644 plugins/postgres/dbt/adapters/postgres/relation_configs/policies.py create mode 100644 plugins/postgres/dbt/adapters/postgres/relation_configs/policy.py create mode 100644 plugins/postgres/dbt/include/postgres/macros/index.sql diff --git a/.flake8 b/.flake8 index e39b2fa4646..26e20a5d209 100644 --- a/.flake8 +++ b/.flake8 @@ -10,3 +10,5 @@ ignore = E741 E501 # long line checking is done in black exclude = test/ +per-file-ignores = + */__init__.py: F401 diff --git a/core/dbt/adapters/base/relation.py b/core/dbt/adapters/base/relation.py index daa27df993c..f827fefc965 100644 --- a/core/dbt/adapters/base/relation.py +++ b/core/dbt/adapters/base/relation.py @@ -1,12 +1,11 @@ from collections.abc import Hashable from dataclasses import dataclass, field -from typing import Optional, TypeVar, Any, Type, Dict, Iterator, Tuple, Set +from typing import Any, Dict, Iterator, Optional, Set, Tuple, Type, TypeVar from dbt.adapters.relation_configs import ( - RelationConfigBase, + RelationConfig, DescribeRelationResults, ) -from dbt.context.providers import RuntimeConfigObject from dbt.contracts.graph.nodes import ( SourceDefinition, ManifestNode, @@ -15,22 +14,22 @@ ModelNode, ) from dbt.contracts.relation import ( - RelationType, ComponentName, - HasQuoting, FakeAPIObject, - Policy, + HasQuoting, Path, + Policy, + RelationType, ) from dbt.exceptions import ( ApproximateMatchError, + CompilationError, DbtInternalError, + DbtRuntimeError, MultipleDatabasesNotAllowedError, ) from dbt.node_types import NodeType -from dbt.utils import filter_null_values, deep_merge, classproperty - -import dbt.exceptions +from dbt.utils import classproperty, deep_merge, filter_null_values, merge Self = TypeVar("Self", bound="BaseRelation") @@ -47,7 +46,10 @@ class BaseRelation(FakeAPIObject, Hashable): quote_policy: Policy = field(default_factory=lambda: Policy()) dbt_created: bool = False # registers RelationConfigBases to RelationTypes - relation_configs: Dict[RelationType, RelationConfigBase] = field(default_factory=dict) + + @classmethod + def relation_configs(cls) -> Dict[RelationType, RelationConfig]: + return {} def _is_exactish_match(self, field: ComponentName, value: str) -> bool: if self.dbt_created and self.quote_policy.get_part(field) is False: @@ -100,9 +102,7 @@ def matches( if not search: # nothing was passed in - raise dbt.exceptions.DbtRuntimeError( - "Tried to match relation, but no search path was passed!" - ) + raise DbtRuntimeError("Tried to match relation, but no search path was passed!") exact_match = True approximate_match = True @@ -247,7 +247,7 @@ def create_from_node( if quote_policy is None: quote_policy = {} - quote_policy = dbt.utils.merge(config.quoting, quote_policy) + quote_policy = merge(config.quoting, quote_policy) return cls.create( database=node.database, @@ -300,7 +300,7 @@ def create( return cls.from_dict(kwargs) @classmethod - def from_runtime_config(cls, runtime_config: RuntimeConfigObject) -> RelationConfigBase: + def from_model_node(cls, model_node: ModelNode) -> RelationConfig: """ Produce a validated relation config from the config available in the global jinja context. @@ -311,17 +311,16 @@ def from_runtime_config(cls, runtime_config: RuntimeConfigObject) -> RelationCon any sql is executed against the database. Args: - runtime_config: the `config` RuntimeConfigObject instance that's in the global jinja context + model_node: the `model` ModelNode instance that's in the global jinja context Returns: a validated adapter-specific, relation_type-specific RelationConfigBase instance """ - model_node: ModelNode = runtime_config.model - relation_type = cls.get_relation_type()(model_node.config.materialized) + relation_type: RelationType = cls.get_relation_type(model_node.config.materialized) - if relation_config := cls.relation_configs.get(relation_type): + if relation_config := cls.relation_configs().get(relation_type): relation = relation_config.from_model_node(model_node) else: - raise dbt.exceptions.DbtRuntimeError( + raise DbtRuntimeError( f"from_runtime_config() is not supported for the provided relation type: {relation_type}" ) @@ -330,7 +329,7 @@ def from_runtime_config(cls, runtime_config: RuntimeConfigObject) -> RelationCon @classmethod def from_describe_relation_results( cls, describe_relation_results: DescribeRelationResults, relation_type: RelationType - ) -> RelationConfigBase: + ) -> RelationConfig: """ Produce a validated relation config from a series of "describe "-type queries. @@ -345,10 +344,10 @@ def from_describe_relation_results( Returns: a validated adapter-specific, relation_type-specific RelationConfigBase instance """ - if relation_config := cls.relation_configs.get(relation_type): + if relation_config := cls.relation_configs().get(relation_type): relation = relation_config.from_describe_relation_results(describe_relation_results) else: - raise dbt.exceptions.DbtRuntimeError( + raise DbtRuntimeError( f"from_relation_results() is not supported for the provided relation type: {relation_type}" ) @@ -434,9 +433,7 @@ class InformationSchema(BaseRelation): def __post_init__(self): if not isinstance(self.information_schema_view, (type(None), str)): - raise dbt.exceptions.CompilationError( - "Got an invalid name: {}".format(self.information_schema_view) - ) + raise CompilationError("Got an invalid name: {}".format(self.information_schema_view)) @classmethod def get_path(cls, relation: BaseRelation, information_schema_view: Optional[str]) -> Path: diff --git a/core/dbt/adapters/relation_configs/README.md b/core/dbt/adapters/relation_configs/README.md index c61f04b3d05..a1d5976f89b 100644 --- a/core/dbt/adapters/relation_configs/README.md +++ b/core/dbt/adapters/relation_configs/README.md @@ -4,26 +4,87 @@ changes on those relations. It arose from the materialized view work and is curr materialized views for Postgres, Redshift, and BigQuery as well as dynamic tables for Snowflake. There are three main classes in this package. -## RelationConfigBase -This is a very small class that only has a handful of methods. It's effectively a parser, but with two sources. -The objective is to provide a stopping point between dbt-specific config and database-specific config for two -primary reasons: +## Base (Data)Classes +These are the core set of classes required to describe, validate, and monitor changes on database objects. All +other classes in `relation_configs` subclass from one of these classes. + +### RelationConfig +This class holds the primary parsing methods required for marshalling data from a user config or a database metadata +query into a `RelationConfigBase` subclass. `RelationConfigBase` is a good class to subclass for smaller, atomic +database objects or objects that may be specific to a subset of adapters. For example, a Postgres index is modelled +from `RelationConfigBase` because not every database has an index and there is not much hierarchy to an index. + +The objective of this parser is to provide a stopping point between dbt-specific config and database-specific config +for two primary reasons: - apply validation rules in the parlance of the database - articular what changes are monitored, and how those changes are applied in the database At some point this could be theoretically be replaced by a more robust framework, like `mashumaro` or `pydantic`. -## RelationConfigChange -A `RelationConfigChange` can be thought of as being analogous to a web request on a `RelationConfigBase`. -You need to know what you're doing (`action`: 'create' = GET, 'drop' = DELETE, etc.) -and the information (`context`) needed to make the change. -In our scenarios, the context tends to be an instance of `RelationConfigBase` corresponding to the new state -or a single value if the change is simple. +### RelationConfigChange +This class holds the methods required for detecting and acting on changes in a materialization. All changes +should subclass from `RelationConfigChange`. A `RelationConfigChange` can be thought of as being analogous +to a web request on a `RelationConfigBase`. You need to know what you're doing +(`action`: 'create' = GET, 'drop' = DELETE, etc.) and the information (`context`) needed to make the change. +In our scenarios, `context` tends to be either an instance of `RelationConfigBase` corresponding to the new state +or a single value if the change is simple. For example, creating an index would require the entire config; +whereas updating a setting like autorefresh for Redshift would require only the setting. -## RelationConfigValidationMixin +### RelationConfigChangeset +This class is effectively a bin for collecting instances of `RelationConfigChange`. It comes with a few helper +methods that facilitate rolling up concepts like `require_full_refresh` to the aggregate level. + +### RelationConfigValidationMixin This mixin provides optional validation mechanics that can be applied to either `RelationConfigBase` or `RelationConfigChange` subclasses. A validation rule is a combination of a `validation_check`, something that should evaluate to `True`, and an optional `validation_error`, an instance of `DbtRuntimeError` that should be raised in the event the `validation_check` fails. While optional, it's recommended that the `validation_error` be provided for clearer transparency to the end user. + +## Basic Building Blocks (Data)Classes + +### DatabaseConfig +This is the most basic version of `RelationConfigBase` that we can have. It adds a `name` and a `fully_qualified_path` +and nothing else. But we generally need a database when dbt runs. In particular, we need to reference a database +in other classes, like `SchemaConfig`. + +### SchemaConfig +As with `DatabaseConfig`, this class is pretty basic. It's existence is the most important thing. While this +may not be needed for certain databases, it's prevalent enough that it's worth building it as an out-of-the-box +object. + +### MaterializationConfig +This is the pearl in the sand. dbt generally interacts at the materialization level. As an adapter maintainer, you'll +need to subclass from most, if not all, objects in `relation_configs`; however you're likely doing so in order +to work with a `MaterializationConfig` subclass. This class has several helper methods that make it easier +to template sql in jinja. + +### IncludePolicy +Identifies whether a component is included in a fully qualified path. + +### QuotePolicy +Identifies whether a component is considered case-sensitive, or should be quoted, in a fully qualified path. This +config also contains the quote character for the database. + +## Functions + +### Policy +There are only a handful of functions here. In order to reduce duplication in docs, the reader is referred to the +docstrings on these functions for more detail: + +- `conform_part` +- `render_part` +- `render` + +### BaseRelation +There are also a handful of methods on `BaseRelation` that are meant to be used with this subpackage: + +- `from_model_node` +- `from_describe_relation_results` +- `relation_configs` (a lookup that gets consumed by the two methods above) + +These are effectively for creating `MaterializationConfig` subclasses given data from either the `model` +attribute in the global jinja context or from data from the database. Ultimately, we're treating +`BaseRelation` as a service layer that gets exposed in the jinja context, and tucking everything else into this +subpackage. diff --git a/core/dbt/adapters/relation_configs/__init__.py b/core/dbt/adapters/relation_configs/__init__.py index 7e850d8509a..c06a31a5461 100644 --- a/core/dbt/adapters/relation_configs/__init__.py +++ b/core/dbt/adapters/relation_configs/__init__.py @@ -1,13 +1,20 @@ -from dbt.adapters.relation_configs.base import ( # noqa: F401 - RelationConfigBase, - DescribeRelationResults, -) -from dbt.adapters.relation_configs.change import ( # noqa: F401 +from dbt.adapters.relation_configs._base import RelationConfig, DescribeRelationResults +from dbt.adapters.relation_configs._change import ( RelationConfigChangeAction, RelationConfigChange, RelationConfigChangeset, ) -from dbt.adapters.relation_configs.validation import ( # noqa: F401 +from dbt.adapters.relation_configs._database import DatabaseConfig +from dbt.adapters.relation_configs._materialization import MaterializationConfig +from dbt.adapters.relation_configs._policy import ( + IncludePolicy, + QuotePolicy, + conform_part, + render_part, + render, +) +from dbt.adapters.relation_configs._schema import SchemaConfig +from dbt.adapters.relation_configs._validation import ( RelationConfigValidationMixin, RelationConfigValidationRule, ) diff --git a/core/dbt/adapters/relation_configs/base.py b/core/dbt/adapters/relation_configs/_base.py similarity index 70% rename from core/dbt/adapters/relation_configs/base.py rename to core/dbt/adapters/relation_configs/_base.py index 283c97b95d8..885f02274b2 100644 --- a/core/dbt/adapters/relation_configs/base.py +++ b/core/dbt/adapters/relation_configs/_base.py @@ -1,12 +1,11 @@ from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import Union, Optional, Dict +from typing import Dict, Union import agate -from dbt.utils import filter_null_values from dbt.contracts.graph.nodes import ModelNode -from dbt.contracts.relation import Policy, ComponentName, RelationType +from dbt.utils import filter_null_values """ @@ -25,8 +24,8 @@ ]), } -Generally speaking, "primary" `RelationConfigBase` instances (e.g. materialized view) will be described with -an `agate.Table` and "dependent" `RelationConfigBase` instances (e.g. index) will be described with an `agate.Row`. +Generally speaking, "primary" `RelationConfig` instances (e.g. materialized view) will be described with +an `agate.Table` and "dependent" `RelationConfig` instances (e.g. index) will be described with an `agate.Row`. This happens simply because the primary instance is the first step in processing the metadata, but the dependent instance can be looped when dispatching to it in `parse_describe_relation_results()`. """ @@ -34,29 +33,24 @@ @dataclass(frozen=True) -class RelationConfigBase(ABC): - relation_type: RelationType - include_policy: Policy - quote_policy: Policy - quote_character: str - +class RelationConfig(ABC): @classmethod - def from_dict(cls, kwargs_dict) -> "RelationConfigBase": + def from_dict(cls, kwargs_dict) -> "RelationConfig": """ - This assumes the subclass of `RelationConfigBase` is flat, in the sense that no attribute is - itself another subclass of `RelationConfigBase`. If that's not the case, this should be overriden + This assumes the subclass of `RelationConfig` is flat, in the sense that no attribute is + itself another subclass of `RelationConfig`. If that's not the case, this should be overriden to manually manage that complexity. But remember to either call `super().from_dict()` at the end, or at least use `filter_null_values()` so that defaults get applied properly for the dataclass. Args: kwargs_dict: the dict representation of this instance - Returns: the `RelationConfigBase` representation associated with the provided dict + Returns: the `RelationConfig` representation associated with the provided dict """ return cls(**filter_null_values(kwargs_dict)) # type: ignore @classmethod - def from_model_node(cls, model_node: ModelNode) -> "RelationConfigBase": + def from_model_node(cls, model_node: ModelNode) -> "RelationConfig": """ A wrapper around `cls.parse_model_node()` and `cls.from_dict()` that pipes the results of the first into the second. This shouldn't really need to be overridden; instead, the component methods should be overridden. @@ -66,7 +60,7 @@ def from_model_node(cls, model_node: ModelNode) -> "RelationConfigBase": jinja context of a materialization Returns: - a validated `RelationConfigBase` instance specific to the adapter and relation type + a validated `RelationConfig` instance specific to the adapter and relation type """ relation_config = cls.parse_model_node(model_node) relation = cls.from_dict(relation_config) @@ -77,7 +71,7 @@ def from_model_node(cls, model_node: ModelNode) -> "RelationConfigBase": def parse_model_node(cls, model_node: ModelNode) -> dict: """ The purpose of this method is to translate the dbt/user generic parlance into the database parlance and - format it for `RelationConfigBase` consumption. + format it for `RelationConfig` consumption. In many cases this may be a one-to-one mapping; e.g. dbt calls it "schema_name" and the database calls it "schema_name". This could also be a renaming, calculation, or dispatch to a lower grain object. @@ -89,15 +83,17 @@ def parse_model_node(cls, model_node: ModelNode) -> dict: jinja context of a materialization Returns: - a non-validated dictionary version of a `RelationConfigBase` instance specific to the adapter and + a non-validated dictionary version of a `RelationConfig` instance specific to the adapter and relation type """ - raise NotImplementedError("`parse_model_node()` needs to be implemented for this adapter.") + raise NotImplementedError( + "`parse_model_node()` needs to be implemented for this relation." + ) @classmethod def from_describe_relation_results( cls, describe_relation_results: DescribeRelationResults - ) -> "RelationConfigBase": + ) -> "RelationConfig": """ A wrapper around `cls.parse_describe_relation_results()` and `cls.from_dict()` that pipes the results of the first into the second. This shouldn't really need to be overridden; instead, the component methods should @@ -108,7 +104,7 @@ def from_describe_relation_results( requisite metadata to describe this relation Returns: - a validated `RelationConfigBase` instance specific to the adapter and relation type + a validated `RelationConfig` instance specific to the adapter and relation type """ relation_config = cls.parse_describe_relation_results(describe_relation_results) relation = cls.from_dict(relation_config) @@ -120,7 +116,7 @@ def parse_describe_relation_results( cls, describe_relation_results: DescribeRelationResults ) -> dict: """ - The purpose of this method is to format the database parlance for `RelationConfigBase` consumption. + The purpose of this method is to format the database parlance for `RelationConfig` consumption. This tends to be one-to-one except for combining grains of data. For example, a single materialized view could have multiple indexes which would result in multiple queries to the database to build one @@ -133,29 +129,9 @@ def parse_describe_relation_results( requisite metadata to describe this relation Returns: - a non-validated dictionary version of a `RelationConfigBase` instance specific to the adapter and + a non-validated dictionary version of a `RelationConfig` instance specific to the adapter and relation type """ raise NotImplementedError( - "`parse_describe_relation_results()` needs to be implemented for this adapter." + "`parse_describe_relation_results()` needs to be implemented for this relation." ) - - @classmethod - def _render_part(cls, component: ComponentName, value: str) -> Optional[str]: - """ - Apply the include and quote policy to the value. - - *Note: The quote character is removed and then re-added to support backup and intermediate relation names - - Args: - component: the component of the policy to apply - value: the value to which the policies should be applied - - Returns: - a policy-compliant value - """ - if cls.include_policy.get_part(component): - if cls.quote_policy.get_part(component): - return f"{cls.quote_character}{value.replace(cls.quote_character, '')}{cls.quote_character}" - return value.lower() - return None diff --git a/core/dbt/adapters/relation_configs/change.py b/core/dbt/adapters/relation_configs/_change.py similarity index 100% rename from core/dbt/adapters/relation_configs/change.py rename to core/dbt/adapters/relation_configs/_change.py diff --git a/core/dbt/adapters/relation_configs/_database.py b/core/dbt/adapters/relation_configs/_database.py new file mode 100644 index 00000000000..27dbaf68a50 --- /dev/null +++ b/core/dbt/adapters/relation_configs/_database.py @@ -0,0 +1,23 @@ +from abc import ABC +from dataclasses import dataclass + +from dbt.adapters.relation_configs._base import RelationConfig + + +@dataclass(frozen=True, eq=True, unsafe_hash=True) +class DatabaseConfig(RelationConfig, ABC): + """ + This config identifies the minimal database parameters required for dbt to function as well + as built-ins that make macros more extensible. Additional parameters may be added by subclassing for your adapter. + """ + + name: str + + @property + def fully_qualified_path(self) -> str: + """ + This is sufficient if there is no quote policy or include policy, otherwise override it to apply those policies. + + Returns: a fully qualified path, run through the quote and include policies, for rendering in a template + """ + return self.name diff --git a/core/dbt/adapters/relation_configs/_materialization.py b/core/dbt/adapters/relation_configs/_materialization.py new file mode 100644 index 00000000000..32f1a11e63c --- /dev/null +++ b/core/dbt/adapters/relation_configs/_materialization.py @@ -0,0 +1,79 @@ +from abc import ABC +from dataclasses import dataclass + +from dbt.contracts.relation import RelationType +from dbt.adapters.relation_configs._base import RelationConfig +from dbt.adapters.relation_configs._database import DatabaseConfig +from dbt.adapters.relation_configs._schema import SchemaConfig + + +@dataclass(frozen=True) +class MaterializationConfig(RelationConfig, ABC): + """ + This config identifies the minimal materialization parameters required for dbt to function as well + as built-ins that make macros more extensible. Additional parameters may be added by subclassing for your adapter. + """ + + name: str + schema: "SchemaConfig" + query: str + relation_type: RelationType + + @property + def schema_name(self) -> str: + return self.schema.name + + @property + def database(self) -> "DatabaseConfig": + return self.schema.database + + @property + def database_name(self) -> str: + return self.database.name + + @property + def backup_name(self) -> str: + """ + Used for hot-swapping during replacement + + Returns: + a name unique to this materialized view + """ + return f"{self.name}__dbt_backup" + + @property + def intermediate_name(self) -> str: + """ + Used for hot-swapping during replacement + + Returns: + a name unique to this materialized view + """ + return f"{self.name}__dbt_tmp" + + @property + def fully_qualified_path(self) -> str: + """ + This is sufficient if there is no quote policy or include policy, otherwise override it to apply those policies. + + Returns: a fully qualified path, run through the quote and include policies, for rendering in a template + """ + return f"{self.schema.fully_qualified_path}.{self.name}" + + @property + def fully_qualified_path_backup(self) -> str: + """ + This is sufficient if there is no quote policy or include policy, otherwise override it to apply those policies. + + Returns: a fully qualified path, run through the quote and include policies, for rendering in a template + """ + return f"{self.schema.fully_qualified_path}.{self.backup_name}" + + @property + def fully_qualified_path_intermediate(self) -> str: + """ + This is sufficient if there is no quote policy or include policy, otherwise override it to apply those policies. + + Returns: a fully qualified path, run through the quote and include policies, for rendering in a template + """ + return f"{self.schema.fully_qualified_path}.{self.intermediate_name}" diff --git a/core/dbt/adapters/relation_configs/_policy.py b/core/dbt/adapters/relation_configs/_policy.py new file mode 100644 index 00000000000..b702e9a922a --- /dev/null +++ b/core/dbt/adapters/relation_configs/_policy.py @@ -0,0 +1,119 @@ +from abc import ABC +from dataclasses import dataclass +from typing import Optional, OrderedDict + +from dbt.contracts.relation import ComponentName, Policy + + +@dataclass +class IncludePolicy(Policy, ABC): + pass + + +@dataclass +class QuotePolicy(Policy, ABC): + @property + def quote_character(self) -> str: + """This is property to appeal to the `Policy` serialization.""" + return '"' + + +""" +It's advised to create your own adapter-specific version of these functions to pipe in the policies +that are specific to your adapter and for easier maintenance. This can be done easily with `functools.partial`. +See `dbt/adapters/postgres/relation_configs/policy.py` for an example. +""" + + +def conform_part( + component: ComponentName, value: str, quote_policy: QuotePolicy = QuotePolicy() +) -> Optional[str]: + """ + Apply the quote policy to the value so that it may be stored on the config object. + + *Note: Parts get quoted as part of methods like list_relations. As a result, a quote policy + of `True` just means "leave it alone", whereas a quote policy of `False` means make it case-insensitive, + which in this case is `str.lower()`. This differs from `render_part` which should only be used + for preparing templates. In that case, the quote character is used. + + It's advised to create your own adapter-specific version to pipe in the policies for easier maintenance. + See `dbt/adapters/postgres/relation_configs/policy.py` for an example. + + Args: + component: the component of the policy to apply + value: the value to which the policies should be applied + quote_policy: the quote policy for the adapter + + Returns: + a policy-compliant value + """ + if quote_policy.get_part(component): + return value + return value.lower() + + +def render_part( + component: ComponentName, + value: str, + quote_policy: QuotePolicy = QuotePolicy(), + include_policy: IncludePolicy = IncludePolicy(), +) -> Optional[str]: + """ + Apply the include and quote policy to the value so that it may be rendered in a template. + + *Note: This differs from `conform_part` in that the quote character actually gets used and + the include policy gets applied. + Additionally, there may be times when `value` shows up already quoted, if that's the case, these + characters are removed so that `value` does not wind up double-quoted. + + It's advised to create your own adapter-specific version to pipe in the policies for easier maintenance. + See `dbt/adapters/postgres/relation_configs/policy.py` for an example. + + Args: + component: the component of the policy to apply + value: the value to which the policies should be applied + quote_policy: the quote policy for the adapter + include_policy: the include policy for the adapter + + Returns: + a policy-compliant value + """ + quote = quote_policy.quote_character + if include_policy.get_part(component): + if quote_policy.get_part(component): + return f"{quote}{value.replace(quote, '')}{quote}" + return value.lower() + return None + + +def render( + parts: OrderedDict[ComponentName, str], + quote_policy: QuotePolicy = QuotePolicy(), + include_policy: IncludePolicy = IncludePolicy(), + delimiter: str = ".", +) -> str: + """ + This does the same thing as `cls.render_part()`, but all at once. + + We need to make sure we join the parts in the correct order, including scenarios where we don't + receive all the components, e.g. someone needs a fully qualified schema (hence no identifier). + + It's advised to create your own adapter-specific version to pipe in the policies for easier maintenance. + See `dbt/adapters/postgres/relation_configs/policy.py` for an example. + + Args: + parts: an ordered dictionary mapping ComponentName to value, provide the dictionary in the order in which + you want the parts joined + quote_policy: the quote policy for the adapter + include_policy: the include policy for the adapter + delimiter: the delimiter to use between parts + + Returns: + a fully rendered path ready for a jinja template + """ + rendered_parts = [ + render_part(component, value, quote_policy, include_policy) + for component, value in parts.items() + ] + rendered_path = delimiter.join(part for part in rendered_parts if part is not None) + return rendered_path diff --git a/core/dbt/adapters/relation_configs/_schema.py b/core/dbt/adapters/relation_configs/_schema.py new file mode 100644 index 00000000000..d9676aa7a64 --- /dev/null +++ b/core/dbt/adapters/relation_configs/_schema.py @@ -0,0 +1,29 @@ +from abc import ABC +from dataclasses import dataclass + +from dbt.adapters.relation_configs._base import RelationConfig +from dbt.adapters.relation_configs._database import DatabaseConfig + + +@dataclass(frozen=True, eq=True, unsafe_hash=True) +class SchemaConfig(RelationConfig, ABC): + """ + This config identifies the minimal schema parameters required for dbt to function as well + as built-ins that make macros more extensible. Additional parameters may be added by subclassing for your adapter. + """ + + name: str + database: DatabaseConfig + + @property + def database_name(self) -> str: + return self.database.name + + @property + def fully_qualified_path(self) -> str: + """ + This is sufficient if there is no quote policy or include policy, otherwise override it to apply those policies. + + Returns: a fully qualified path, run through the quote and include policies, for rendering in a template + """ + return f"{self.database.fully_qualified_path}.{self.name}" diff --git a/core/dbt/adapters/relation_configs/validation.py b/core/dbt/adapters/relation_configs/_validation.py similarity index 98% rename from core/dbt/adapters/relation_configs/validation.py rename to core/dbt/adapters/relation_configs/_validation.py index 9164a0e94cb..cc1fe9a9c97 100644 --- a/core/dbt/adapters/relation_configs/validation.py +++ b/core/dbt/adapters/relation_configs/_validation.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import Set, Optional +from typing import Optional, Set from dbt.exceptions import DbtRuntimeError diff --git a/core/dbt/contracts/relation.py b/core/dbt/contracts/relation.py index 2cf811f9f6c..5fce5308214 100644 --- a/core/dbt/contracts/relation.py +++ b/core/dbt/contracts/relation.py @@ -86,7 +86,7 @@ class Path(FakeAPIObject): identifier: Optional[str] = None def __post_init__(self): - # handle pesky jinja2.Undefined sneaking in here and messing up rende + # handle pesky jinja2.Undefined sneaking in here and messing up render if not isinstance(self.database, (type(None), str)): raise CompilationError("Got an invalid path database: {}".format(self.database)) if not isinstance(self.schema, (type(None), str)): diff --git a/core/dbt/include/global_project/macros/adapters/relation.sql b/core/dbt/include/global_project/macros/adapters/relation.sql index f0dde7f20f0..0c16ee6e33a 100644 --- a/core/dbt/include/global_project/macros/adapters/relation.sql +++ b/core/dbt/include/global_project/macros/adapters/relation.sql @@ -96,3 +96,33 @@ {{ adapter.drop_relation(relation) }} {% endif %} {% endmacro %} + + +-- a user-friendly interface into adapter.get_relation for relation_configs.MaterializationConfigBase instances +{% macro load_cached_relation_config(relation_config) %} + {% do return(adapter.get_relation( + database=relation_config.database_name, + schema=relation_config.schema_name, + identifier=relation_config.name + )) -%} +{% endmacro %} + + +-- a user-friendly interface into adapter.get_relation for relation_configs.MaterializationConfigBase instances +{% macro load_cached_relation_config_backup(relation_config) %} + {% do return(adapter.get_relation( + database=relation_config.database_name, + schema=relation_config.schema_name, + identifier=relation_config.backup_name + )) -%} +{% endmacro %} + + +-- a user-friendly interface into adapter.get_relation for relation_configs.MaterializationConfigBase instances +{% macro load_cached_relation_config_intermediate(relation_config) %} + {% do return(adapter.get_relation( + database=relation_config.database_name, + schema=relation_config.schema_name, + identifier=relation_config.intermediate_name + )) -%} +{% endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/materialization.sql b/core/dbt/include/global_project/macros/materializations/materialization.sql new file mode 100644 index 00000000000..23c3446202d --- /dev/null +++ b/core/dbt/include/global_project/macros/materializations/materialization.sql @@ -0,0 +1,69 @@ +{# /* + These are step macros for building out a materialization based on `MaterializationConfigBase`, which can be + found in `dbt/adapters/relation_configs/materialization.py` + + Note: You cannot start the macro name with "materialization" because then dbt thinks it's a materialization + and not a macro. +*/ #} + + +{% macro mat_setup(materialization_config, pre_hooks) %} + + -- backup_relation and intermediate_relation should not already exist in the database + -- it's possible these exist because of a previous run that exited unexpectedly + {% set backup_relation = load_cached_relation_config_backup(materialization_config) %} + {% set intermediate_relation = load_cached_relation_config_intermediate(materialization_config) %} + + -- drop the temp relations if they exist already in the database + {{ drop_relation_if_exists(backup_relation) }} + {{ drop_relation_if_exists(intermediate_relation) }} + + {{ run_hooks(pre_hooks, inside_transaction=False) }} + +{% endmacro %} + + +{% macro mat_teardown(materialization_config, post_hooks) %} + + -- backup_relation and intermediate_relation may exist if the materialized view was replaced + {% set backup_relation = load_cached_relation_config_backup(materialization_config) %} + {% set intermediate_relation = load_cached_relation_config_intermediate(materialization_config) %} + + -- drop the temp relations if they exist to leave the database clean for the next run + {{ drop_relation_if_exists(backup_relation) }} + {{ drop_relation_if_exists(intermediate_relation) }} + + {{ run_hooks(post_hooks, inside_transaction=False) }} + +{% endmacro %} + + +{% macro mat_execute_no_op(materialization_config) %} + {% do store_raw_result( + name="main", + message="skip " ~ materialization_config.fully_qualified_path, + code="skip", + rows_affected="-1" + ) %} +{% endmacro %} + + +{% macro mat_execute_build_sql(materialization_config, build_sql, relation_exists, post_hooks) %} + + -- `BEGIN` happens here: + {{ run_hooks(pre_hooks, inside_transaction=True) }} + + {% set grant_config = config.get('grants') %} + + {% call statement(name="main") %} + {{ build_sql }} + {% endcall %} + + {% set should_revoke = should_revoke(relation_exists, full_refresh_mode=True) %} + {% do apply_grants(materialization_config.fully_qualified_path, grant_config, should_revoke=should_revoke) %} + + {{ run_hooks(post_hooks, inside_transaction=True) }} + + {{ adapter.commit() }} + +{% endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/create.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view/create.sql index 7ac572c3b49..13e1cf752e5 100644 --- a/core/dbt/include/global_project/macros/materializations/models/materialized_view/create.sql +++ b/core/dbt/include/global_project/macros/materializations/models/materialized_view/create.sql @@ -1,9 +1,9 @@ -{% macro create_materialized_view_sql(materialized_view, intermediate=False) -%} +{% macro create_materialized_view_sql(materialized_view, as_intermediate=False) -%} {{- log('Applying CREATE to: ' ~ materialized_view.fully_qualified_path) -}} - {{- adapter.dispatch('create_materialized_view_sql', 'dbt')(materialized_view, intermediate) -}} + {{- adapter.dispatch('create_materialized_view_sql', 'dbt')(materialized_view, as_intermediate) -}} {%- endmacro %} -{% macro default__create_materialized_view_sql(materialized_view, intermediate=False) -%} +{% macro default__create_materialized_view_sql(materialized_view, as_intermediate=False) -%} {{ exceptions.raise_compiler_error("Materialized views have not been implemented for this adapter.") }} {% endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/describe.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view/describe.sql index e55f1c409a9..be855880a01 100644 --- a/core/dbt/include/global_project/macros/materializations/models/materialized_view/describe.sql +++ b/core/dbt/include/global_project/macros/materializations/models/materialized_view/describe.sql @@ -1,9 +1,13 @@ -{% macro describe_materialized_view(materialized_view) %} +{# /* + This needs to be a {% do return(...) %} because the macro returns a dictionary, not a string. +*/ #} + +{% macro describe_materialized_view_sql(materialized_view) %} {{- log('Applying DESCRIBE to: ' ~ materialized_view.fully_qualified_path) -}} - {{- adapter.dispatch('describe_materialized_view', 'dbt')(materialized_view) -}} + {% do return(adapter.dispatch('describe_materialized_view_sql', 'dbt')(materialized_view)) %} {% endmacro %} -{% macro default__describe_materialized_view(materialized_view) %} +{% macro default__describe_materialized_view_sql(materialized_view) %} {{ exceptions.raise_compiler_error("Materialized views have not been implemented for this adapter.") }} {% endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/materialized_view.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view/materialized_view.sql index 62a58bc6c61..76cce6d38f2 100644 --- a/core/dbt/include/global_project/macros/materializations/models/materialized_view/materialized_view.sql +++ b/core/dbt/include/global_project/macros/materializations/models/materialized_view/materialized_view.sql @@ -1,68 +1,37 @@ {% materialization materialized_view, default %} -- Try to create a valid materialized view from the config before doing anything else - {% set new_materialized_view = adapter.Relation.from_runtime_config(config) %} + {% set new_materialized_view = adapter.Relation.from_model_node(config.model) %} -- We still need these because they tie into the existing process (e.g. RelationBase vs. RelationConfigBase) - {% set existing_relation = load_cached_relation(this) %} + {% set existing_relation = load_cached_relation_config(new_materialized_view) %} {% set target_relation = this.incorporate(type=this.MaterializedView) %} - {% set intermediate_relation = make_intermediate_relation(target_relation) %} - {% set backup_relation_type = target_relation.MaterializedView if existing_relation is none else existing_relation.type %} - {% set backup_relation = make_backup_relation(target_relation, backup_relation_type) %} - {{ materialized_view_setup(backup_relation, intermediate_relation, pre_hooks) }} + {{ mat_setup(new_materialized_view, pre_hooks) }} - {% set build_sql = materialized_view_build_sql(new_materialized_view, existing_relation, backup_relation, intermediate_relation) %} + {% set build_sql = materialized_view_build_sql(new_materialized_view, existing_relation) %} + {% set relation_exists = existing_relation is not none %} {% if build_sql == '' %} - {{ materialized_view_execute_no_op(new_materialized_view) }} + {{ mat_execute_no_op(new_materialized_view) }} {% else %} - {{ materialized_view_execute_build_sql(build_sql, new_materialized_view, post_hooks) }} + {{ mat_execute_build_sql(new_materialized_view, build_sql, relation_exists, post_hooks) }} {% endif %} - {{ materialized_view_teardown(backup_relation, intermediate_relation, post_hooks) }} + {{ mat_teardown(new_materialized_view, post_hooks) }} {{ return({'relations': [target_relation]}) }} {% endmaterialization %} -{% macro materialized_view_setup(backup_relation, intermediate_relation, pre_hooks) %} - - -- backup_relation and intermediate_relation should not already exist in the database - -- it's possible these exist because of a previous run that exited unexpectedly - {% set preexisting_backup_relation = load_cached_relation(backup_relation) %} - {% set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) %} - - -- drop the temp relations if they exist already in the database - {{ drop_relation_if_exists(preexisting_backup_relation) }} - {{ drop_relation_if_exists(preexisting_intermediate_relation) }} - - {{ run_hooks(pre_hooks, inside_transaction=False) }} - -{% endmacro %} - - -{% macro materialized_view_teardown(backup_relation, intermediate_relation, post_hooks) %} - - -- drop the temp relations if they exist to leave the database clean for the next run - {{ drop_relation_if_exists(backup_relation) }} - {{ drop_relation_if_exists(intermediate_relation) }} - - {{ run_hooks(post_hooks, inside_transaction=False) }} - -{% endmacro %} - - -{% macro materialized_view_build_sql(new_materialized_view, existing_relation, backup_relation, intermediate_relation) %} - - {% set full_refresh_mode = should_full_refresh() %} +{% macro materialized_view_build_sql(new_materialized_view, existing_relation) %} -- determine the scenario we're in: create, full_refresh, alter {% if existing_relation is none %} {% set build_sql = create_materialized_view_sql(new_materialized_view) %} - {% elif full_refresh_mode or not existing_relation.is_materialized_view %} - {% set build_sql = replace_materialized_view_sql(new_materialized_view, existing_relation, backup_relation, intermediate_relation) %} + {% elif should_full_refresh() or not existing_relation.is_materialized_view %} + {% set build_sql = replace_materialized_view_sql(new_materialized_view, existing_relation) %} {% else %} {% set build_sql = alter_materialized_view_with_on_configuration_option_sql(new_materialized_view) %} {% endif %} @@ -74,7 +43,7 @@ {% macro alter_materialized_view_with_on_configuration_option_sql(new_materialized_view) %} - {% set describe_relation_results = describe_materialized_view(new_materialized_view) %} + {% set describe_relation_results = describe_materialized_view_sql(new_materialized_view) %} {% set existing_materialized_view = adapter.Relation.from_describe_relation_results(describe_relation_results, adapter.Relation.MaterializedView) %} {% set on_configuration_change = config.get('on_configuration_change') %} @@ -98,36 +67,3 @@ {% do return(build_sql) %} {% endmacro %} - - -{% macro materialized_view_execute_no_op(new_materialized_view) %} - {% do store_raw_result( - name="main", - message="skip " ~ new_materialized_view.fully_qualified_path, - code="skip", - rows_affected="-1" - ) %} -{% endmacro %} - - -{% macro materialized_view_execute_build_sql(build_sql, new_materialized_view, post_hooks) %} - - -- `BEGIN` happens here: - {{ run_hooks(pre_hooks, inside_transaction=True) }} - - {% set grant_config = config.get('grants') %} - - {% call statement(name="main") %} - {{ build_sql }} - {% endcall %} - - {% set should_revoke = should_revoke(new_materialized_view.fully_qualified_path, full_refresh_mode=True) %} - {% do apply_grants(new_materialized_view.fully_qualified_path, grant_config, should_revoke=should_revoke) %} - - {% do persist_docs(new_materialized_view.fully_qualified_path, model) %} - - {{ run_hooks(post_hooks, inside_transaction=True) }} - - {{ adapter.commit() }} - -{% endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/refresh.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view/refresh.sql index 6f8605dbacc..f0e5039f968 100644 --- a/core/dbt/include/global_project/macros/materializations/models/materialized_view/refresh.sql +++ b/core/dbt/include/global_project/macros/materializations/models/materialized_view/refresh.sql @@ -1,9 +1,9 @@ -{% macro refresh_materialized_view(materialized_view) %} +{% macro refresh_materialized_view_sql(materialized_view) %} {{- log('Applying REFRESH to: ' ~ materialized_view.fully_qualified_path) -}} - {{- adapter.dispatch('refresh_materialized_view', 'dbt')(materialized_view) -}} + {{- adapter.dispatch('refresh_materialized_view_sql', 'dbt')(materialized_view) -}} {% endmacro %} -{% macro default__refresh_materialized_view(materialized_view) %} +{% macro default__refresh_materialized_view_sql(materialized_view) %} {{ exceptions.raise_compiler_error("Materialized views have not been implemented for this adapter.") }} {% endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/rename.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view/rename.sql index 53087873d86..d9d31294aa2 100644 --- a/core/dbt/include/global_project/macros/materializations/models/materialized_view/rename.sql +++ b/core/dbt/include/global_project/macros/materializations/models/materialized_view/rename.sql @@ -1,9 +1,9 @@ -{% macro rename_materialized_view(materialized_view, name, intermediate=False) %} +{% macro rename_materialized_view_sql(materialized_view, name, from_intermediate=False) %} {{- log('Applying RENAME to: ' ~ materialized_view.fully_qualified_path) -}} - {{- adapter.dispatch('rename_materialized_view', 'dbt')(materialized_view, name, intermediate) -}} + {{- adapter.dispatch('rename_materialized_view_sql', 'dbt')(materialized_view, name, from_intermediate) -}} {% endmacro %} -{% macro default__rename_materialized_view(materialized_view, name, intermediate=False) %} +{% macro default__rename_materialized_view_sql(materialized_view, name, from_intermediate=False) %} {{ exceptions.raise_compiler_error("Materialized views have not been implemented for this adapter.") }} {% endmacro %} diff --git a/plugins/postgres/dbt/adapters/postgres/relation.py b/plugins/postgres/dbt/adapters/postgres/relation.py index 986114b83d2..6fc537d405f 100644 --- a/plugins/postgres/dbt/adapters/postgres/relation.py +++ b/plugins/postgres/dbt/adapters/postgres/relation.py @@ -1,18 +1,26 @@ from dataclasses import dataclass, field -from typing import Set, FrozenSet +from datetime import datetime +from typing import Set, FrozenSet, Dict from dbt.adapters.base.relation import BaseRelation -from dbt.adapters.relation_configs import RelationConfigChangeAction +from dbt.adapters.relation_configs import ( + RelationConfigChangeAction, + RelationConfig, + MaterializationConfig, +) +from dbt.contracts.relation import ComponentName, RelationType from dbt.exceptions import DbtRuntimeError +import dbt.utils from dbt.adapters.postgres.relation_configs import ( PostgresIndexConfig, PostgresIndexConfigChange, PostgresMaterializedViewConfig, PostgresMaterializedViewConfigChangeset, - MAX_CHARACTERS_IN_IDENTIFIER, PostgresIncludePolicy, PostgresQuotePolicy, + postgres_conform_part, + MAX_CHARACTERS_IN_IDENTIFIER, ) @@ -21,6 +29,10 @@ class PostgresRelation(BaseRelation): include_policy: PostgresIncludePolicy = field(default_factory=PostgresIncludePolicy) quote_policy: PostgresQuotePolicy = field(default_factory=PostgresQuotePolicy) + @classmethod + def relation_configs(cls) -> Dict[RelationType, RelationConfig]: + return {RelationType.MaterializedView: PostgresMaterializedViewConfig} # type: ignore + def __post_init__(self): # Check for length of Postgres table/view names. # Check self.type to exclude test relation identifiers @@ -92,3 +104,28 @@ def index_config_changeset( for index in new_indexes.difference(existing_indexes) ) return set().union(drop_changes, create_changes) + + @staticmethod + def generate_index_name( + materialization_config: MaterializationConfig, index_config: PostgresIndexConfig + ) -> str: + return dbt.utils.md5( + "_".join( + { + postgres_conform_part( + ComponentName.Database, materialization_config.database_name + ), + postgres_conform_part( + ComponentName.Schema, materialization_config.schema_name + ), + postgres_conform_part(ComponentName.Identifier, materialization_config.name), + *sorted( + postgres_conform_part(ComponentName.Identifier, column) + for column in index_config.column_names + ), + str(index_config.unique), + str(index_config.method), + str(datetime.utcnow().isoformat()), + } + ) + ) diff --git a/plugins/postgres/dbt/adapters/postgres/relation_configs/__init__.py b/plugins/postgres/dbt/adapters/postgres/relation_configs/__init__.py index ac2def44060..f34e59ddc07 100644 --- a/plugins/postgres/dbt/adapters/postgres/relation_configs/__init__.py +++ b/plugins/postgres/dbt/adapters/postgres/relation_configs/__init__.py @@ -1,13 +1,16 @@ -from dbt.adapters.postgres.relation_configs.index import ( # noqa: F401 +from dbt.adapters.postgres.relation_configs.index import ( PostgresIndexConfig, PostgresIndexConfigChange, ) -from dbt.adapters.postgres.relation_configs.policies import ( # noqa: F401 - PostgresIncludePolicy, - PostgresQuotePolicy, - MAX_CHARACTERS_IN_IDENTIFIER, -) -from dbt.adapters.postgres.relation_configs.materialized_view import ( # noqa: F401 +from dbt.adapters.postgres.relation_configs.materialized_view import ( PostgresMaterializedViewConfig, PostgresMaterializedViewConfigChangeset, ) +from dbt.adapters.postgres.relation_configs.policy import ( + MAX_CHARACTERS_IN_IDENTIFIER, + PostgresIncludePolicy, + PostgresQuotePolicy, + postgres_conform_part, + postgres_render_part, + postgres_render, +) diff --git a/plugins/postgres/dbt/adapters/postgres/relation_configs/base.py b/plugins/postgres/dbt/adapters/postgres/relation_configs/base.py deleted file mode 100644 index 40a966140d2..00000000000 --- a/plugins/postgres/dbt/adapters/postgres/relation_configs/base.py +++ /dev/null @@ -1,15 +0,0 @@ -from abc import ABC -from dataclasses import dataclass - -from dbt.adapters.relation_configs import RelationConfigBase - -from dbt.adapters.postgres.relation_configs.policies import ( - PostgresIncludePolicy, - PostgresQuotePolicy, -) - - -@dataclass(frozen=True, eq=True, unsafe_hash=True) -class PostgresRelationConfigBase(RelationConfigBase, ABC): - include_policy = PostgresIncludePolicy() - quote_policy = PostgresQuotePolicy() diff --git a/plugins/postgres/dbt/adapters/postgres/relation_configs/database.py b/plugins/postgres/dbt/adapters/postgres/relation_configs/database.py index ff399f15716..ea8509d66ff 100644 --- a/plugins/postgres/dbt/adapters/postgres/relation_configs/database.py +++ b/plugins/postgres/dbt/adapters/postgres/relation_configs/database.py @@ -1,8 +1,10 @@ +from collections import OrderedDict from dataclasses import dataclass from typing import Set import agate from dbt.adapters.relation_configs import ( + DatabaseConfig, RelationConfigValidationMixin, RelationConfigValidationRule, ) @@ -10,11 +12,11 @@ from dbt.contracts.relation import ComponentName from dbt.exceptions import DbtRuntimeError -from dbt.adapters.postgres.relation_configs.base import PostgresRelationConfigBase +from dbt.adapters.postgres.relation_configs.policy import postgres_render, postgres_conform_part @dataclass(frozen=True, eq=True, unsafe_hash=True) -class PostgresDatabaseConfig(PostgresRelationConfigBase, RelationConfigValidationMixin): +class PostgresDatabaseConfig(DatabaseConfig, RelationConfigValidationMixin): """ This config follow the specs found here: https://www.postgresql.org/docs/current/sql-createdatabase.html @@ -27,7 +29,7 @@ class PostgresDatabaseConfig(PostgresRelationConfigBase, RelationConfigValidatio @property def fully_qualified_path(self) -> str: - return self.name + return postgres_render(OrderedDict({ComponentName.Database: self.name})) @property def validation_rules(self) -> Set[RelationConfigValidationRule]: @@ -41,24 +43,26 @@ def validation_rules(self) -> Set[RelationConfigValidationRule]: } @classmethod - def from_dict(cls, config_dict: dict) -> "PostgresDatabaseConfig": - kwargs_dict = { - "name": cls._render_part(ComponentName.Database, config_dict["name"]), - } - - database: "PostgresDatabaseConfig" = super().from_dict(kwargs_dict) # type: ignore + def from_dict(cls, config_dict: dict) -> "DatabaseConfig": + """ + Because this returns a frozen dataclass, this method should be overridden if additional parameters are supplied. + """ + kwargs_dict = {"name": postgres_conform_part(ComponentName.Database, config_dict["name"])} + database: "DatabaseConfig" = super().from_dict(kwargs_dict) # type: ignore return database @classmethod def parse_model_node(cls, model_node: ModelNode) -> dict: - config_dict = { - "name": model_node.database, - } + """ + Because this returns a `dict`, this method should be extended if additional parameters are supplied. + """ + config_dict = {"name": model_node.database} return config_dict @classmethod def parse_describe_relation_results(cls, describe_relation_results: agate.Row) -> dict: - config_dict = { - "name": describe_relation_results["database_name"], - } + """ + Because this returns a `dict`, this method should be extended if additional parameters are supplied. + """ + config_dict = {"name": describe_relation_results["databasename"]} return config_dict diff --git a/plugins/postgres/dbt/adapters/postgres/relation_configs/index.py b/plugins/postgres/dbt/adapters/postgres/relation_configs/index.py index 1602d0e802b..23c94dba1b9 100644 --- a/plugins/postgres/dbt/adapters/postgres/relation_configs/index.py +++ b/plugins/postgres/dbt/adapters/postgres/relation_configs/index.py @@ -2,17 +2,18 @@ from typing import Set, FrozenSet import agate -from dbt.contracts.relation import ComponentName -from dbt.dataclass_schema import StrEnum -from dbt.exceptions import DbtRuntimeError from dbt.adapters.relation_configs import ( + RelationConfig, RelationConfigValidationMixin, RelationConfigValidationRule, RelationConfigChangeAction, RelationConfigChange, ) +from dbt.contracts.relation import ComponentName +from dbt.dataclass_schema import StrEnum +from dbt.exceptions import DbtRuntimeError -from dbt.adapters.postgres.relation_configs.base import PostgresRelationConfigBase +from dbt.adapters.postgres.relation_configs.policy import postgres_conform_part class PostgresIndexMethod(StrEnum): @@ -29,24 +30,30 @@ def default(cls) -> "PostgresIndexMethod": @dataclass(frozen=True, eq=True, unsafe_hash=True) -class PostgresIndexConfig(PostgresRelationConfigBase, RelationConfigValidationMixin): +class PostgresIndexConfig(RelationConfig, RelationConfigValidationMixin): """ This config fallows the specs found here: https://www.postgresql.org/docs/current/sql-createindex.html The following parameters are configurable by dbt: - - name: the name of the index in the database, this isn't predictable since we apply a timestamp + - column_names: the columns in the index - unique: checks for duplicate values when the index is created and on data updates - method: the index method to be used - - column_names: the columns in the index + + The following parameters are not configurable by dbt, but are required for certain functionality: + - name: the name of the index in the database Applicable defaults for non-configurable parameters: - concurrently: `False` - nulls_distinct: `True` + + *Note: The index does not have a name until it is created in the database. The name also must be globally + unique, not just within the materialization to which it belongs. Hence, the name is a combination of attributes + on both the index and the materialization. This is calculated with `PostgresRelation.generate_index_name()`. """ + column_names: FrozenSet[str] = field(hash=True) name: str = field(default=None, hash=False, compare=False) - column_names: FrozenSet[str] = field(default_factory=frozenset, hash=True) unique: bool = field(default=False, hash=True) method: PostgresIndexMethod = field(default=PostgresIndexMethod.default(), hash=True) @@ -66,12 +73,15 @@ def from_dict(cls, config_dict) -> "PostgresIndexConfig": kwargs_dict = { "name": config_dict.get("name"), "column_names": frozenset( - cls._render_part(ComponentName.Identifier, column) + postgres_conform_part(ComponentName.Identifier, column) for column in config_dict.get("column_names", set()) ), "unique": config_dict.get("unique"), - "method": config_dict.get("method"), } + + if method := config_dict.get("method"): + kwargs_dict.update({"method": PostgresIndexMethod(method)}) + index: "PostgresIndexConfig" = super().from_dict(kwargs_dict) # type: ignore return index @@ -87,25 +97,13 @@ def parse_model_node(cls, model_node_entry: dict) -> dict: @classmethod def parse_describe_relation_results(cls, describe_relation_results: agate.Row) -> dict: config_dict = { - "name": describe_relation_results.get("name"), - "column_names": set(describe_relation_results.get("column_names", "").split(",")), - "unique": describe_relation_results.get("unique"), - "method": describe_relation_results.get("method"), + "name": describe_relation_results["name"], + "column_names": set(describe_relation_results["column_names"].split(",")), + "unique": describe_relation_results["unique"], + "method": describe_relation_results["method"], } return config_dict - @property - def as_node_config(self) -> dict: - """ - Returns: a dictionary that can be passed into `get_create_index_sql()` - """ - node_config = { - "columns": list(self.column_names), - "unique": self.unique, - "type": self.method.value, - } - return node_config - @dataclass(frozen=True, eq=True, unsafe_hash=True) class PostgresIndexConfigChange(RelationConfigChange, RelationConfigValidationMixin): @@ -131,7 +129,6 @@ class PostgresIndexConfigChange(RelationConfigChange, RelationConfigValidationMi } """ - action: RelationConfigChangeAction context: PostgresIndexConfig @property diff --git a/plugins/postgres/dbt/adapters/postgres/relation_configs/materialized_view.py b/plugins/postgres/dbt/adapters/postgres/relation_configs/materialized_view.py index 9a6ded32af7..3162359a713 100644 --- a/plugins/postgres/dbt/adapters/postgres/relation_configs/materialized_view.py +++ b/plugins/postgres/dbt/adapters/postgres/relation_configs/materialized_view.py @@ -1,8 +1,10 @@ +from collections import OrderedDict from dataclasses import dataclass, field -from typing import Set, FrozenSet, List, Dict +from typing import Set, FrozenSet, List, Dict, Optional import agate from dbt.adapters.relation_configs import ( + MaterializationConfig, RelationConfigChangeset, RelationConfigValidationMixin, RelationConfigValidationRule, @@ -10,25 +12,29 @@ from dbt.contracts.graph.nodes import ModelNode from dbt.contracts.relation import ComponentName, RelationType from dbt.exceptions import DbtRuntimeError +import dbt.utils -from dbt.adapters.postgres.relation_configs.base import PostgresRelationConfigBase -from dbt.adapters.postgres.relation_configs.database import PostgresDatabaseConfig from dbt.adapters.postgres.relation_configs.index import ( PostgresIndexConfig, PostgresIndexConfigChange, ) -from dbt.adapters.postgres.relation_configs.policies import MAX_CHARACTERS_IN_IDENTIFIER +from dbt.adapters.postgres.relation_configs.policy import ( + postgres_render, + postgres_conform_part, + MAX_CHARACTERS_IN_IDENTIFIER, +) from dbt.adapters.postgres.relation_configs.schema import PostgresSchemaConfig @dataclass(frozen=True, eq=True, unsafe_hash=True) -class PostgresMaterializedViewConfig(PostgresRelationConfigBase, RelationConfigValidationMixin): +class PostgresMaterializedViewConfig(MaterializationConfig, RelationConfigValidationMixin): """ This config follows the specs found here: https://www.postgresql.org/docs/current/sql-creatematerializedview.html The following parameters are configurable by dbt: - - table_name: name of the materialized view + - name: name of the materialized view + - schema: schema that contains the materialized view - query: the query that defines the view - indexes: the collection (set) of indexes on the materialized view @@ -38,59 +44,46 @@ class PostgresMaterializedViewConfig(PostgresRelationConfigBase, RelationConfigV - with_data: `True` """ - table_name: str + name: str schema: PostgresSchemaConfig - query: str + query: str = field(hash=False, compare=False) indexes: FrozenSet[PostgresIndexConfig] = field(default_factory=frozenset) - relation_type = RelationType.MaterializedView - - @property - def schema_name(self) -> str: - return self.schema.schema_name - - @property - def database(self) -> PostgresDatabaseConfig: - return self.schema.database - - @property - def database_name(self) -> str: - return self.database.name - - @property - def backup_name(self) -> str: - """ - Used for hot-swapping during replacement - - Returns: - a name unique to this materialized view - """ - return self._render_part(ComponentName.Identifier, f"{self.table_name}__dbt_backup") - - @property - def intermediate_name(self) -> str: - """ - Used for hot-swapping during replacement - - Returns: - a name unique to this materialized view - """ - return self._render_part(ComponentName.Identifier, f"{self.table_name}__dbt_tmp") + relation_type: Optional[RelationType] = RelationType.MaterializedView @property def fully_qualified_path(self) -> str: - return self._fully_qualified_path(self.table_name) + return postgres_render( + OrderedDict( + { + ComponentName.Database: self.database_name, + ComponentName.Schema: self.schema_name, + ComponentName.Identifier: self.name, + } + ) + ) @property def fully_qualified_path_backup(self) -> str: - return self._fully_qualified_path(self.backup_name) + return postgres_render( + OrderedDict( + { + ComponentName.Database: self.database_name, + ComponentName.Schema: self.schema_name, + ComponentName.Identifier: self.backup_name, + } + ) + ) @property def fully_qualified_path_intermediate(self) -> str: - return self._fully_qualified_path(self.intermediate_name) - - def _fully_qualified_path(self, table_name) -> str: - return ".".join( - part for part in [self.schema.fully_qualified_path, table_name] if part is not None + return postgres_render( + OrderedDict( + { + ComponentName.Database: self.database_name, + ComponentName.Schema: self.schema_name, + ComponentName.Identifier: self.intermediate_name, + } + ) ) @property @@ -103,20 +96,20 @@ def validation_rules(self) -> Set[RelationConfigValidationRule]: """ return { RelationConfigValidationRule( - validation_check=self.table_name is None - or len(self.table_name) <= MAX_CHARACTERS_IN_IDENTIFIER, + validation_check=self.name is None + or len(self.name) <= MAX_CHARACTERS_IN_IDENTIFIER, validation_error=DbtRuntimeError( f"The materialized view name is more than {MAX_CHARACTERS_IN_IDENTIFIER} " - f"characters: {self.table_name}" + f"characters: {self.name}" ), ), RelationConfigValidationRule( - validation_check=all({self.database_name, self.schema_name, self.table_name}), + validation_check=all({self.database_name, self.schema_name, self.name}), validation_error=DbtRuntimeError( f"dbt-snowflake requires all three parts of an object's path, received:/n" f" database: {self.database_name}/n" f" schema: {self.schema_name}/n" - f" identifier: {self.table_name}/n" + f" identifier: {self.name}/n" ), ), } @@ -134,7 +127,7 @@ def from_dict(cls, config_dict: dict) -> "PostgresMaterializedViewConfig": Returns: an instance of this class """ kwargs_dict = { - "table_name": cls._render_part(ComponentName.Identifier, config_dict["table_name"]), + "name": postgres_conform_part(ComponentName.Identifier, config_dict["name"]), "schema": PostgresSchemaConfig.from_dict(config_dict["schema"]), "query": config_dict["query"], "indexes": frozenset( @@ -181,7 +174,7 @@ def parse_model_node(cls, model_node: ModelNode) -> dict: """ indexes: List[dict] = model_node.config.extra.get("indexes", []) config_dict = { - "table_name": model_node.identifier, + "name": model_node.identifier, "schema": PostgresSchemaConfig.parse_model_node(model_node), "query": (model_node.compiled_code or "").strip(), "indexes": [PostgresIndexConfig.parse_model_node(index) for index in indexes], @@ -216,13 +209,14 @@ def parse_describe_relation_results( Returns: a dict representation of an instance of this class that can be passed into `from_dict()` """ - materialized_view: agate.Row = describe_relation_results["materialized_view"].rows[0] + materialized_view_config: agate.Table = describe_relation_results.get("materialized_view") + materialized_view = materialized_view_config.rows[0] indexes: agate.Table = describe_relation_results["indexes"] config_dict = { - "table_name": materialized_view["table_name"], + "name": materialized_view["matviewname"], "schema": PostgresSchemaConfig.parse_describe_relation_results(materialized_view), - "query": materialized_view["query"].strip(), + "query": materialized_view["definition"].strip(), "indexes": [ PostgresIndexConfig.parse_describe_relation_results(index) for index in indexes.rows @@ -230,6 +224,18 @@ def parse_describe_relation_results( } return config_dict + def generate_index_name(self, index_fully_qualified_path) -> str: + return dbt.utils.md5( + "_".join( + { + self.database_name, + self.schema_name, + self.name, + index_fully_qualified_path, + } + ) + ) + @dataclass class PostgresMaterializedViewConfigChangeset(RelationConfigChangeset): diff --git a/plugins/postgres/dbt/adapters/postgres/relation_configs/policies.py b/plugins/postgres/dbt/adapters/postgres/relation_configs/policies.py deleted file mode 100644 index 2b7b3f78d7c..00000000000 --- a/plugins/postgres/dbt/adapters/postgres/relation_configs/policies.py +++ /dev/null @@ -1,20 +0,0 @@ -from dataclasses import dataclass - -from dbt.adapters.base.relation import Policy - - -QUOTE_CHAR = '"' -MAX_CHARACTERS_IN_IDENTIFIER = 63 - - -class PostgresIncludePolicy(Policy): - database: bool = True - schema: bool = True - identifier: bool = True - - -@dataclass -class PostgresQuotePolicy(Policy): - database: bool = True - schema: bool = True - identifier: bool = True diff --git a/plugins/postgres/dbt/adapters/postgres/relation_configs/policy.py b/plugins/postgres/dbt/adapters/postgres/relation_configs/policy.py new file mode 100644 index 00000000000..30eada2a42a --- /dev/null +++ b/plugins/postgres/dbt/adapters/postgres/relation_configs/policy.py @@ -0,0 +1,48 @@ +from dataclasses import dataclass +from functools import partial + +from dbt.adapters.relation_configs import ( + IncludePolicy, + QuotePolicy, + conform_part, + render_part, + render, +) + + +MAX_CHARACTERS_IN_IDENTIFIER = 63 + + +class PostgresIncludePolicy(IncludePolicy): + database: bool = True + schema: bool = True + identifier: bool = True + + +@dataclass +class PostgresQuotePolicy(QuotePolicy): + database: bool = True + schema: bool = True + identifier: bool = True + + @property + def quote_character(self) -> str: + """This is property to appeal to the `Policy` serialization.""" + return '"' + + +postgres_conform_part = partial( + conform_part, + quote_policy=PostgresQuotePolicy(), +) +postgres_render_part = partial( + render_part, + quote_policy=PostgresQuotePolicy(), + include_policy=PostgresIncludePolicy(), +) +postgres_render = partial( + render, + quote_policy=PostgresQuotePolicy(), + include_policy=PostgresIncludePolicy(), + delimiter=".", +) diff --git a/plugins/postgres/dbt/adapters/postgres/relation_configs/schema.py b/plugins/postgres/dbt/adapters/postgres/relation_configs/schema.py index 349772a745c..093c301fff4 100644 --- a/plugins/postgres/dbt/adapters/postgres/relation_configs/schema.py +++ b/plugins/postgres/dbt/adapters/postgres/relation_configs/schema.py @@ -1,8 +1,10 @@ +from collections import OrderedDict from dataclasses import dataclass from typing import Set import agate from dbt.adapters.relation_configs import ( + SchemaConfig, RelationConfigValidationMixin, RelationConfigValidationRule, ) @@ -10,12 +12,12 @@ from dbt.contracts.relation import ComponentName from dbt.exceptions import DbtRuntimeError -from dbt.adapters.postgres.relation_configs.base import PostgresRelationConfigBase from dbt.adapters.postgres.relation_configs.database import PostgresDatabaseConfig +from dbt.adapters.postgres.relation_configs.policy import postgres_render, postgres_conform_part @dataclass(frozen=True, eq=True, unsafe_hash=True) -class PostgresSchemaConfig(PostgresRelationConfigBase, RelationConfigValidationMixin): +class PostgresSchemaConfig(SchemaConfig, RelationConfigValidationMixin): """ This config follow the specs found here: https://www.postgresql.org/docs/15/sql-createschema.html @@ -25,46 +27,41 @@ class PostgresSchemaConfig(PostgresRelationConfigBase, RelationConfigValidationM - database_name: name of the database """ - schema_name: str + name: str database: PostgresDatabaseConfig - @property - def database_name(self) -> str: - return self.database.name - @property def fully_qualified_path(self) -> str: - return ".".join( - part - for part in [self.database.fully_qualified_path, self.schema_name] - if part is not None + return postgres_render( + OrderedDict( + {ComponentName.Database: self.database_name, ComponentName.Schema: self.name} + ) ) @property def validation_rules(self) -> Set[RelationConfigValidationRule]: return { RelationConfigValidationRule( - validation_check=len(self.schema_name or "") > 0, + validation_check=len(self.name or "") > 0, validation_error=DbtRuntimeError( - f"dbt-postgres requires a name for a schema, received: {self.schema_name}" + f"dbt-postgres requires a name for a schema, received: {self.name}" ), ) } @classmethod - def from_dict(cls, config_dict: dict) -> "PostgresSchemaConfig": + def from_dict(cls, config_dict: dict) -> "SchemaConfig": kwargs_dict = { - "schema_name": cls._render_part(ComponentName.Schema, config_dict["schema_name"]), + "name": postgres_conform_part(ComponentName.Schema, config_dict["name"]), "database": PostgresDatabaseConfig.from_dict(config_dict["database"]), } - - schema: "PostgresSchemaConfig" = super().from_dict(kwargs_dict) # type: ignore + schema: "SchemaConfig" = super().from_dict(kwargs_dict) # type: ignore return schema @classmethod def parse_model_node(cls, model_node: ModelNode) -> dict: config_dict = { - "schema_name": model_node.schema, + "name": model_node.schema, "database": PostgresDatabaseConfig.parse_model_node(model_node), } return config_dict @@ -72,7 +69,7 @@ def parse_model_node(cls, model_node: ModelNode) -> dict: @classmethod def parse_describe_relation_results(cls, describe_relation_results: agate.Row) -> dict: config_dict = { - "schema_name": describe_relation_results["schema_name"], + "name": describe_relation_results["schemaname"], "database": PostgresDatabaseConfig.parse_describe_relation_results( describe_relation_results ), diff --git a/plugins/postgres/dbt/include/postgres/macros/index.sql b/plugins/postgres/dbt/include/postgres/macros/index.sql new file mode 100644 index 00000000000..dcfeff400b6 --- /dev/null +++ b/plugins/postgres/dbt/include/postgres/macros/index.sql @@ -0,0 +1,71 @@ +{#- /* + This file contains DDL that gets consumed in the Postgres implementation of the materialized view materialization + in `dbt/include/postgres/macros/materializations/materialized_view.sql`. + These macros could be used elsewhere as they do not care that they are being called by a materialization; + but the original intention was to support the materialization of materialized views. These macros represent + the basic interactions dbt-postgres requires of indexes in Postgres: + - ALTER + - CREATE + - DROP + These macros all take a PostgresIndexConfig instance and/or a MaterializationConfigBase as an input. + These classes can be found in the following files, respectively: + `dbt/adapters/postgres/relation_configs/index.py` + `dbt/adapters/relation_configs/materialization.py` + + Used in: + `dbt/include/postgres/macros/materializations/materialized_view.sql` + Uses: + `dbt/adapters/postgres/relation_configs/index.py` + `dbt/adapters/postgres/relation_configs/materialized_view.py` +*/ -#} + + +{% macro postgres__alter_indexes_sql(materialization_config, index_changeset) -%} + {{- log('Applying UPDATE INDEXES to: ' ~ materialization_config.fully_qualified_path) -}} + + {%- for _index_change in index_changeset -%} + {%- set _index_config = _index_change.context -%} + + {%- if _index_change.action == "drop" -%} + {{ postgres__drop_index_sql(_index_config) }}; + + {%- elif _index_change.action == "create" -%} + {{ postgres__create_index_sql(materialization_config, _index_config) }}; + + {%- endif -%} + {%- endfor -%} + +{%- endmacro %} + + +{% macro postgres__create_indexes_sql(materialization_config, on_intermediate=False) -%} + + {% for _index_config in materialization_config.indexes -%} + {{- postgres__create_index_sql(materialization_config, _index_config, on_intermediate) -}}; + {%- endfor -%} + +{%- endmacro %} + + +{% macro postgres__create_index_sql(materialization_config, index_config, on_intermediate=False) -%} + {%- if on_intermediate -%} + {%- set _materialization_path = materialization_config.fully_qualified_path_intermediate -%} + {%- else -%} + {%- set _materialization_path = materialization_config.fully_qualified_path -%} + {%- endif -%} + + {%- set _index_name = adapter.Relation.generate_index_name(materialization_config, index_config) -%} + + create {% if index_config.unique -%}unique{%- endif %} index if not exists "{{ _index_name }}" + on {{ _materialization_path }} + using {{ index_config.method }} + ( + {{ ", ".join(index_config.column_names) }} + ) + +{%- endmacro %} + + +{% macro postgres__drop_index_sql(index_config) -%} + drop index if exists "{{ index_config.name }}" cascade; +{%- endmacro %} diff --git a/plugins/postgres/dbt/include/postgres/macros/materializations/materialized_view.sql b/plugins/postgres/dbt/include/postgres/macros/materializations/materialized_view.sql index da0551c8f01..716f5d9119a 100644 --- a/plugins/postgres/dbt/include/postgres/macros/materializations/materialized_view.sql +++ b/plugins/postgres/dbt/include/postgres/macros/materializations/materialized_view.sql @@ -28,36 +28,21 @@ in the changeset requires a full refresh or if an unmonitored change was detected) or if we can get away with altering the dynamic table in place. */ -#} - {% set config_changeset = adapter.Relation.materialized_view_config_changeset(new_materialized_view, existing_materialized_view) %} - - {% if configuration_changes.requires_full_refresh %} + {% set _config_changeset = adapter.Relation.materialized_view_config_changeset(new_materialized_view, existing_materialized_view) %} + {% if _config_changeset.requires_full_refresh %} {{ replace_materialized_view_sql(new_materialized_view) }} {% else %} --- TODO: make sure this works with old/new - {%- for _index_change in configuration_changes.index_changes -%} - {%- set _index = _index_change.context -%} - - {%- if _index_change.action == "drop" -%} - - {{ postgres__get_drop_index_sql(relation, _index.name) }}; - - {%- elif _index_change.action == "create" -%} - - {{ postgres__get_create_index_sql(relation, _index.as_node_config) }} - - {%- endif -%} - - {%- endfor -%} + {{ postgres__alter_indexes_sql(new_materialized_view, _config_changeset.indexes) }} {%- endif -%} {% endmacro %} -{% macro postgres__create_materialized_view_sql(materialized_view, intermediate=False) %} - {%- if intermediate -%} +{% macro postgres__create_materialized_view_sql(materialized_view, as_intermediate=False) %} + {%- if as_intermediate -%} {%- set materialized_view_path = materialized_view.fully_qualified_path_intermediate -%} {%- else -%} {%- set materialized_view_path = materialized_view.fully_qualified_path -%} @@ -66,10 +51,8 @@ create materialized view {{ materialized_view_path }} as {{ materialized_view.query }} ; ---TODO: replace this - {% for _index_dict in config.get('indexes', []) -%} - {{- get_create_index_sql(relation, _index_dict) -}} - {%- endfor -%} + + {{- postgres__create_indexes_sql(materialized_view, on_intermediate) -}} {% endmacro %} @@ -78,16 +61,13 @@ {% set _materialized_view_sql -%} select - t.tablename, - t.schemaname, - {{ this.database }} as databasename, + v.matviewname, + v.schemaname, + '{{ this.database }}' as databasename, v.definition - from pg_tables t - join pg_views v - on v.viewname = t.tablename - and v.schemaname = t.schemaname - where t.tablename ilike '{{ materialized_view.table_name }}' - and t.schemaname ilike '{{ materialized_view.schema_name }}' + from pg_matviews v + where v.matviewname ilike '{{ materialized_view.name }}' + and v.schemaname ilike '{{ materialized_view.schema_name }}' {%- endset -%} {% set _materialized_view = run_query(_materialized_view_sql) %} @@ -109,7 +89,7 @@ join pg_attribute a on a.attrelid = t.oid and a.attnum = ANY(ix.indkey) - where t.relname ilike '{{ materialized_view.table_name }}' + where t.relname ilike '{{ materialized_view.name }}' and n.nspname ilike '{{ materialized_view.schema_name }}' and t.relkind = 'm' group by 1, 2, 3 @@ -118,6 +98,7 @@ {% set _indexes = run_query(_indexes_sql) %} {% do return({'materialized_view': _materialized_view, 'indexes': _indexes}) %} + {% endmacro %} @@ -134,24 +115,24 @@ {% macro postgres__refresh_materialized_view_sql(materialized_view) %} - refresh materialized view {{ materialized_view.fully_qualified_path }}; + refresh materialized view {{ materialized_view.fully_qualified_path }} {% endmacro %} -{% macro postgres__rename_materialized_view_sql(materialized_view, name, intermediate=False) %} - {%- if intermediate -%} +{% macro postgres__rename_materialized_view_sql(materialized_view, name, from_intermediate=False) %} + {%- if from_intermediate -%} {%- set materialized_view_path = materialized_view.fully_qualified_path_intermediate -%} {%- else -%} {%- set materialized_view_path = materialized_view.fully_qualified_path -%} {%- endif -%} - alter materialized view {{ materialized_view_path }} rename to {{ name }}; + alter materialized view {{ materialized_view_path }} rename to {{ name }} {% endmacro %} {% macro postgres__replace_materialized_view_sql(new_materialized_view, existing_relation) %} - {{- create_materialized_view_sql(new_materialized_view, True) -}} + {{- create_materialized_view_sql(new_materialized_view, as_intermediate=True) -}} --TODO: this only works for existing materialized views - {{ rename_materialized_view(new_materialized_view, new_materialized_view.backup_name) }}; - {{ rename_materialized_view(new_materialized_view, new_materialized_view.name, True) }} + {{ rename_materialized_view_sql(new_materialized_view, new_materialized_view.backup_name) }}; + {{ rename_materialized_view_sql(new_materialized_view, new_materialized_view.name, from_intermediate=True) }} {% endmacro %} diff --git a/tests/adapter/dbt/tests/adapter/materialized_view/on_configuration_change.py b/tests/adapter/dbt/tests/adapter/materialized_view/on_configuration_change.py index f77d9aade03..5f91d06fd75 100644 --- a/tests/adapter/dbt/tests/adapter/materialized_view/on_configuration_change.py +++ b/tests/adapter/dbt/tests/adapter/materialized_view/on_configuration_change.py @@ -69,13 +69,6 @@ def refresh_message(self, project): def replace_message(self, project): return f"Applying REPLACE to: {relation_from_name(project.adapter, self.base_materialized_view)}" - @pytest.fixture(scope="function") - def configuration_change_message(self, project): - return ( - f"Determining configuration changes on: " - f"{relation_from_name(project.adapter, self.base_materialized_view)}" - ) - @pytest.fixture(scope="function") def configuration_change_continue_message(self, project): return ( diff --git a/tests/functional/materializations/materialized_view_tests/test_materialized_view.py b/tests/functional/materializations/materialized_view_tests/test_materialized_view.py index 733329b42ff..b332c0fc57b 100644 --- a/tests/functional/materializations/materialized_view_tests/test_materialized_view.py +++ b/tests/functional/materializations/materialized_view_tests/test_materialized_view.py @@ -73,7 +73,7 @@ class TestOnConfigurationChangeApply(PostgresOnConfigurationChangeBase): # this is part of the test def test_full_refresh_takes_precedence_over_any_configuration_changes( - self, configuration_changes, replace_message, configuration_change_message + self, configuration_changes, replace_message ): results, logs = run_model("base_materialized_view", full_refresh=True) assert_proper_scenario( @@ -82,19 +82,16 @@ def test_full_refresh_takes_precedence_over_any_configuration_changes( logs, RunStatus.Success, messages_in_logs=[replace_message], - messages_not_in_logs=[configuration_change_message], ) - def test_model_is_refreshed_with_no_configuration_changes( - self, refresh_message, configuration_change_message - ): + def test_model_is_refreshed_with_no_configuration_changes(self, refresh_message): results, logs = run_model("base_materialized_view") assert_proper_scenario( OnConfigurationChangeOption.Apply, results, logs, RunStatus.Success, - messages_in_logs=[refresh_message, configuration_change_message], + messages_in_logs=[refresh_message], ) def test_model_applies_changes_with_configuration_changes( @@ -116,7 +113,7 @@ def project_config_update(self): return {"models": {"on_configuration_change": OnConfigurationChangeOption.Continue.value}} def test_full_refresh_takes_precedence_over_any_configuration_changes( - self, configuration_changes, replace_message, configuration_change_message + self, configuration_changes, replace_message ): results, logs = run_model("base_materialized_view", full_refresh=True) assert_proper_scenario( @@ -125,19 +122,16 @@ def test_full_refresh_takes_precedence_over_any_configuration_changes( logs, RunStatus.Success, messages_in_logs=[replace_message], - messages_not_in_logs=[configuration_change_message], ) - def test_model_is_refreshed_with_no_configuration_changes( - self, refresh_message, configuration_change_message - ): + def test_model_is_refreshed_with_no_configuration_changes(self, refresh_message): results, logs = run_model("base_materialized_view") assert_proper_scenario( OnConfigurationChangeOption.Continue, results, logs, RunStatus.Success, - messages_in_logs=[refresh_message, configuration_change_message], + messages_in_logs=[refresh_message], ) def test_model_is_not_refreshed_with_configuration_changes( @@ -160,7 +154,7 @@ def project_config_update(self): return {"models": {"on_configuration_change": OnConfigurationChangeOption.Fail.value}} def test_full_refresh_takes_precedence_over_any_configuration_changes( - self, configuration_changes, replace_message, configuration_change_message + self, configuration_changes, replace_message ): results, logs = run_model("base_materialized_view", full_refresh=True) assert_proper_scenario( @@ -169,19 +163,16 @@ def test_full_refresh_takes_precedence_over_any_configuration_changes( logs, RunStatus.Success, messages_in_logs=[replace_message], - messages_not_in_logs=[configuration_change_message], ) - def test_model_is_refreshed_with_no_configuration_changes( - self, refresh_message, configuration_change_message - ): + def test_model_is_refreshed_with_no_configuration_changes(self, refresh_message): results, logs = run_model("base_materialized_view") assert_proper_scenario( OnConfigurationChangeOption.Fail, results, logs, RunStatus.Success, - messages_in_logs=[refresh_message, configuration_change_message], + messages_in_logs=[refresh_message], ) def test_run_fails_with_configuration_changes( From 6b4fbec2d626a23d61a549fc765d16cae86b2006 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 29 Jun 2023 03:14:42 -0400 Subject: [PATCH 04/28] added updates from snowflake to core, pushed down to postgres, final draft --- core/dbt/adapters/base/impl.py | 252 +++++++++++++++--- core/dbt/adapters/base/relation.py | 63 ----- .../relation_configs/_materialization.py | 66 +++-- .../macros/adapters/relation.sql | 30 --- .../materializations/materialization.sql | 58 ++-- .../materializations/models/all/alter.sql | 20 ++ .../materializations/models/all/create.sql | 20 ++ .../materializations/models/all/describe.sql | 24 ++ .../materializations/models/all/drop.sql | 24 ++ .../materializations/models/all/rename.sql | 33 +++ .../materializations/models/all/replace.sql | 16 ++ .../models/materialized_view/alter.sql | 13 +- .../models/materialized_view/create.sql | 13 +- .../models/materialized_view/describe.sql | 15 +- .../models/materialized_view/drop.sql | 29 +- .../materialized_view/materialized_view.sql | 85 +++--- .../models/materialized_view/refresh.sql | 19 +- .../models/materialized_view/rename.sql | 13 +- .../models/materialized_view/replace.sql | 9 - .../materializations/models/table/rename.sql | 8 + .../materializations/models/view/rename.sql | 8 + .../postgres/dbt/adapters/postgres/impl.py | 110 +++++++- .../dbt/adapters/postgres/relation.py | 110 +------- .../relation_configs/materialized_view.py | 39 +-- .../dbt/include/postgres/macros/index.sql | 19 +- .../materializations/materialized_view.sql | 68 ++--- .../macros/materializations/replace.sql | 25 ++ .../macros/materializations/table.sql | 8 + .../postgres/macros/materializations/view.sql | 8 + 29 files changed, 716 insertions(+), 489 deletions(-) create mode 100644 core/dbt/include/global_project/macros/materializations/models/all/alter.sql create mode 100644 core/dbt/include/global_project/macros/materializations/models/all/create.sql create mode 100644 core/dbt/include/global_project/macros/materializations/models/all/describe.sql create mode 100644 core/dbt/include/global_project/macros/materializations/models/all/drop.sql create mode 100644 core/dbt/include/global_project/macros/materializations/models/all/rename.sql create mode 100644 core/dbt/include/global_project/macros/materializations/models/all/replace.sql delete mode 100644 core/dbt/include/global_project/macros/materializations/models/materialized_view/replace.sql create mode 100644 core/dbt/include/global_project/macros/materializations/models/table/rename.sql create mode 100644 core/dbt/include/global_project/macros/materializations/models/view/rename.sql create mode 100644 plugins/postgres/dbt/include/postgres/macros/materializations/replace.sql create mode 100644 plugins/postgres/dbt/include/postgres/macros/materializations/table.sql create mode 100644 plugins/postgres/dbt/include/postgres/macros/materializations/view.sql diff --git a/core/dbt/adapters/base/impl.py b/core/dbt/adapters/base/impl.py index 1fa2ce903f6..400446c9887 100644 --- a/core/dbt/adapters/base/impl.py +++ b/core/dbt/adapters/base/impl.py @@ -1,10 +1,11 @@ import abc from concurrent.futures import as_completed, Future from contextlib import contextmanager +import dataclasses from datetime import datetime from enum import Enum -import time from itertools import chain +import time from typing import ( Any, Callable, @@ -20,11 +21,47 @@ Union, ) -from dbt.contracts.graph.nodes import ColumnLevelConstraint, ConstraintType, ModelLevelConstraint - import agate import pytz +from dbt import deprecations +from dbt.adapters.base import Credentials, Column as BaseColumn +from dbt.adapters.base.connections import AdapterResponse, Connection +from dbt.adapters.base.meta import AdapterMeta, available +from dbt.adapters.base.relation import ( + ComponentName, + BaseRelation, + InformationSchema, + SchemaSearchMap, +) +from dbt.adapters.cache import RelationsCache, _make_ref_key_dict +from dbt.adapters.protocol import AdapterConfig, ConnectionManagerProtocol +from dbt.adapters.relation_configs import ( + DescribeRelationResults, + MaterializationConfig, + RelationConfigChangeAction, +) +from dbt.clients.agate_helper import empty_table, merge_tables, table_from_rows +from dbt.clients.jinja import MacroGenerator +from dbt.contracts.graph.manifest import MacroManifest, Manifest +from dbt.contracts.graph.nodes import ( + ColumnLevelConstraint, + ConstraintType, + ModelLevelConstraint, + ModelNode, + ResultNode, +) +from dbt.contracts.relation import RelationType +from dbt.events.functions import fire_event, warn_or_error +from dbt.events.types import ( + CacheMiss, + CatalogGenerationError, + CodeExecution, + CodeExecutionStatus, + ConstraintNotEnforced, + ConstraintNotSupported, + ListRelations, +) from dbt.exceptions import ( DbtInternalError, DbtRuntimeError, @@ -42,36 +79,8 @@ UnexpectedNonTimestampError, UnexpectedNullError, ) +from dbt.utils import AttrDict, cast_to_str, filter_null_values, executor -from dbt.adapters.protocol import AdapterConfig, ConnectionManagerProtocol -from dbt.clients.agate_helper import empty_table, merge_tables, table_from_rows -from dbt.clients.jinja import MacroGenerator -from dbt.contracts.graph.manifest import Manifest, MacroManifest -from dbt.contracts.graph.nodes import ResultNode -from dbt.events.functions import fire_event, warn_or_error -from dbt.events.types import ( - CacheMiss, - ListRelations, - CodeExecution, - CodeExecutionStatus, - CatalogGenerationError, - ConstraintNotSupported, - ConstraintNotEnforced, -) -from dbt.utils import filter_null_values, executor, cast_to_str, AttrDict - -from dbt.adapters.base.connections import Connection, AdapterResponse -from dbt.adapters.base.meta import AdapterMeta, available -from dbt.adapters.base.relation import ( - ComponentName, - BaseRelation, - InformationSchema, - SchemaSearchMap, -) -from dbt.adapters.base import Column as BaseColumn -from dbt.adapters.base import Credentials -from dbt.adapters.cache import RelationsCache, _make_ref_key_dict -from dbt import deprecations GET_CATALOG_MACRO_NAME = "get_catalog" FRESHNESS_MACRO_NAME = "collect_freshness" @@ -222,6 +231,18 @@ class BaseAdapter(metaclass=AdapterMeta): ConstraintType.foreign_key: ConstraintSupport.ENFORCED, } + materialization_configs: Dict[RelationType, MaterializationConfig] = {} + + @available + @classmethod + def relation_type(cls) -> Type[RelationType]: + return RelationType + + @available + @classmethod + def relation_config_change_action(cls) -> Type[RelationConfigChangeAction]: + return RelationConfigChangeAction + def __init__(self, config): self.config = config self.cache = RelationsCache() @@ -1420,6 +1441,173 @@ def render_model_constraint(cls, constraint: ModelLevelConstraint) -> Optional[s else: return None + @available + @classmethod + def is_base_relation(cls, materialization: Union[BaseRelation, MaterializationConfig]) -> bool: + return isinstance(materialization, BaseRelation) + + @available + @classmethod + def is_materialization_config( + cls, materialization: Union[BaseRelation, MaterializationConfig] + ) -> bool: + return isinstance(materialization, MaterializationConfig) + + @available + def base_relation_from_materialization_config( + self, materialization_config: MaterializationConfig + ) -> BaseRelation: + """ + Produce a `BaseRelation` instance from a `MaterializationConfig` instance. This is primarily done to + reuse existing functionality based on `BaseRelation` while working with `MaterializationConfig` instances. + + Args: + materialization_config: a `MaterializationConfig` to be converted + + Returns: + a converted `BaseRelation` instance + """ + relation = self.Relation.create( + database=materialization_config.database_name, + schema=materialization_config.schema_name, + identifier=materialization_config.name, + quote_policy=self.config.quoting, + type=materialization_config.type, + ) + return relation + + @available + @classmethod + def make_backup_materialization_config( + cls, materialization_config: MaterializationConfig + ) -> MaterializationConfig: + """ + Return a copy of the materialization config, but with a backup name instead of the original name. + + Args: + materialization_config: the materialization that needs a backup + + Returns: + a renamed copy of the materialization config + """ + return dataclasses.replace( + materialization_config, name=cls.backup_name(materialization_config) + ) + + @available + @classmethod + def make_intermediate_materialization_config( + cls, materialization_config: MaterializationConfig + ) -> MaterializationConfig: + """ + Return a copy of the materialization config, but with a backup name instead of the original name. + + Args: + materialization_config: the materialization that needs a backup + + Returns: + a renamed copy of the materialization config + """ + return dataclasses.replace( + materialization_config, name=cls.intermediate_name(materialization_config) + ) + + @available + def get_cached_relation_from_materialization_config( + self, materialization_config: MaterializationConfig + ) -> BaseRelation: + return self.get_relation( + database=materialization_config.database_name, + schema=materialization_config.schema_name, + identifier=materialization_config.name, + ) + + @available + def get_cached_backup_relation_from_materialization_config( + self, materialization_config: MaterializationConfig + ) -> BaseRelation: + return self.get_relation( + database=materialization_config.database_name, + schema=materialization_config.schema_name, + identifier=self.backup_name(materialization_config), + ) + + @available + def get_cached_intermediate_relation_from_materialization_config( + self, materialization_config: MaterializationConfig + ) -> BaseRelation: + return self.get_relation( + database=materialization_config.database_name, + schema=materialization_config.schema_name, + identifier=self.intermediate_name(materialization_config), + ) + + @available + @classmethod + def backup_name(cls, materialization_config: MaterializationConfig) -> str: + return f"{materialization_config.name}__dbt_backup" + + @available + @classmethod + def intermediate_name(cls, materialization_config: MaterializationConfig) -> str: + return f"{materialization_config.name}__dbt_tmp" + + @available + @classmethod + def materialization_config_from_model_node( + cls, model_node: ModelNode + ) -> MaterializationConfig: + """ + Produce a validated materialization config from the config available in the global jinja context. + + The intention is to remove validation from the jinja context and put it in python. This method gets + called in a jinja template and it's results are used in the jinja template. For an example, please + refer to `dbt/include/global_project/macros/materializations/models/materialized_view/materialization.sql`. + In this file, the relation config is retrieved right away, to ensure that the config is validated before + any sql is executed against the database. + + Args: + model_node: the `model` ModelNode instance that's in the global jinja context + + Returns: a validated adapter-specific, relation_type-specific MaterializationConfig instance + """ + relation_type = RelationType(model_node.config.materialized) + + if materialization_config := cls.materialization_configs.get(relation_type): + return materialization_config.from_model_node(model_node) + + raise DbtRuntimeError( + f"materialization_config_from_model_node() is not supported" + f" for the provided relation type: {relation_type}" + ) + + @available + @classmethod + def materialization_config_from_describe_relation_results( + cls, describe_relation_results: DescribeRelationResults, relation_type: RelationType + ) -> MaterializationConfig: + """ + Produce a validated materialization config from a series of "describe "-type queries. + + The intention is to remove validation from the jinja context and put it in python. This method gets + called in a jinja template and it's results are used in the jinja template. For an example, please + refer to `dbt/include/global_project/macros/materializations/models/materialized_view/materialization.sql`. + + Args: + describe_relation_results: the results of one or more queries run against the database + to describe this relation + relation_type: the type of relation associated with the relation results + + Returns: a validated adapter-specific, relation_type-specific MaterializationConfig instance + """ + if materialization_config := cls.materialization_configs.get(relation_type): + return materialization_config.from_describe_relation_results(describe_relation_results) + + raise DbtRuntimeError( + f"materialization_config_from_describe_relation_results() is not" + f" supported for the provided relation type: {relation_type}" + ) + COLUMNS_EQUAL_SQL = """ with diff_count as ( diff --git a/core/dbt/adapters/base/relation.py b/core/dbt/adapters/base/relation.py index f827fefc965..b7fc331f448 100644 --- a/core/dbt/adapters/base/relation.py +++ b/core/dbt/adapters/base/relation.py @@ -2,16 +2,11 @@ from dataclasses import dataclass, field from typing import Any, Dict, Iterator, Optional, Set, Tuple, Type, TypeVar -from dbt.adapters.relation_configs import ( - RelationConfig, - DescribeRelationResults, -) from dbt.contracts.graph.nodes import ( SourceDefinition, ManifestNode, ResultNode, ParsedNode, - ModelNode, ) from dbt.contracts.relation import ( ComponentName, @@ -47,10 +42,6 @@ class BaseRelation(FakeAPIObject, Hashable): dbt_created: bool = False # registers RelationConfigBases to RelationTypes - @classmethod - def relation_configs(cls) -> Dict[RelationType, RelationConfig]: - return {} - def _is_exactish_match(self, field: ComponentName, value: str) -> bool: if self.dbt_created and self.quote_policy.get_part(field) is False: return self.path.get_lowered_part(field) == value.lower() @@ -299,60 +290,6 @@ def create( ) return cls.from_dict(kwargs) - @classmethod - def from_model_node(cls, model_node: ModelNode) -> RelationConfig: - """ - Produce a validated relation config from the config available in the global jinja context. - - The intention is to remove validation from the jinja context and put it in python. This method gets - called in a jinja template and it's results are used in the jinja template. For an example, please - refer to `dbt/include/global_project/macros/materializations/models/materialized_view/materialization.sql`. - In this file, the relation config is retrieved right away, to ensure that the config is validated before - any sql is executed against the database. - - Args: - model_node: the `model` ModelNode instance that's in the global jinja context - - Returns: a validated adapter-specific, relation_type-specific RelationConfigBase instance - """ - relation_type: RelationType = cls.get_relation_type(model_node.config.materialized) - - if relation_config := cls.relation_configs().get(relation_type): - relation = relation_config.from_model_node(model_node) - else: - raise DbtRuntimeError( - f"from_runtime_config() is not supported for the provided relation type: {relation_type}" - ) - - return relation - - @classmethod - def from_describe_relation_results( - cls, describe_relation_results: DescribeRelationResults, relation_type: RelationType - ) -> RelationConfig: - """ - Produce a validated relation config from a series of "describe "-type queries. - - The intention is to remove validation from the jinja context and put it in python. This method gets - called in a jinja template and it's results are used in the jinja template. For an example, please - refer to `dbt/include/global_project/macros/materializations/models/materialized_view/materialization.sql`. - - Args: - describe_relation_results: the results of one or more queries run against the database - to describe this relation - relation_type: the type of relation associated with the relation results - - Returns: a validated adapter-specific, relation_type-specific RelationConfigBase instance - """ - if relation_config := cls.relation_configs().get(relation_type): - relation = relation_config.from_describe_relation_results(describe_relation_results) - else: - raise DbtRuntimeError( - f"from_relation_results() is not supported for the provided relation type: {relation_type}" - ) - - return relation - def __repr__(self) -> str: return "<{} {}>".format(self.__class__.__name__, self.render()) diff --git a/core/dbt/adapters/relation_configs/_materialization.py b/core/dbt/adapters/relation_configs/_materialization.py index 32f1a11e63c..75983b8392f 100644 --- a/core/dbt/adapters/relation_configs/_materialization.py +++ b/core/dbt/adapters/relation_configs/_materialization.py @@ -1,10 +1,11 @@ from abc import ABC from dataclasses import dataclass -from dbt.contracts.relation import RelationType -from dbt.adapters.relation_configs._base import RelationConfig +from dbt.adapters.relation_configs._base import DescribeRelationResults, RelationConfig from dbt.adapters.relation_configs._database import DatabaseConfig from dbt.adapters.relation_configs._schema import SchemaConfig +from dbt.contracts.graph.nodes import ModelNode +from dbt.contracts.relation import RelationType @dataclass(frozen=True) @@ -17,7 +18,7 @@ class MaterializationConfig(RelationConfig, ABC): name: str schema: "SchemaConfig" query: str - relation_type: RelationType + type: RelationType @property def schema_name(self) -> str: @@ -31,26 +32,6 @@ def database(self) -> "DatabaseConfig": def database_name(self) -> str: return self.database.name - @property - def backup_name(self) -> str: - """ - Used for hot-swapping during replacement - - Returns: - a name unique to this materialized view - """ - return f"{self.name}__dbt_backup" - - @property - def intermediate_name(self) -> str: - """ - Used for hot-swapping during replacement - - Returns: - a name unique to this materialized view - """ - return f"{self.name}__dbt_tmp" - @property def fully_qualified_path(self) -> str: """ @@ -60,20 +41,37 @@ def fully_qualified_path(self) -> str: """ return f"{self.schema.fully_qualified_path}.{self.name}" - @property - def fully_qualified_path_backup(self) -> str: + @classmethod + def from_dict(cls, kwargs_dict) -> "MaterializationConfig": """ - This is sufficient if there is no quote policy or include policy, otherwise override it to apply those policies. - - Returns: a fully qualified path, run through the quote and include policies, for rendering in a template + Supports type annotations """ - return f"{self.schema.fully_qualified_path}.{self.backup_name}" + config = super().from_dict(kwargs_dict) + assert isinstance(config, MaterializationConfig) + return config - @property - def fully_qualified_path_intermediate(self) -> str: + @classmethod + def from_model_node(cls, model_node: ModelNode) -> "MaterializationConfig": """ - This is sufficient if there is no quote policy or include policy, otherwise override it to apply those policies. + Supports type annotations + """ + config = super().from_model_node(model_node) + assert isinstance(config, MaterializationConfig) + return config + + @classmethod + def from_describe_relation_results( + cls, describe_relation_results: DescribeRelationResults + ) -> "MaterializationConfig": + """ + Supports type annotations + """ + config = super().from_describe_relation_results(describe_relation_results) + assert isinstance(config, MaterializationConfig) + return config - Returns: a fully qualified path, run through the quote and include policies, for rendering in a template + def __str__(self): + """ + Useful for template rendering and aligns with BaseRelation so that they are interchangeable """ - return f"{self.schema.fully_qualified_path}.{self.intermediate_name}" + return self.fully_qualified_path diff --git a/core/dbt/include/global_project/macros/adapters/relation.sql b/core/dbt/include/global_project/macros/adapters/relation.sql index 0c16ee6e33a..f0dde7f20f0 100644 --- a/core/dbt/include/global_project/macros/adapters/relation.sql +++ b/core/dbt/include/global_project/macros/adapters/relation.sql @@ -96,33 +96,3 @@ {{ adapter.drop_relation(relation) }} {% endif %} {% endmacro %} - - --- a user-friendly interface into adapter.get_relation for relation_configs.MaterializationConfigBase instances -{% macro load_cached_relation_config(relation_config) %} - {% do return(adapter.get_relation( - database=relation_config.database_name, - schema=relation_config.schema_name, - identifier=relation_config.name - )) -%} -{% endmacro %} - - --- a user-friendly interface into adapter.get_relation for relation_configs.MaterializationConfigBase instances -{% macro load_cached_relation_config_backup(relation_config) %} - {% do return(adapter.get_relation( - database=relation_config.database_name, - schema=relation_config.schema_name, - identifier=relation_config.backup_name - )) -%} -{% endmacro %} - - --- a user-friendly interface into adapter.get_relation for relation_configs.MaterializationConfigBase instances -{% macro load_cached_relation_config_intermediate(relation_config) %} - {% do return(adapter.get_relation( - database=relation_config.database_name, - schema=relation_config.schema_name, - identifier=relation_config.intermediate_name - )) -%} -{% endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/materialization.sql b/core/dbt/include/global_project/macros/materializations/materialization.sql index 23c3446202d..ac60b92c563 100644 --- a/core/dbt/include/global_project/macros/materializations/materialization.sql +++ b/core/dbt/include/global_project/macros/materializations/materialization.sql @@ -1,5 +1,5 @@ {# /* - These are step macros for building out a materialization based on `MaterializationConfigBase`, which can be + These are step macros for building out a materialization based on `MaterializationConfig`, which can be found in `dbt/adapters/relation_configs/materialization.py` Note: You cannot start the macro name with "materialization" because then dbt thinks it's a materialization @@ -7,63 +7,63 @@ */ #} -{% macro mat_setup(materialization_config, pre_hooks) %} +{%- macro mat_setup(materialization_config, pre_hooks) -%} -- backup_relation and intermediate_relation should not already exist in the database -- it's possible these exist because of a previous run that exited unexpectedly - {% set backup_relation = load_cached_relation_config_backup(materialization_config) %} - {% set intermediate_relation = load_cached_relation_config_intermediate(materialization_config) %} + {%- set backup_relation = adapter.get_cached_backup_relation_from_materialization_config(materialization_config) -%} + {%- set intermediate_relation = adapter.get_cached_intermediate_relation_from_materialization_config(materialization_config) -%} -- drop the temp relations if they exist already in the database - {{ drop_relation_if_exists(backup_relation) }} - {{ drop_relation_if_exists(intermediate_relation) }} + {{- drop_relation_if_exists(backup_relation) -}} + {{- drop_relation_if_exists(intermediate_relation) -}} - {{ run_hooks(pre_hooks, inside_transaction=False) }} + {{- run_hooks(pre_hooks, inside_transaction=False) -}} -{% endmacro %} +{%- endmacro -%} -{% macro mat_teardown(materialization_config, post_hooks) %} +{%- macro mat_teardown(materialization_config, post_hooks) -%} -- backup_relation and intermediate_relation may exist if the materialized view was replaced - {% set backup_relation = load_cached_relation_config_backup(materialization_config) %} - {% set intermediate_relation = load_cached_relation_config_intermediate(materialization_config) %} + {%- set backup_relation = adapter.get_cached_backup_relation_from_materialization_config(materialization_config) -%} + {%- set intermediate_relation = adapter.get_cached_intermediate_relation_from_materialization_config(materialization_config) -%} -- drop the temp relations if they exist to leave the database clean for the next run - {{ drop_relation_if_exists(backup_relation) }} - {{ drop_relation_if_exists(intermediate_relation) }} + {{- drop_relation_if_exists(backup_relation) -}} + {{- drop_relation_if_exists(intermediate_relation) -}} - {{ run_hooks(post_hooks, inside_transaction=False) }} + {{- run_hooks(post_hooks, inside_transaction=False) -}} -{% endmacro %} +{%- endmacro -%} -{% macro mat_execute_no_op(materialization_config) %} - {% do store_raw_result( +{%- macro mat_execute_no_op(materialization_config) -%} + {%- do store_raw_result( name="main", message="skip " ~ materialization_config.fully_qualified_path, code="skip", rows_affected="-1" - ) %} -{% endmacro %} + ) -%} +{%- endmacro -%} -{% macro mat_execute_build_sql(materialization_config, build_sql, relation_exists, post_hooks) %} +{%- macro mat_execute_build_sql(materialization_config, existing_relation, build_sql, post_hooks) -%} -- `BEGIN` happens here: - {{ run_hooks(pre_hooks, inside_transaction=True) }} + {{- run_hooks(pre_hooks, inside_transaction=True) -}} - {% set grant_config = config.get('grants') %} + {%- set grant_config = config.get('grants') -%} - {% call statement(name="main") %} + {%- call statement(name="main") -%} {{ build_sql }} - {% endcall %} + {%- endcall -%} - {% set should_revoke = should_revoke(relation_exists, full_refresh_mode=True) %} - {% do apply_grants(materialization_config.fully_qualified_path, grant_config, should_revoke=should_revoke) %} + {%- set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) -%} + {%- do apply_grants(materialization_config.fully_qualified_path, grant_config, should_revoke=should_revoke) -%} - {{ run_hooks(post_hooks, inside_transaction=True) }} + {{- run_hooks(post_hooks, inside_transaction=True) -}} - {{ adapter.commit() }} + {{- adapter.commit() -}} -{% endmacro %} +{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/materializations/models/all/alter.sql b/core/dbt/include/global_project/macros/materializations/models/all/alter.sql new file mode 100644 index 00000000000..b3a24894602 --- /dev/null +++ b/core/dbt/include/global_project/macros/materializations/models/all/alter.sql @@ -0,0 +1,20 @@ +{%- macro alter_sql(existing_materialization, new_materialization) -%} + + {{- log('Applying ALTER to: ' ~ existing_materialization) -}} + + {{- adapter.dispatch('alter_sql', 'dbt')(existing_materialization, new_materialization) -}} + +{%- endmacro -%} + + +{%- macro default__alter_sql(existing_materialization, new_materialization) -%} + + {%- set relation_type = existing_materialization.type -%} + + {%- if relation_type == 'materialized_view' -%} + {{ alter_materialized_view_sql(existing_materialization, new_materialization) }} + {%- else -%} + {{- exceptions.raise_compiler_error("`alter_sql()` has not been implemented for the relation type" ~ relation_type ) -}} + {%- endif -%} + +{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/materializations/models/all/create.sql b/core/dbt/include/global_project/macros/materializations/models/all/create.sql new file mode 100644 index 00000000000..588e3cd9a9f --- /dev/null +++ b/core/dbt/include/global_project/macros/materializations/models/all/create.sql @@ -0,0 +1,20 @@ +{%- macro create_sql(materialization) -%} + + {{- log('Applying CREATE to: ' ~ materialization) -}} + + {{- adapter.dispatch('create_sql', 'dbt')(materialization) -}} + +{%- endmacro -%} + + +{%- macro default__create_sql(materialization) -%} + + {%- set relation_type = materialization.type -%} + + {%- if relation_type == 'materialized_view' -%} + {{ create_materialized_view_sql(materialization) }} + {%- else -%} + {{- exceptions.raise_compiler_error("`create_sql()` has not been implemented for the relation type" ~ relation_type ) -}} + {%- endif -%} + +{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/materializations/models/all/describe.sql b/core/dbt/include/global_project/macros/materializations/models/all/describe.sql new file mode 100644 index 00000000000..e40448f15b1 --- /dev/null +++ b/core/dbt/include/global_project/macros/materializations/models/all/describe.sql @@ -0,0 +1,24 @@ +{# /* + This needs to be a {% do return(...) %} because the macro returns a dictionary, not a template. +*/ #} + +{%- macro describe_sql(materialization) -%} + + {{- log('Applying DESCRIBE to: ' ~ materialization) -}} + + {%- do return(adapter.dispatch('describe_sql', 'dbt')(materialization)) -%} + +{%- endmacro -%} + + +{%- macro default__describe_sql(materialization) -%} + + {%- set relation_type = materialization.type -%} + + {%- if relation_type == 'materialized_view' -%} + {%- do return(describe_materialized_view_sql(materialization)) -%} + {%- else -%} + {{- exceptions.raise_compiler_error("`describe_sql()` has not been implemented for the relation type" ~ relation_type ) -}} + {%- endif -%} + +{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/materializations/models/all/drop.sql b/core/dbt/include/global_project/macros/materializations/models/all/drop.sql new file mode 100644 index 00000000000..4ed7b2f0ac3 --- /dev/null +++ b/core/dbt/include/global_project/macros/materializations/models/all/drop.sql @@ -0,0 +1,24 @@ +{%- macro drop_sql(materialization) -%} + + {{- log('Applying DROP to: ' ~ materialization) -}} + + {%- set relation = adapter.base_relation_from_materialization_config(materialization) -%} + + {{- adapter.dispatch('drop_sql', 'dbt')(materialization) -}} + + {{- adapter.cache_dropped(relation) -}} + +{%- endmacro -%} + + +{%- macro default__drop_sql(materialization) -%} + + {%- set relation_type = materialization.type -%} + + {%- if relation_type == 'materialized_view' -%} + {{ drop_materialized_view_sql(materialization) }} + {%- else -%} + {{- exceptions.raise_compiler_error("`drop_sql()` has not been implemented for the relation type" ~ relation_type ) -}} + {%- endif -%} + +{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/materializations/models/all/rename.sql b/core/dbt/include/global_project/macros/materializations/models/all/rename.sql new file mode 100644 index 00000000000..5d59e114c0c --- /dev/null +++ b/core/dbt/include/global_project/macros/materializations/models/all/rename.sql @@ -0,0 +1,33 @@ +{%- macro rename_sql(materialization, new_name) -%} + + {{- log('Applying RENAME to: ' ~ materialization) -}} + + {%- if adapter.is_materialization_config(materialization) -%} + {%- set from_relation = adapter.base_relation_from_materialization_config(materialization) -%} + {%- else -%} + {%- set from_relation = materialization -%} + {%- endif-%} + {%- set to_relation = from_relation.incorporate(path={"identifier": new_name}) -%} + + {{- adapter.dispatch('rename_sql', 'dbt')(materialization, new_name) -}} + + {{- adapter.cache_renamed(from_relation, to_relation) -}} + +{%- endmacro -%} + + +{%- macro default__rename_sql(materialization, new_name) -%} + + {%- set relation_type = materialization.type -%} + + {%- if relation_type == 'view' -%} + {{ rename_view_sql(materialization, new_name) }} + {%- elif relation_type == 'table' -%} + {{ rename_table_sql(materialization, new_name) }} + {%- elif relation_type == 'materialized_view' -%} + {{ rename_materialized_view_sql(materialization, new_name) }} + {%- else -%} + {{- exceptions.raise_compiler_error("`rename_sql()` has not been implemented for the relation type" ~ relation_type ) -}} + {%- endif -%} + +{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/materializations/models/all/replace.sql b/core/dbt/include/global_project/macros/materializations/models/all/replace.sql new file mode 100644 index 00000000000..6603d345b8a --- /dev/null +++ b/core/dbt/include/global_project/macros/materializations/models/all/replace.sql @@ -0,0 +1,16 @@ +{# /* + This macro does dispatch to a relation_type-specific macro because it's actually a composite. +*/ #} + +{%- macro replace_sql(existing_materialization, new_materialization) -%} + + {{- log('Applying REPLACE to: ' ~ existing_materialization) -}} + + {{- adapter.dispatch('replace_sql', 'dbt')(existing_materialization, new_materialization) -}} + +{%- endmacro -%} + + +{%- macro default__replace_sql(existing_materialization, new_materialization) -%} + {{- exceptions.raise_compiler_error("`replace_sql` has not been implemented for this adapter.") -}} +{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/alter.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view/alter.sql index f7450314094..2eb7ff9e27e 100644 --- a/core/dbt/include/global_project/macros/materializations/models/materialized_view/alter.sql +++ b/core/dbt/include/global_project/macros/materializations/models/materialized_view/alter.sql @@ -1,9 +1,8 @@ -{% macro alter_materialized_view_sql(new_materialized_view, existing_materialized_view) %} - {{- log('Applying ALTER to: ' ~ new_materialized_view.fully_qualified_path) -}} - {{- adapter.dispatch('alter_materialized_view_sql', 'dbt')(new_materialized_view, existing_materialized_view) -}} -{% endmacro %} +{%- macro alter_materialized_view_sql(existing_materialized_view, new_materialized_view) -%} + {{- adapter.dispatch('alter_materialized_view_sql', 'dbt')(existing_materialized_view, new_materialized_view) -}} +{%- endmacro -%} -{% macro default__alter_materialized_view_sql(new_materialized_view, existing_materialized_view) %} - {{ exceptions.raise_compiler_error("Materialized views have not been implemented for this adapter.") }} -{% endmacro %} +{%- macro default__alter_materialized_view_sql(existing_materialized_view, new_materialized_view) -%} + {{- exceptions.raise_compiler_error("`alter_materialized_view_sql()` has not been implemented for this adapter.") -}} +{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/create.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view/create.sql index 13e1cf752e5..44ccf427669 100644 --- a/core/dbt/include/global_project/macros/materializations/models/materialized_view/create.sql +++ b/core/dbt/include/global_project/macros/materializations/models/materialized_view/create.sql @@ -1,9 +1,8 @@ -{% macro create_materialized_view_sql(materialized_view, as_intermediate=False) -%} - {{- log('Applying CREATE to: ' ~ materialized_view.fully_qualified_path) -}} - {{- adapter.dispatch('create_materialized_view_sql', 'dbt')(materialized_view, as_intermediate) -}} -{%- endmacro %} +{%- macro create_materialized_view_sql(materialized_view) -%} + {{- adapter.dispatch('create_materialized_view_sql', 'dbt')(materialized_view) -}} +{%- endmacro -%} -{% macro default__create_materialized_view_sql(materialized_view, as_intermediate=False) -%} - {{ exceptions.raise_compiler_error("Materialized views have not been implemented for this adapter.") }} -{% endmacro %} +{%- macro default__create_materialized_view_sql(materialized_view) -%} + {{- exceptions.raise_compiler_error("`create_materialized_view_sql()` has not been implemented for this adapter.") -}} +{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/describe.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view/describe.sql index be855880a01..0c06cb9e128 100644 --- a/core/dbt/include/global_project/macros/materializations/models/materialized_view/describe.sql +++ b/core/dbt/include/global_project/macros/materializations/models/materialized_view/describe.sql @@ -1,13 +1,12 @@ {# /* - This needs to be a {% do return(...) %} because the macro returns a dictionary, not a string. + This needs to be a {% do return(...) %} because the macro returns a dictionary, not a template. */ #} -{% macro describe_materialized_view_sql(materialized_view) %} - {{- log('Applying DESCRIBE to: ' ~ materialized_view.fully_qualified_path) -}} - {% do return(adapter.dispatch('describe_materialized_view_sql', 'dbt')(materialized_view)) %} -{% endmacro %} +{%- macro describe_materialized_view_sql(materialized_view) -%} + {%- do return(adapter.dispatch('describe_materialized_view_sql', 'dbt')(materialized_view)) -%} +{%- endmacro -%} -{% macro default__describe_materialized_view_sql(materialized_view) %} - {{ exceptions.raise_compiler_error("Materialized views have not been implemented for this adapter.") }} -{% endmacro %} +{%- macro default__describe_materialized_view_sql(materialized_view) -%} + {{- exceptions.raise_compiler_error("`describe_materialized_view_sql()` has not been implemented for this adapter.") -}} +{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/drop.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view/drop.sql index 325bf224549..867f1870ad1 100644 --- a/core/dbt/include/global_project/macros/materializations/models/materialized_view/drop.sql +++ b/core/dbt/include/global_project/macros/materializations/models/materialized_view/drop.sql @@ -1,20 +1,25 @@ --- This is the new one that gets used with RelationConfigBase instances -{% macro drop_materialized_view_sql(materialized_view) -%} - {{ return(adapter.dispatch('drop_materialized_view_sql', 'dbt')(materialized_view)) }} -{%- endmacro %} +{%- macro drop_materialized_view_sql(materialized_view) -%} + {{- adapter.dispatch('drop_materialized_view_sql', 'dbt')(materialized_view) -}} +{%- endmacro -%} -{% macro default__drop_materialized_view_sql(materialized_view) -%} - {{ exceptions.raise_compiler_error("Materialized views have not been implemented for this adapter.") }} -{%- endmacro %} +{%- macro default__drop_materialized_view_sql(materialized_view) -%} + {{- exceptions.raise_compiler_error("`drop_materialized_view_sql()` has not been implemented for this adapter.") -}} +{%- endmacro -%} + + +-- This is the one that gets used when non-materialized-views call drop +{%- macro drop_materialized_view(relation) -%} + + {{- log('Applying DROP to: ' ~ relation) -}} + + {{- return(adapter.dispatch('drop_materialized_view', 'dbt')(relation)) -}} + {{- adapter.cache.drop(relation) -}} --- This is the old one that gets used with BaseRelation instances -{% macro drop_materialized_view(relation) -%} - {{ return(adapter.dispatch('drop_materialized_view', 'dbt')(relation)) }} {%- endmacro %} -{% macro default__drop_materialized_view(relation) -%} - {{ exceptions.raise_compiler_error("Materialized views have not been implemented for this adapter.") }} +{%- macro default__drop_materialized_view(relation) -%} + {{- exceptions.raise_compiler_error("`drop_materialized_view()` has not been implemented for this adapter.") -}} {%- endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/materialized_view.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view/materialized_view.sql index 76cce6d38f2..93fdb4445ca 100644 --- a/core/dbt/include/global_project/macros/materializations/models/materialized_view/materialized_view.sql +++ b/core/dbt/include/global_project/macros/materializations/models/materialized_view/materialized_view.sql @@ -1,69 +1,70 @@ -{% materialization materialized_view, default %} +{%- materialization materialized_view, default -%} -- Try to create a valid materialized view from the config before doing anything else - {% set new_materialized_view = adapter.Relation.from_model_node(config.model) %} + {%- set new_materialized_view = adapter.materialization_config_from_model_node(config.model) -%} -- We still need these because they tie into the existing process (e.g. RelationBase vs. RelationConfigBase) - {% set existing_relation = load_cached_relation_config(new_materialized_view) %} - {% set target_relation = this.incorporate(type=this.MaterializedView) %} + {%- set existing_relation = adapter.get_cached_relation_from_materialization_config(new_materialized_view) -%} - {{ mat_setup(new_materialized_view, pre_hooks) }} + {{- mat_setup(new_materialized_view, pre_hooks) -}} - {% set build_sql = materialized_view_build_sql(new_materialized_view, existing_relation) %} - {% set relation_exists = existing_relation is not none %} + {%- set build_sql = materialized_view_build_sql(new_materialized_view, existing_relation) -%} - {% if build_sql == '' %} - {{ mat_execute_no_op(new_materialized_view) }} - {% else %} - {{ mat_execute_build_sql(new_materialized_view, build_sql, relation_exists, post_hooks) }} - {% endif %} + {%- if build_sql == '' -%} + {{- mat_execute_no_op(new_materialized_view) -}} + {%- else -%} + {{- mat_execute_build_sql(new_materialized_view, existing_relation, build_sql, post_hooks) -}} + {%- endif -%} - {{ mat_teardown(new_materialized_view, post_hooks) }} + {{- mat_teardown(new_materialized_view, post_hooks) -}} - {{ return({'relations': [target_relation]}) }} + {%- set new_relation = adapter.base_relation_from_materialization_config(new_materialized_view) -%} + {{- return({'relations': [new_relation]}) -}} -{% endmaterialization %} +{%- endmaterialization -%} -{% macro materialized_view_build_sql(new_materialized_view, existing_relation) %} +{%- macro materialized_view_build_sql(new_materialized_view, existing_relation) -%} -- determine the scenario we're in: create, full_refresh, alter - {% if existing_relation is none %} - {% set build_sql = create_materialized_view_sql(new_materialized_view) %} - {% elif should_full_refresh() or not existing_relation.is_materialized_view %} - {% set build_sql = replace_materialized_view_sql(new_materialized_view, existing_relation) %} - {% else %} - {% set build_sql = alter_materialized_view_with_on_configuration_option_sql(new_materialized_view) %} - {% endif %} + {%- if existing_relation is none -%} + {%- set build_sql = create_sql(new_materialized_view) -%} + {%- elif should_full_refresh() or not existing_relation.is_materialized_view -%} + {%- set build_sql = replace_sql(existing_relation, new_materialized_view) -%} + {%- else -%} + {%- set build_sql = alter_materialized_view_with_on_configuration_change_option_sql(new_materialized_view) -%} + {%- endif -%} - {% do return(build_sql) %} + {%- do return(build_sql) -%} {% endmacro %} -{% macro alter_materialized_view_with_on_configuration_option_sql(new_materialized_view) %} +{%- macro alter_materialized_view_with_on_configuration_change_option_sql(new_materialized_view) -%} - {% set describe_relation_results = describe_materialized_view_sql(new_materialized_view) %} - {% set existing_materialized_view = adapter.Relation.from_describe_relation_results(describe_relation_results, adapter.Relation.MaterializedView) %} - {% set on_configuration_change = config.get('on_configuration_change') %} + {%- set describe_relation_results = describe_materialized_view_sql(new_materialized_view) -%} + {%- set existing_materialized_view = adapter.materialization_config_from_describe_relation_results( + describe_relation_results, adapter.relation_type().MaterializedView + ) -%} + {%- set on_configuration_change = config.get('on_configuration_change') -%} - {% if new_materialized_view == existing_materialized_view %} - {% set build_sql = refresh_materialized_view_sql(new_materialized_view) %} + {%- if existing_materialized_view == new_materialized_view -%} + {%- set build_sql = refresh_materialized_view_sql(existing_materialized_view) -%} - {% elif on_configuration_change == 'apply' %} - {% set build_sql = alter_materialized_view_sql(new_materialized_view, existing_materialized_view) %} - {% elif on_configuration_change == 'continue' %} - {% set build_sql = '' %} - {{ exceptions.warn("Configuration changes were identified and `on_configuration_change` was set to `continue` for `" ~ new_materialized_view.fully_qualified_path ~ "`") }} - {% elif on_configuration_change == 'fail' %} - {{ exceptions.raise_fail_fast_error("Configuration changes were identified and `on_configuration_change` was set to `fail` for `" ~ new_materialized_view.fully_qualified_path ~ "`") }} + {%- elif on_configuration_change == 'apply' -%} + {%- set build_sql = alter_sql(existing_materialized_view, new_materialized_view) -%} + {%- elif on_configuration_change == 'continue' -%} + {%- set build_sql = '' -%} + {{- exceptions.warn("Configuration changes were identified and `on_configuration_change` was set to `continue` for `" ~ new_materialized_view.fully_qualified_path ~ "`") -}} + {%- elif on_configuration_change == 'fail' -%} + {{- exceptions.raise_fail_fast_error("Configuration changes were identified and `on_configuration_change` was set to `fail` for `" ~ new_materialized_view.fully_qualified_path ~ "`") -}} - {% else %} + {%- else -%} -- this only happens if the user provides a value other than `apply`, 'continue', 'fail', which should have already raised an exception - {{ exceptions.raise_compiler_error("Unexpected configuration scenario: `" ~ on_configuration_change ~ "`") }} + {{- exceptions.raise_compiler_error("Unexpected configuration scenario: `" ~ on_configuration_change ~ "`") -}} - {% endif %} + {%- endif -%} - {% do return(build_sql) %} + {%- do return(build_sql) -%} -{% endmacro %} +{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/refresh.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view/refresh.sql index f0e5039f968..acfd571618e 100644 --- a/core/dbt/include/global_project/macros/materializations/models/materialized_view/refresh.sql +++ b/core/dbt/include/global_project/macros/materializations/models/materialized_view/refresh.sql @@ -1,9 +1,16 @@ -{% macro refresh_materialized_view_sql(materialized_view) %} - {{- log('Applying REFRESH to: ' ~ materialized_view.fully_qualified_path) -}} +{# /* + This macro is elevated to `all` because it only applies to materialized views. +*/ #} + +{%- macro refresh_materialized_view_sql(materialized_view) -%} + + {{- log('Applying REFRESH to: ' ~ materialized_view) -}} + {{- adapter.dispatch('refresh_materialized_view_sql', 'dbt')(materialized_view) -}} -{% endmacro %} + +{%- endmacro -%} -{% macro default__refresh_materialized_view_sql(materialized_view) %} - {{ exceptions.raise_compiler_error("Materialized views have not been implemented for this adapter.") }} -{% endmacro %} +{%- macro default__refresh_materialized_view_sql(materialized_view) -%} + {{- exceptions.raise_compiler_error("`refresh_materialized_view_sql()` has not been implemented for this adapter.") -}} +{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/rename.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view/rename.sql index d9d31294aa2..44abc2a5a9a 100644 --- a/core/dbt/include/global_project/macros/materializations/models/materialized_view/rename.sql +++ b/core/dbt/include/global_project/macros/materializations/models/materialized_view/rename.sql @@ -1,9 +1,8 @@ -{% macro rename_materialized_view_sql(materialized_view, name, from_intermediate=False) %} - {{- log('Applying RENAME to: ' ~ materialized_view.fully_qualified_path) -}} - {{- adapter.dispatch('rename_materialized_view_sql', 'dbt')(materialized_view, name, from_intermediate) -}} -{% endmacro %} +{%- macro rename_materialized_view_sql(materialized_view, new_name) -%} + {{- adapter.dispatch('rename_materialized_view_sql', 'dbt')(materialized_view, new_name) -}} +{%- endmacro -%} -{% macro default__rename_materialized_view_sql(materialized_view, name, from_intermediate=False) %} - {{ exceptions.raise_compiler_error("Materialized views have not been implemented for this adapter.") }} -{% endmacro %} +{%- macro default__rename_materialized_view_sql(materialized_view, new_name) -%} + {{- exceptions.raise_compiler_error("`rename_materialized_view_sql()` has not been implemented for this adapter.") -}} +{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/replace.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view/replace.sql deleted file mode 100644 index 2d68f04884c..00000000000 --- a/core/dbt/include/global_project/macros/materializations/models/materialized_view/replace.sql +++ /dev/null @@ -1,9 +0,0 @@ -{% macro replace_materialized_view_sql(new_materialized_view, existing_relation) %} - {{- log('Applying REPLACE to: ' ~ new_materialized_view.fully_qualified_path) -}} - {{- adapter.dispatch('replace_materialized_view_sql', 'dbt')(new_materialized_view, existing_relation) -}} -{% endmacro %} - - -{% macro default__replace_materialized_view_sql(new_materialized_view, existing_relation) %} - {{ exceptions.raise_compiler_error("Materialized views have not been implemented for this adapter.") }} -{% endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/models/table/rename.sql b/core/dbt/include/global_project/macros/materializations/models/table/rename.sql new file mode 100644 index 00000000000..d3baafa4b1f --- /dev/null +++ b/core/dbt/include/global_project/macros/materializations/models/table/rename.sql @@ -0,0 +1,8 @@ +{%- macro rename_table_sql(table, new_name) -%} + {{- adapter.dispatch('rename_table_sql', 'dbt')(table, new_name) -}} +{%- endmacro -%} + + +{%- macro default__rename_table_sql(table, new_name) -%} + {{- exceptions.raise_compiler_error("`rename_table_sql()` has not been implemented for this adapter.") -}} +{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/materializations/models/view/rename.sql b/core/dbt/include/global_project/macros/materializations/models/view/rename.sql new file mode 100644 index 00000000000..cd31333044d --- /dev/null +++ b/core/dbt/include/global_project/macros/materializations/models/view/rename.sql @@ -0,0 +1,8 @@ +{%- macro rename_view_sql(view, new_name) -%} + {{- adapter.dispatch('rename_view_sql', 'dbt')(view, new_name) -}} +{%- endmacro -%} + + +{%- macro default__rename_view_sql(view, new_name) -%} + {{- exceptions.raise_compiler_error("`rename_view_sql()` has not been implemented for this adapter.") -}} +{%- endmacro -%} diff --git a/plugins/postgres/dbt/adapters/postgres/impl.py b/plugins/postgres/dbt/adapters/postgres/impl.py index a2c6d652d42..dc0cc4407a4 100644 --- a/plugins/postgres/dbt/adapters/postgres/impl.py +++ b/plugins/postgres/dbt/adapters/postgres/impl.py @@ -1,15 +1,14 @@ from datetime import datetime from dataclasses import dataclass -from typing import Optional, Set, List, Any +from typing import Any, FrozenSet, List, Optional, Set from dbt.adapters.base.meta import available from dbt.adapters.base.impl import AdapterConfig, ConstraintSupport +from dbt.adapters.relation_configs import MaterializationConfig, RelationConfigChangeAction from dbt.adapters.sql import SQLAdapter -from dbt.adapters.postgres import PostgresConnectionManager -from dbt.adapters.postgres.column import PostgresColumn -from dbt.adapters.postgres import PostgresRelation -from dbt.dataclass_schema import dbtClassMixin, ValidationError from dbt.contracts.graph.nodes import ConstraintType +from dbt.contracts.relation import ComponentName, RelationType +from dbt.dataclass_schema import dbtClassMixin, ValidationError from dbt.exceptions import ( CrossDbReferenceProhibitedError, IndexConfigNotDictError, @@ -19,6 +18,18 @@ ) import dbt.utils +from dbt.adapters.postgres import PostgresConnectionManager, PostgresRelation +from dbt.adapters.postgres.column import PostgresColumn +from dbt.adapters.postgres.relation_configs import ( + PostgresIndexConfig as PostgresIndexConfigMatView, + PostgresIndexConfigChange, + PostgresMaterializedViewConfig, + PostgresMaterializedViewConfigChangeset, + PostgresIncludePolicy, + PostgresQuotePolicy, + postgres_conform_part, +) + # note that this isn't an adapter macro, so just a single underscore GET_RELATIONS_MACRO_NAME = "postgres_get_relations" @@ -73,6 +84,9 @@ class PostgresAdapter(SQLAdapter): ConstraintType.primary_key: ConstraintSupport.ENFORCED, ConstraintType.foreign_key: ConstraintSupport.ENFORCED, } + materialization_configs = {RelationType.MaterializedView: PostgresMaterializedViewConfig} + include_policy = PostgresIncludePolicy() + quote_policy = PostgresQuotePolicy() @classmethod def date_function(cls): @@ -144,3 +158,89 @@ def valid_incremental_strategies(self): def debug_query(self): self.execute("select 1 as id") + + @available + @classmethod + def materialized_view_config_changeset( + cls, + existing_materialized_view: PostgresMaterializedViewConfig, + new_materialized_view: PostgresMaterializedViewConfig, + ) -> PostgresMaterializedViewConfigChangeset: + try: + assert isinstance(existing_materialized_view, PostgresMaterializedViewConfig) + assert isinstance(new_materialized_view, PostgresMaterializedViewConfig) + except AssertionError: + raise DbtRuntimeError( + f"Two materialized view configs were expected, but received:" + f"/n {existing_materialized_view}" + f"/n {new_materialized_view}" + ) + + config_changeset = PostgresMaterializedViewConfigChangeset() + + config_changeset.indexes = cls.index_config_changeset( + existing_materialized_view.indexes, new_materialized_view.indexes + ) + + if config_changeset.is_empty and existing_materialized_view != new_materialized_view: + # we need to force a full refresh if we didn't detect any changes but the objects are not the same + config_changeset.force_full_refresh() + + return config_changeset + + @available + @classmethod + def index_config_changeset( + cls, + existing_indexes: FrozenSet[PostgresIndexConfigMatView], + new_indexes: FrozenSet[PostgresIndexConfigMatView], + ) -> Set[PostgresIndexConfigChange]: + """ + Get the index updates that will occur as a result of a new run + + There are four scenarios: + + 1. Indexes are equal -> don't return these + 2. Index is new -> create these + 3. Index is old -> drop these + 4. Indexes are not equal -> drop old, create new -> two actions + + Returns: a set of index updates in the form {"action": "drop/create", "context": } + """ + drop_changes = set( + PostgresIndexConfigChange(action=RelationConfigChangeAction.drop, context=index) + for index in existing_indexes.difference(new_indexes) + ) + create_changes = set( + PostgresIndexConfigChange(action=RelationConfigChangeAction.create, context=index) + for index in new_indexes.difference(existing_indexes) + ) + return set().union(drop_changes, create_changes) + + @available + @classmethod + def generate_index_name( + cls, + materialization_config: MaterializationConfig, + index_config: PostgresIndexConfigMatView, + ) -> str: + return dbt.utils.md5( + "_".join( + { + postgres_conform_part( + ComponentName.Database, materialization_config.database_name + ), + postgres_conform_part( + ComponentName.Schema, materialization_config.schema_name + ), + postgres_conform_part(ComponentName.Identifier, materialization_config.name), + *sorted( + postgres_conform_part(ComponentName.Identifier, column) + for column in index_config.column_names + ), + str(index_config.unique), + str(index_config.method), + str(datetime.utcnow().isoformat()), + } + ) + ) diff --git a/plugins/postgres/dbt/adapters/postgres/relation.py b/plugins/postgres/dbt/adapters/postgres/relation.py index 6fc537d405f..b2e0d6d93e8 100644 --- a/plugins/postgres/dbt/adapters/postgres/relation.py +++ b/plugins/postgres/dbt/adapters/postgres/relation.py @@ -1,38 +1,13 @@ -from dataclasses import dataclass, field -from datetime import datetime -from typing import Set, FrozenSet, Dict +from dataclasses import dataclass from dbt.adapters.base.relation import BaseRelation -from dbt.adapters.relation_configs import ( - RelationConfigChangeAction, - RelationConfig, - MaterializationConfig, -) -from dbt.contracts.relation import ComponentName, RelationType from dbt.exceptions import DbtRuntimeError -import dbt.utils -from dbt.adapters.postgres.relation_configs import ( - PostgresIndexConfig, - PostgresIndexConfigChange, - PostgresMaterializedViewConfig, - PostgresMaterializedViewConfigChangeset, - PostgresIncludePolicy, - PostgresQuotePolicy, - postgres_conform_part, - MAX_CHARACTERS_IN_IDENTIFIER, -) +from dbt.adapters.postgres.relation_configs import MAX_CHARACTERS_IN_IDENTIFIER @dataclass(frozen=True, eq=False, repr=False) class PostgresRelation(BaseRelation): - include_policy: PostgresIncludePolicy = field(default_factory=PostgresIncludePolicy) - quote_policy: PostgresQuotePolicy = field(default_factory=PostgresQuotePolicy) - - @classmethod - def relation_configs(cls) -> Dict[RelationType, RelationConfig]: - return {RelationType.MaterializedView: PostgresMaterializedViewConfig} # type: ignore - def __post_init__(self): # Check for length of Postgres table/view names. # Check self.type to exclude test relation identifiers @@ -48,84 +23,3 @@ def __post_init__(self): def relation_max_name_length(self): return MAX_CHARACTERS_IN_IDENTIFIER - - @classmethod - def materialized_view_config_changeset( - cls, - new_materialized_view: PostgresMaterializedViewConfig, - existing_materialized_view: PostgresMaterializedViewConfig, - ) -> PostgresMaterializedViewConfigChangeset: - try: - assert isinstance(new_materialized_view, PostgresMaterializedViewConfig) - assert isinstance(existing_materialized_view, PostgresMaterializedViewConfig) - except AssertionError: - raise DbtRuntimeError( - f"Two materialized view configs were expected, but received:" - f"/n {new_materialized_view}" - f"/n {existing_materialized_view}" - ) - - config_changeset = PostgresMaterializedViewConfigChangeset() - - config_changeset.indexes = cls.index_config_changeset( - new_materialized_view.indexes, existing_materialized_view.indexes - ) - - if config_changeset.is_empty and new_materialized_view != existing_materialized_view: - # we need to force a full refresh if we didn't detect any changes but the objects are not the same - config_changeset.force_full_refresh() - - return config_changeset - - @classmethod - def index_config_changeset( - cls, - new_indexes: FrozenSet[PostgresIndexConfig], - existing_indexes: FrozenSet[PostgresIndexConfig], - ) -> Set[PostgresIndexConfigChange]: - """ - Get the index updates that will occur as a result of a new run - - There are four scenarios: - - 1. Indexes are equal -> don't return these - 2. Index is new -> create these - 3. Index is old -> drop these - 4. Indexes are not equal -> drop old, create new -> two actions - - Returns: a set of index updates in the form {"action": "drop/create", "context": } - """ - drop_changes = set( - PostgresIndexConfigChange(action=RelationConfigChangeAction.drop, context=index) - for index in existing_indexes.difference(new_indexes) - ) - create_changes = set( - PostgresIndexConfigChange(action=RelationConfigChangeAction.create, context=index) - for index in new_indexes.difference(existing_indexes) - ) - return set().union(drop_changes, create_changes) - - @staticmethod - def generate_index_name( - materialization_config: MaterializationConfig, index_config: PostgresIndexConfig - ) -> str: - return dbt.utils.md5( - "_".join( - { - postgres_conform_part( - ComponentName.Database, materialization_config.database_name - ), - postgres_conform_part( - ComponentName.Schema, materialization_config.schema_name - ), - postgres_conform_part(ComponentName.Identifier, materialization_config.name), - *sorted( - postgres_conform_part(ComponentName.Identifier, column) - for column in index_config.column_names - ), - str(index_config.unique), - str(index_config.method), - str(datetime.utcnow().isoformat()), - } - ) - ) diff --git a/plugins/postgres/dbt/adapters/postgres/relation_configs/materialized_view.py b/plugins/postgres/dbt/adapters/postgres/relation_configs/materialized_view.py index 3162359a713..1c0df94d87d 100644 --- a/plugins/postgres/dbt/adapters/postgres/relation_configs/materialized_view.py +++ b/plugins/postgres/dbt/adapters/postgres/relation_configs/materialized_view.py @@ -12,7 +12,6 @@ from dbt.contracts.graph.nodes import ModelNode from dbt.contracts.relation import ComponentName, RelationType from dbt.exceptions import DbtRuntimeError -import dbt.utils from dbt.adapters.postgres.relation_configs.index import ( PostgresIndexConfig, @@ -48,7 +47,7 @@ class PostgresMaterializedViewConfig(MaterializationConfig, RelationConfigValida schema: PostgresSchemaConfig query: str = field(hash=False, compare=False) indexes: FrozenSet[PostgresIndexConfig] = field(default_factory=frozenset) - relation_type: Optional[RelationType] = RelationType.MaterializedView + type: Optional[RelationType] = RelationType.MaterializedView @property def fully_qualified_path(self) -> str: @@ -62,30 +61,6 @@ def fully_qualified_path(self) -> str: ) ) - @property - def fully_qualified_path_backup(self) -> str: - return postgres_render( - OrderedDict( - { - ComponentName.Database: self.database_name, - ComponentName.Schema: self.schema_name, - ComponentName.Identifier: self.backup_name, - } - ) - ) - - @property - def fully_qualified_path_intermediate(self) -> str: - return postgres_render( - OrderedDict( - { - ComponentName.Database: self.database_name, - ComponentName.Schema: self.schema_name, - ComponentName.Identifier: self.intermediate_name, - } - ) - ) - @property def validation_rules(self) -> Set[RelationConfigValidationRule]: """ @@ -224,18 +199,6 @@ def parse_describe_relation_results( } return config_dict - def generate_index_name(self, index_fully_qualified_path) -> str: - return dbt.utils.md5( - "_".join( - { - self.database_name, - self.schema_name, - self.name, - index_fully_qualified_path, - } - ) - ) - @dataclass class PostgresMaterializedViewConfigChangeset(RelationConfigChangeset): diff --git a/plugins/postgres/dbt/include/postgres/macros/index.sql b/plugins/postgres/dbt/include/postgres/macros/index.sql index dcfeff400b6..e19da6052e4 100644 --- a/plugins/postgres/dbt/include/postgres/macros/index.sql +++ b/plugins/postgres/dbt/include/postgres/macros/index.sql @@ -26,10 +26,10 @@ {%- for _index_change in index_changeset -%} {%- set _index_config = _index_change.context -%} - {%- if _index_change.action == "drop" -%} + {%- if _index_change.action == adapter.relation_config_change_action.drop -%} {{ postgres__drop_index_sql(_index_config) }}; - {%- elif _index_change.action == "create" -%} + {%- elif _index_change.action == adapter.relation_config_change_action.create -%} {{ postgres__create_index_sql(materialization_config, _index_config) }}; {%- endif -%} @@ -38,26 +38,21 @@ {%- endmacro %} -{% macro postgres__create_indexes_sql(materialization_config, on_intermediate=False) -%} +{% macro postgres__create_indexes_sql(materialization_config) -%} {% for _index_config in materialization_config.indexes -%} - {{- postgres__create_index_sql(materialization_config, _index_config, on_intermediate) -}}; + {{- postgres__create_index_sql(materialization_config, _index_config) -}}; {%- endfor -%} {%- endmacro %} -{% macro postgres__create_index_sql(materialization_config, index_config, on_intermediate=False) -%} - {%- if on_intermediate -%} - {%- set _materialization_path = materialization_config.fully_qualified_path_intermediate -%} - {%- else -%} - {%- set _materialization_path = materialization_config.fully_qualified_path -%} - {%- endif -%} +{% macro postgres__create_index_sql(materialization_config, index_config) -%} - {%- set _index_name = adapter.Relation.generate_index_name(materialization_config, index_config) -%} + {%- set _index_name = adapter.generate_index_name(materialization_config, index_config) -%} create {% if index_config.unique -%}unique{%- endif %} index if not exists "{{ _index_name }}" - on {{ _materialization_path }} + on {{ materialization_config.fully_qualified_path }} using {{ index_config.method }} ( {{ ", ".join(index_config.column_names) }} diff --git a/plugins/postgres/dbt/include/postgres/macros/materializations/materialized_view.sql b/plugins/postgres/dbt/include/postgres/macros/materializations/materialized_view.sql index 716f5d9119a..d82870597c8 100644 --- a/plugins/postgres/dbt/include/postgres/macros/materializations/materialized_view.sql +++ b/plugins/postgres/dbt/include/postgres/macros/materializations/materialized_view.sql @@ -21,45 +21,40 @@ */ -#} -{% macro postgres__alter_materialized_view_sql(new_materialized_view, existing_materialized_view) %} +{%- macro postgres__alter_materialized_view_sql(existing_materialized_view, new_materialized_view) -%} {#- /* We need to get the config changeset to determine if we require a full refresh (happens if any change in the changeset requires a full refresh or if an unmonitored change was detected) or if we can get away with altering the dynamic table in place. */ -#} - {% set _config_changeset = adapter.Relation.materialized_view_config_changeset(new_materialized_view, existing_materialized_view) %} + {%- set _config_changeset = adapter.materialized_view_config_changeset(existing_materialized_view, new_materialized_view) -%} - {% if _config_changeset.requires_full_refresh %} + {%- if _config_changeset.requires_full_refresh -%} {{ replace_materialized_view_sql(new_materialized_view) }} - {% else %} + {%- else -%} {{ postgres__alter_indexes_sql(new_materialized_view, _config_changeset.indexes) }} {%- endif -%} -{% endmacro %} +{%- endmacro -%} -{% macro postgres__create_materialized_view_sql(materialized_view, as_intermediate=False) %} - {%- if as_intermediate -%} - {%- set materialized_view_path = materialized_view.fully_qualified_path_intermediate -%} - {%- else -%} - {%- set materialized_view_path = materialized_view.fully_qualified_path -%} - {%- endif -%} +{%- macro postgres__create_materialized_view_sql(materialized_view) -%} - create materialized view {{ materialized_view_path }} as + create materialized view {{ materialized_view.fully_qualified_path }} as {{ materialized_view.query }} ; - {{- postgres__create_indexes_sql(materialized_view, on_intermediate) -}} + {{ postgres__create_indexes_sql(materialized_view) -}} -{% endmacro %} +{%- endmacro -%} -{% macro postgres__describe_materialized_view_sql(materialized_view) %} +{%- macro postgres__describe_materialized_view_sql(materialized_view) -%} - {% set _materialized_view_sql -%} + {%- set _materialized_view_sql -%} select v.matviewname, v.schemaname, @@ -69,7 +64,7 @@ where v.matviewname ilike '{{ materialized_view.name }}' and v.schemaname ilike '{{ materialized_view.schema_name }}' {%- endset -%} - {% set _materialized_view = run_query(_materialized_view_sql) %} + {%- set _materialized_view = run_query(_materialized_view_sql) -%} {%- set _indexes_sql -%} select @@ -95,44 +90,37 @@ group by 1, 2, 3 order by 1, 2, 3 {%- endset -%} - {% set _indexes = run_query(_indexes_sql) %} + {%- set _indexes = run_query(_indexes_sql) -%} - {% do return({'materialized_view': _materialized_view, 'indexes': _indexes}) %} + {%- do return({'materialized_view': _materialized_view, 'indexes': _indexes}) -%} -{% endmacro %} +{%- endmacro -%} --- This is the new one that gets used with RelationConfigBase instances -{% macro postgres__drop_materialized_view_sql(materialized_view) -%} +{%- macro postgres__drop_materialized_view_sql(materialized_view) -%} drop materialized view if exists {{ materialized_view.fully_qualified_path }} cascade -{%- endmacro %} +{%- endmacro -%} -- This is the old one that gets used with BaseRelation instances -{% macro postgres__drop_materialized_view(relation) -%} +{%- macro postgres__drop_materialized_view(relation) -%} drop materialized view if exists {{ relation }} cascade -{%- endmacro %} +{%- endmacro -%} -{% macro postgres__refresh_materialized_view_sql(materialized_view) %} +{%- macro postgres__refresh_materialized_view_sql(materialized_view) -%} refresh materialized view {{ materialized_view.fully_qualified_path }} -{% endmacro %} +{%- endmacro -%} -{% macro postgres__rename_materialized_view_sql(materialized_view, name, from_intermediate=False) %} - {%- if from_intermediate -%} - {%- set materialized_view_path = materialized_view.fully_qualified_path_intermediate -%} +{%- macro postgres__rename_materialized_view_sql(materialized_view, new_name) -%} + + {%- if adapter.is_materialization_config(materialized_view) -%} + {%- set fully_qualified_path = materialized_view.fully_qualified_path -%} {%- else -%} - {%- set materialized_view_path = materialized_view.fully_qualified_path -%} + {%- set fully_qualified_path = materialized_view -%} {%- endif -%} - alter materialized view {{ materialized_view_path }} rename to {{ name }} -{% endmacro %} - + alter materialized view {{ fully_qualified_path }} rename to {{ new_name }} -{% macro postgres__replace_materialized_view_sql(new_materialized_view, existing_relation) %} - {{- create_materialized_view_sql(new_materialized_view, as_intermediate=True) -}} ---TODO: this only works for existing materialized views - {{ rename_materialized_view_sql(new_materialized_view, new_materialized_view.backup_name) }}; - {{ rename_materialized_view_sql(new_materialized_view, new_materialized_view.name, from_intermediate=True) }} -{% endmacro %} +{%- endmacro -%} diff --git a/plugins/postgres/dbt/include/postgres/macros/materializations/replace.sql b/plugins/postgres/dbt/include/postgres/macros/materializations/replace.sql new file mode 100644 index 00000000000..c596c693e03 --- /dev/null +++ b/plugins/postgres/dbt/include/postgres/macros/materializations/replace.sql @@ -0,0 +1,25 @@ +{%- macro postgres__replace_sql(existing_materialization, new_materialization) -%} + + {#- new_materialization is always a MaterializationConfig instance -#} + {%- set intermediate_materialization = adapter.make_intermediate_materialization_config(new_materialization) -%} + {%- set new_name = new_materialization.name -%} + + + {%- if adapter.is_materialization_config(existing_materialization) -%} + {%- set backup_name = adapter.make_backup_name(existing_materialization) -%} + + {%- elif adapter.is_base_relation(existing_materialization) -%} + {%- set backup_relation = make_backup_relation(existing_materialization, existing_materialization.type) -%} + {%- set backup_name = backup_relation.identifier -%} + + {%- else -%} + {{- exceptions.raise_compiler_error("Unexpected materialization type: " ~ existing_materialization) -}} + + {%- endif -%} + + + {{ create_sql(intermediate_materialization) }}; + {{ rename_sql(existing_materialization, backup_name) }}; + {{ rename_sql(intermediate_materialization, new_name) }} + +{%- endmacro -%} diff --git a/plugins/postgres/dbt/include/postgres/macros/materializations/table.sql b/plugins/postgres/dbt/include/postgres/macros/materializations/table.sql new file mode 100644 index 00000000000..eb70eb1c1d5 --- /dev/null +++ b/plugins/postgres/dbt/include/postgres/macros/materializations/table.sql @@ -0,0 +1,8 @@ +{# /* + These are BaseRelation instances, not MaterializationConfig instances +*/ #} + + +{%- macro postgres__rename_table_sql(table, new_name) -%} + alter table {{ table }} rename to {{ new_name }} +{%- endmacro -%} diff --git a/plugins/postgres/dbt/include/postgres/macros/materializations/view.sql b/plugins/postgres/dbt/include/postgres/macros/materializations/view.sql new file mode 100644 index 00000000000..ffbd24f3325 --- /dev/null +++ b/plugins/postgres/dbt/include/postgres/macros/materializations/view.sql @@ -0,0 +1,8 @@ +{# /* + These are BaseRelation instances, not MaterializationConfig instances +*/ #} + + +{%- macro postgres__rename_view_sql(view, new_name) -%} + alter view {{ view }} rename to {{ new_name }} +{%- endmacro -%} From 6dfe0f381e74195c4671dc5369c051c20a2c5296 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 29 Jun 2023 03:33:05 -0400 Subject: [PATCH 05/28] added updates from snowflake to core, pushed down to postgres, final draft --- core/dbt/adapters/base/impl.py | 6 +----- core/dbt/adapters/base/relation.py | 1 - .../macros/adapters/drop_relation.sql | 13 ------------- .../models/materialized_view/materialized_view.sql | 2 +- 4 files changed, 2 insertions(+), 20 deletions(-) diff --git a/core/dbt/adapters/base/impl.py b/core/dbt/adapters/base/impl.py index 400446c9887..2c7fe537ff0 100644 --- a/core/dbt/adapters/base/impl.py +++ b/core/dbt/adapters/base/impl.py @@ -231,13 +231,9 @@ class BaseAdapter(metaclass=AdapterMeta): ConstraintType.foreign_key: ConstraintSupport.ENFORCED, } + # registers MaterializationConfigs to RelationTypes materialization_configs: Dict[RelationType, MaterializationConfig] = {} - @available - @classmethod - def relation_type(cls) -> Type[RelationType]: - return RelationType - @available @classmethod def relation_config_change_action(cls) -> Type[RelationConfigChangeAction]: diff --git a/core/dbt/adapters/base/relation.py b/core/dbt/adapters/base/relation.py index b7fc331f448..5af5ec8aa36 100644 --- a/core/dbt/adapters/base/relation.py +++ b/core/dbt/adapters/base/relation.py @@ -40,7 +40,6 @@ class BaseRelation(FakeAPIObject, Hashable): include_policy: Policy = field(default_factory=lambda: Policy()) quote_policy: Policy = field(default_factory=lambda: Policy()) dbt_created: bool = False - # registers RelationConfigBases to RelationTypes def _is_exactish_match(self, field: ComponentName, value: str) -> bool: if self.dbt_created and self.quote_policy.get_part(field) is False: diff --git a/core/dbt/include/global_project/macros/adapters/drop_relation.sql b/core/dbt/include/global_project/macros/adapters/drop_relation.sql index 5fc3ec79f70..06eafcf1d39 100644 --- a/core/dbt/include/global_project/macros/adapters/drop_relation.sql +++ b/core/dbt/include/global_project/macros/adapters/drop_relation.sql @@ -33,16 +33,3 @@ {% macro default__drop_view(relation) -%} drop view if exists {{ relation }} cascade {%- endmacro %} - - -{% macro drop_relation_sql(relation_config) -%} - {{ return(adapter.dispatch('drop_relation_sql', 'dbt')(relation_config)) }} -{%- endmacro %} - -{%- macro default__drop_relation_sql(relation_config) -%} - {%- if relation_config.relation_type == adapter.Relation.MaterializedView -%} - {{- drop_materialized_view_sql(relation_config) -}} - {%- else -%} - drop {{ relation_config.relation_type }} if exists {{ relation_config.fully_qualified_path }} cascade - {%- endif -%} -{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/materialized_view.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view/materialized_view.sql index 93fdb4445ca..b25e15630fc 100644 --- a/core/dbt/include/global_project/macros/materializations/models/materialized_view/materialized_view.sql +++ b/core/dbt/include/global_project/macros/materializations/models/materialized_view/materialized_view.sql @@ -44,7 +44,7 @@ {%- set describe_relation_results = describe_materialized_view_sql(new_materialized_view) -%} {%- set existing_materialized_view = adapter.materialization_config_from_describe_relation_results( - describe_relation_results, adapter.relation_type().MaterializedView + describe_relation_results, adapter.Relation.MaterializedView ) -%} {%- set on_configuration_change = config.get('on_configuration_change') -%} From 459d7ff4f97c4312db500851697265d4f7969940 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 29 Jun 2023 03:33:51 -0400 Subject: [PATCH 06/28] changie --- .changes/unreleased/Features-20230629-033005.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Features-20230629-033005.yaml diff --git a/.changes/unreleased/Features-20230629-033005.yaml b/.changes/unreleased/Features-20230629-033005.yaml new file mode 100644 index 00000000000..65fda5420fa --- /dev/null +++ b/.changes/unreleased/Features-20230629-033005.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Establish framework for materialized views and materialization change management +time: 2023-06-29T03:30:05.527325-04:00 +custom: + Author: mikealfare + Issue: "6911" From c0f52b5a63d5d4f782ba268c347f549108559119 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 29 Jun 2023 03:36:27 -0400 Subject: [PATCH 07/28] docs --- core/dbt/adapters/relation_configs/README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/core/dbt/adapters/relation_configs/README.md b/core/dbt/adapters/relation_configs/README.md index a1d5976f89b..7550ffe5856 100644 --- a/core/dbt/adapters/relation_configs/README.md +++ b/core/dbt/adapters/relation_configs/README.md @@ -10,9 +10,9 @@ other classes in `relation_configs` subclass from one of these classes. ### RelationConfig This class holds the primary parsing methods required for marshalling data from a user config or a database metadata -query into a `RelationConfigBase` subclass. `RelationConfigBase` is a good class to subclass for smaller, atomic +query into a `RelationConfig` subclass. `RelationConfig` is a good class to subclass for smaller, atomic database objects or objects that may be specific to a subset of adapters. For example, a Postgres index is modelled -from `RelationConfigBase` because not every database has an index and there is not much hierarchy to an index. +from `RelationConfig` because not every database has an index and there is not much hierarchy to an index. The objective of this parser is to provide a stopping point between dbt-specific config and database-specific config for two primary reasons: @@ -25,9 +25,9 @@ At some point this could be theoretically be replaced by a more robust framework ### RelationConfigChange This class holds the methods required for detecting and acting on changes in a materialization. All changes should subclass from `RelationConfigChange`. A `RelationConfigChange` can be thought of as being analogous -to a web request on a `RelationConfigBase`. You need to know what you're doing +to a web request on a `RelationConfig`. You need to know what you're doing (`action`: 'create' = GET, 'drop' = DELETE, etc.) and the information (`context`) needed to make the change. -In our scenarios, `context` tends to be either an instance of `RelationConfigBase` corresponding to the new state +In our scenarios, `context` tends to be either an instance of `RelationConfig` corresponding to the new state or a single value if the change is simple. For example, creating an index would require the entire config; whereas updating a setting like autorefresh for Redshift would require only the setting. @@ -45,7 +45,7 @@ the `validation_error` be provided for clearer transparency to the end user. ## Basic Building Blocks (Data)Classes ### DatabaseConfig -This is the most basic version of `RelationConfigBase` that we can have. It adds a `name` and a `fully_qualified_path` +This is the most basic version of `RelationConfig` that we can have. It adds a `name` and a `fully_qualified_path` and nothing else. But we generally need a database when dbt runs. In particular, we need to reference a database in other classes, like `SchemaConfig`. @@ -86,5 +86,5 @@ There are also a handful of methods on `BaseRelation` that are meant to be used These are effectively for creating `MaterializationConfig` subclasses given data from either the `model` attribute in the global jinja context or from data from the database. Ultimately, we're treating -`BaseRelation` as a service layer that gets exposed in the jinja context, and tucking everything else into this +`BaseAdapter` as a service layer that gets exposed in the jinja context, and tucking everything else into this subpackage. From 7e63255ac779d6dfc045dd03055c951fc8097f92 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 29 Jun 2023 15:00:15 -0400 Subject: [PATCH 08/28] created a Materialization class, analogous to BaseRelation --- core/dbt/adapters/base/__init__.py | 16 +- core/dbt/adapters/base/impl.py | 217 ++++-------------- core/dbt/adapters/base/materialization.py | 130 +++++++++++ core/dbt/adapters/base/relation.py | 45 +++- .../README.md | 0 .../materialization_config/__init__.py | 20 ++ .../_base.py | 0 .../_change.py | 0 .../_database.py | 2 +- .../_materialization.py | 6 +- .../_policy.py | 0 .../_schema.py | 4 +- .../_validation.py | 0 .../dbt/adapters/relation_configs/__init__.py | 20 -- core/dbt/context/providers.py | 10 + .../materializations/materialization.sql | 8 +- .../materializations/models/all/alter.sql | 2 +- .../materializations/models/all/create.sql | 2 +- .../materializations/models/all/describe.sql | 2 +- .../materializations/models/all/drop.sql | 4 +- .../materializations/models/all/rename.sql | 8 +- .../materialized_view/materialized_view.sql | 16 +- .../postgres/dbt/adapters/postgres/impl.py | 104 +-------- .../dbt/adapters/postgres/materialization.py | 110 +++++++++ .../__init__.py | 6 +- .../database.py | 7 +- .../index.py | 4 +- .../materialized_view.py | 8 +- .../policy.py | 2 +- .../schema.py | 9 +- .../dbt/adapters/postgres/relation.py | 9 +- .../dbt/include/postgres/macros/adapters.sql | 26 --- .../dbt/include/postgres/macros/index.sql | 40 +++- .../materializations/materialized_view.sql | 27 +-- .../macros/materializations/replace.sql | 2 +- 35 files changed, 454 insertions(+), 412 deletions(-) create mode 100644 core/dbt/adapters/base/materialization.py rename core/dbt/adapters/{relation_configs => materialization_config}/README.md (100%) create mode 100644 core/dbt/adapters/materialization_config/__init__.py rename core/dbt/adapters/{relation_configs => materialization_config}/_base.py (100%) rename core/dbt/adapters/{relation_configs => materialization_config}/_change.py (100%) rename core/dbt/adapters/{relation_configs => materialization_config}/_database.py (91%) rename core/dbt/adapters/{relation_configs => materialization_config}/_materialization.py (90%) rename core/dbt/adapters/{relation_configs => materialization_config}/_policy.py (100%) rename core/dbt/adapters/{relation_configs => materialization_config}/_schema.py (86%) rename core/dbt/adapters/{relation_configs => materialization_config}/_validation.py (100%) delete mode 100644 core/dbt/adapters/relation_configs/__init__.py create mode 100644 plugins/postgres/dbt/adapters/postgres/materialization.py rename plugins/postgres/dbt/adapters/postgres/{relation_configs => materialization_config}/__init__.py (59%) rename plugins/postgres/dbt/adapters/postgres/{relation_configs => materialization_config}/database.py (93%) rename plugins/postgres/dbt/adapters/postgres/{relation_configs => materialization_config}/index.py (97%) rename plugins/postgres/dbt/adapters/postgres/{relation_configs => materialization_config}/materialized_view.py (96%) rename plugins/postgres/dbt/adapters/postgres/{relation_configs => materialization_config}/policy.py (95%) rename plugins/postgres/dbt/adapters/postgres/{relation_configs => materialization_config}/schema.py (90%) diff --git a/core/dbt/adapters/base/__init__.py b/core/dbt/adapters/base/__init__.py index 07f5303992e..34685155c17 100644 --- a/core/dbt/adapters/base/__init__.py +++ b/core/dbt/adapters/base/__init__.py @@ -1,19 +1,17 @@ -# these are all just exports, #noqa them so flake8 will be happy - # TODO: Should we still include this in the `adapters` namespace? -from dbt.contracts.connection import Credentials # noqa: F401 -from dbt.adapters.base.meta import available # noqa: F401 -from dbt.adapters.base.connections import BaseConnectionManager # noqa: F401 -from dbt.adapters.base.relation import ( # noqa: F401 +from dbt.contracts.connection import Credentials +from dbt.adapters.base.meta import available +from dbt.adapters.base.connections import BaseConnectionManager +from dbt.adapters.base.relation import ( BaseRelation, RelationType, SchemaSearchMap, ) -from dbt.adapters.base.column import Column # noqa: F401 -from dbt.adapters.base.impl import ( # noqa: F401 +from dbt.adapters.base.column import Column +from dbt.adapters.base.impl import ( AdapterConfig, BaseAdapter, PythonJobHelper, ConstraintSupport, ) -from dbt.adapters.base.plugin import AdapterPlugin # noqa: F401 +from dbt.adapters.base.plugin import AdapterPlugin diff --git a/core/dbt/adapters/base/impl.py b/core/dbt/adapters/base/impl.py index 2c7fe537ff0..c2602df2787 100644 --- a/core/dbt/adapters/base/impl.py +++ b/core/dbt/adapters/base/impl.py @@ -1,7 +1,6 @@ import abc from concurrent.futures import as_completed, Future from contextlib import contextmanager -import dataclasses from datetime import datetime from enum import Enum from itertools import chain @@ -27,6 +26,7 @@ from dbt import deprecations from dbt.adapters.base import Credentials, Column as BaseColumn from dbt.adapters.base.connections import AdapterResponse, Connection +from dbt.adapters.base.materialization import Materialization as _Materialization from dbt.adapters.base.meta import AdapterMeta, available from dbt.adapters.base.relation import ( ComponentName, @@ -35,12 +35,8 @@ SchemaSearchMap, ) from dbt.adapters.cache import RelationsCache, _make_ref_key_dict +from dbt.adapters.materialization_config import MaterializationConfig from dbt.adapters.protocol import AdapterConfig, ConnectionManagerProtocol -from dbt.adapters.relation_configs import ( - DescribeRelationResults, - MaterializationConfig, - RelationConfigChangeAction, -) from dbt.clients.agate_helper import empty_table, merge_tables, table_from_rows from dbt.clients.jinja import MacroGenerator from dbt.contracts.graph.manifest import MacroManifest, Manifest @@ -48,10 +44,9 @@ ColumnLevelConstraint, ConstraintType, ModelLevelConstraint, - ModelNode, ResultNode, ) -from dbt.contracts.relation import RelationType +from dbt.contracts.relation import RelationType as _RelationType from dbt.events.functions import fire_event, warn_or_error from dbt.events.types import ( CacheMiss, @@ -216,6 +211,8 @@ class BaseAdapter(metaclass=AdapterMeta): """ Relation: Type[BaseRelation] = BaseRelation + RelationType: Type[_RelationType] = _RelationType + Materialization: Type[_Materialization] = _Materialization Column: Type[BaseColumn] = BaseColumn ConnectionManager: Type[ConnectionManagerProtocol] @@ -231,14 +228,6 @@ class BaseAdapter(metaclass=AdapterMeta): ConstraintType.foreign_key: ConstraintSupport.ENFORCED, } - # registers MaterializationConfigs to RelationTypes - materialization_configs: Dict[RelationType, MaterializationConfig] = {} - - @available - @classmethod - def relation_config_change_action(cls) -> Type[RelationConfigChangeAction]: - return RelationConfigChangeAction - def __init__(self, config): self.config = config self.cache = RelationsCache() @@ -832,6 +821,33 @@ def get_relation(self, database: str, schema: str, identifier: str) -> Optional[ return None + @available + def get_cached_relation( + self, materialization_config: MaterializationConfig, kind: Optional[str] = None + ) -> BaseRelation: + """ + Expose `get_relation()` to `MaterializationConfig` instances. Includes a `kind` argument to + combine `load_cached_relation()` macro with `make_backup_relation()` and `make_intermediate_relation()` + with `get_relation()`. + """ + cached_kind = { + "existing": materialization_config.name, + "backup": self.Materialization.backup_name(materialization_config), + "intermediate": self.Materialization.intermediate_name(materialization_config), + } + + if name := cached_kind.get(kind or "existing"): + return self.get_relation( + database=materialization_config.database_name, + schema=materialization_config.schema_name, + identifier=name, + ) + + raise DbtRuntimeError( + f"Invalid cached relation kind for `get_cached_relation()`: {kind}" + f" Please supply one of: {', '.join(cached_kind.keys())}" + ) + @available.deprecated("get_relation", lambda *a, **k: False) def already_exists(self, schema: str, name: str) -> bool: """DEPRECATED: Return if a model already exists in the database""" @@ -1185,7 +1201,7 @@ def post_model_hook(self, config: Mapping[str, Any], context: Any) -> None: available in the materialization context). It should be considered read-only. - The second parameter is the value returned by pre_mdoel_hook. + The second parameter is the value returned by pre_model_hook. """ pass @@ -1438,171 +1454,22 @@ def render_model_constraint(cls, constraint: ModelLevelConstraint) -> Optional[s return None @available - @classmethod - def is_base_relation(cls, materialization: Union[BaseRelation, MaterializationConfig]) -> bool: - return isinstance(materialization, BaseRelation) - - @available - @classmethod - def is_materialization_config( - cls, materialization: Union[BaseRelation, MaterializationConfig] + def is_base_relation( + self, materialization: Union[BaseRelation, MaterializationConfig] ) -> bool: - return isinstance(materialization, MaterializationConfig) - - @available - def base_relation_from_materialization_config( - self, materialization_config: MaterializationConfig - ) -> BaseRelation: - """ - Produce a `BaseRelation` instance from a `MaterializationConfig` instance. This is primarily done to - reuse existing functionality based on `BaseRelation` while working with `MaterializationConfig` instances. - - Args: - materialization_config: a `MaterializationConfig` to be converted - - Returns: - a converted `BaseRelation` instance - """ - relation = self.Relation.create( - database=materialization_config.database_name, - schema=materialization_config.schema_name, - identifier=materialization_config.name, - quote_policy=self.config.quoting, - type=materialization_config.type, - ) - return relation - - @available - @classmethod - def make_backup_materialization_config( - cls, materialization_config: MaterializationConfig - ) -> MaterializationConfig: - """ - Return a copy of the materialization config, but with a backup name instead of the original name. - - Args: - materialization_config: the materialization that needs a backup - - Returns: - a renamed copy of the materialization config - """ - return dataclasses.replace( - materialization_config, name=cls.backup_name(materialization_config) - ) - - @available - @classmethod - def make_intermediate_materialization_config( - cls, materialization_config: MaterializationConfig - ) -> MaterializationConfig: """ - Return a copy of the materialization config, but with a backup name instead of the original name. - - Args: - materialization_config: the materialization that needs a backup - - Returns: - a renamed copy of the materialization config - """ - return dataclasses.replace( - materialization_config, name=cls.intermediate_name(materialization_config) - ) - - @available - def get_cached_relation_from_materialization_config( - self, materialization_config: MaterializationConfig - ) -> BaseRelation: - return self.get_relation( - database=materialization_config.database_name, - schema=materialization_config.schema_name, - identifier=materialization_config.name, - ) - - @available - def get_cached_backup_relation_from_materialization_config( - self, materialization_config: MaterializationConfig - ) -> BaseRelation: - return self.get_relation( - database=materialization_config.database_name, - schema=materialization_config.schema_name, - identifier=self.backup_name(materialization_config), - ) - - @available - def get_cached_intermediate_relation_from_materialization_config( - self, materialization_config: MaterializationConfig - ) -> BaseRelation: - return self.get_relation( - database=materialization_config.database_name, - schema=materialization_config.schema_name, - identifier=self.intermediate_name(materialization_config), - ) - - @available - @classmethod - def backup_name(cls, materialization_config: MaterializationConfig) -> str: - return f"{materialization_config.name}__dbt_backup" - - @available - @classmethod - def intermediate_name(cls, materialization_config: MaterializationConfig) -> str: - return f"{materialization_config.name}__dbt_tmp" - - @available - @classmethod - def materialization_config_from_model_node( - cls, model_node: ModelNode - ) -> MaterializationConfig: + Convenient for templating, given the mix of `BaseRelation` and `MaterializationConfig` """ - Produce a validated materialization config from the config available in the global jinja context. - - The intention is to remove validation from the jinja context and put it in python. This method gets - called in a jinja template and it's results are used in the jinja template. For an example, please - refer to `dbt/include/global_project/macros/materializations/models/materialized_view/materialization.sql`. - In this file, the relation config is retrieved right away, to ensure that the config is validated before - any sql is executed against the database. - - Args: - model_node: the `model` ModelNode instance that's in the global jinja context - - Returns: a validated adapter-specific, relation_type-specific MaterializationConfig instance - """ - relation_type = RelationType(model_node.config.materialized) - - if materialization_config := cls.materialization_configs.get(relation_type): - return materialization_config.from_model_node(model_node) - - raise DbtRuntimeError( - f"materialization_config_from_model_node() is not supported" - f" for the provided relation type: {relation_type}" - ) + return isinstance(materialization, BaseRelation) @available - @classmethod - def materialization_config_from_describe_relation_results( - cls, describe_relation_results: DescribeRelationResults, relation_type: RelationType - ) -> MaterializationConfig: + def is_materialization_config( + self, materialization: Union[BaseRelation, MaterializationConfig] + ) -> bool: """ - Produce a validated materialization config from a series of "describe "-type queries. - - The intention is to remove validation from the jinja context and put it in python. This method gets - called in a jinja template and it's results are used in the jinja template. For an example, please - refer to `dbt/include/global_project/macros/materializations/models/materialized_view/materialization.sql`. - - Args: - describe_relation_results: the results of one or more queries run against the database - to describe this relation - relation_type: the type of relation associated with the relation results - - Returns: a validated adapter-specific, relation_type-specific MaterializationConfig instance + Convenient for templating, given the mix of `BaseRelation` and `MaterializationConfig` """ - if materialization_config := cls.materialization_configs.get(relation_type): - return materialization_config.from_describe_relation_results(describe_relation_results) - - raise DbtRuntimeError( - f"materialization_config_from_describe_relation_results() is not" - f" supported for the provided relation type: {relation_type}" - ) + return isinstance(materialization, MaterializationConfig) COLUMNS_EQUAL_SQL = """ diff --git a/core/dbt/adapters/base/materialization.py b/core/dbt/adapters/base/materialization.py new file mode 100644 index 00000000000..1c9550e3abb --- /dev/null +++ b/core/dbt/adapters/base/materialization.py @@ -0,0 +1,130 @@ +import dataclasses +from typing import Dict, Type + +from dbt.adapters.materialization_config import ( + DescribeRelationResults, + MaterializationConfig, + RelationConfigChangeAction, +) +from dbt.contracts.graph.model_config import OnConfigurationChangeOption +from dbt.contracts.graph.nodes import ModelNode +from dbt.contracts.relation import Policy, RelationType as _RelationType +from dbt.exceptions import DbtRuntimeError + + +class Materialization: + """ + This class is a service layer version of `BaseRelation` that exposes `MaterializationConfig` + functionality on `BaseAdapter`. + """ + + # registers MaterializationConfigs to RelationTypes + materialization_configs: Dict[_RelationType, MaterializationConfig] = dataclasses.field( + default_factory=dict + ) + + include_policy: Policy = Policy() + quote_policy: Policy = Policy() + + # useful Enums for templating + ChangeAction: Type[RelationConfigChangeAction] = RelationConfigChangeAction + ChangeOption: Type[OnConfigurationChangeOption] = OnConfigurationChangeOption + RelationType: Type[_RelationType] = _RelationType + + @classmethod + def make_backup(cls, materialization_config: MaterializationConfig) -> MaterializationConfig: + """ + Return a copy of the materialization config, but with a backup name instead of the original name. + + Args: + materialization_config: the materialization that needs a backup + + Returns: + a renamed copy of the materialization config + """ + return dataclasses.replace( + materialization_config, name=cls.backup_name(materialization_config) + ) + + @classmethod + def make_intermediate( + cls, materialization_config: MaterializationConfig + ) -> MaterializationConfig: + """ + Return a copy of the materialization config, but with a backup name instead of the original name. + + Args: + materialization_config: the materialization that needs a backup + + Returns: + a renamed copy of the materialization config + """ + return dataclasses.replace( + materialization_config, name=cls.intermediate_name(materialization_config) + ) + + @classmethod + def backup_name(cls, materialization_config: MaterializationConfig) -> str: + """ + Mimic the macro `make_backup_relation()` for `MaterializationConfig` instances. + """ + return f"{materialization_config.name}__dbt_backup" + + @classmethod + def intermediate_name(cls, materialization_config: MaterializationConfig) -> str: + """ + Mimic the macro `make_intermediate_relation()` for `MaterializationConfig` instances + """ + return f"{materialization_config.name}__dbt_tmp" + + @classmethod + def from_model_node(cls, model_node: ModelNode) -> MaterializationConfig: + """ + Produce a validated materialization config from the config available in the global jinja context. + + The intention is to remove validation from the jinja context and put it in python. This method gets + called in a jinja template and it's results are used in the jinja template. For an example, please + refer to `dbt/include/global_project/macros/materializations/models/materialized_view/materialization.sql`. + In this file, the relation config is retrieved right away, to ensure that the config is validated before + any sql is executed against the database. + + Args: + model_node: the `model` ModelNode instance that's in the global jinja context + + Returns: a validated adapter-specific, relation_type-specific MaterializationConfig instance + """ + relation_type = _RelationType(model_node.config.materialized) + + if materialization_config := cls.materialization_configs.get(relation_type): + return materialization_config.from_model_node(model_node) + + raise DbtRuntimeError( + f"materialization_config_from_model_node() is not supported" + f" for the provided relation type: {relation_type}" + ) + + @classmethod + def from_describe_relation_results( + cls, describe_relation_results: DescribeRelationResults, relation_type: _RelationType + ) -> MaterializationConfig: + """ + Produce a validated materialization config from a series of "describe "-type queries. + + The intention is to remove validation from the jinja context and put it in python. This method gets + called in a jinja template and it's results are used in the jinja template. For an example, please + refer to `dbt/include/global_project/macros/materializations/models/materialized_view/materialization.sql`. + + Args: + describe_relation_results: the results of one or more queries run against the database + to describe this relation + relation_type: the type of relation associated with the relation results + + Returns: a validated adapter-specific, relation_type-specific MaterializationConfig instance + """ + if materialization_config := cls.materialization_configs.get(relation_type): + return materialization_config.from_describe_relation_results(describe_relation_results) + + raise DbtRuntimeError( + f"materialization_config_from_describe_relation_results() is not" + f" supported for the provided relation type: {relation_type}" + ) diff --git a/core/dbt/adapters/base/relation.py b/core/dbt/adapters/base/relation.py index 5af5ec8aa36..87c8159d19e 100644 --- a/core/dbt/adapters/base/relation.py +++ b/core/dbt/adapters/base/relation.py @@ -1,7 +1,8 @@ from collections.abc import Hashable -from dataclasses import dataclass, field +import dataclasses from typing import Any, Dict, Iterator, Optional, Set, Tuple, Type, TypeVar +from dbt.adapters.materialization_config import MaterializationConfig from dbt.contracts.graph.nodes import ( SourceDefinition, ManifestNode, @@ -30,15 +31,15 @@ Self = TypeVar("Self", bound="BaseRelation") -@dataclass(frozen=True, eq=False, repr=False) +@dataclasses.dataclass(frozen=True, eq=False, repr=False) class BaseRelation(FakeAPIObject, Hashable): path: Path type: Optional[RelationType] = None quote_character: str = '"' # Python 3.11 requires that these use default_factory instead of simple default # ValueError: mutable default for field include_policy is not allowed: use default_factory - include_policy: Policy = field(default_factory=lambda: Policy()) - quote_policy: Policy = field(default_factory=lambda: Policy()) + include_policy: Policy = dataclasses.field(default_factory=lambda: Policy()) + quote_policy: Policy = dataclasses.field(default_factory=lambda: Policy()) dbt_created: bool = False def _is_exactish_match(self, field: ComponentName, value: str) -> bool: @@ -174,10 +175,11 @@ def without_identifier(self) -> "BaseRelation": def _render_iterator(self) -> Iterator[Tuple[Optional[ComponentName], Optional[str]]]: for key in ComponentName: + component = ComponentName(key) path_part: Optional[str] = None - if self.include_policy.get_part(key): - path_part = self.path.get_part(key) - if path_part is not None and self.quote_policy.get_part(key): + if self.include_policy.get_part(component): + path_part = self.path.get_part(component) + if path_part is not None and self.quote_policy.get_part(component): path_part = self.quoted(path_part) yield key, path_part @@ -262,7 +264,7 @@ def create_from( return cls.create_from_source(node, **kwargs) else: # Can't use ManifestNode here because of parameterized generics - if not isinstance(node, (ParsedNode)): + if not isinstance(node, ParsedNode): raise DbtInternalError( f"type mismatch, expected ManifestNode but got {type(node)}" ) @@ -289,6 +291,31 @@ def create( ) return cls.from_dict(kwargs) + @classmethod + def from_materialization_config( + cls, materialization_config: MaterializationConfig + ) -> "BaseRelation": + """ + Produce a `BaseRelation` instance from a `MaterializationConfig` instance. This is primarily done to + reuse existing functionality based on `BaseRelation` while working with `MaterializationConfig` instances. + + Useful in combination with `is_materialization_config`. + + Args: + materialization_config: a `MaterializationConfig` to be converted + + Returns: + a converted `BaseRelation` instance + """ + relation = cls.create( + database=materialization_config.database_name, + schema=materialization_config.schema_name, + identifier=materialization_config.name, + quote_policy=cls.quote_policy, + type=materialization_config.type, + ) + return relation + def __repr__(self) -> str: return "<{} {}>".format(self.__class__.__name__, self.render()) @@ -363,7 +390,7 @@ def get_relation_type(cls) -> Type[RelationType]: Info = TypeVar("Info", bound="InformationSchema") -@dataclass(frozen=True, eq=False, repr=False) +@dataclasses.dataclass(frozen=True, eq=False, repr=False) class InformationSchema(BaseRelation): information_schema_view: Optional[str] = None diff --git a/core/dbt/adapters/relation_configs/README.md b/core/dbt/adapters/materialization_config/README.md similarity index 100% rename from core/dbt/adapters/relation_configs/README.md rename to core/dbt/adapters/materialization_config/README.md diff --git a/core/dbt/adapters/materialization_config/__init__.py b/core/dbt/adapters/materialization_config/__init__.py new file mode 100644 index 00000000000..1d52ac2dc4d --- /dev/null +++ b/core/dbt/adapters/materialization_config/__init__.py @@ -0,0 +1,20 @@ +from dbt.adapters.materialization_config._base import RelationConfig, DescribeRelationResults +from dbt.adapters.materialization_config._change import ( + RelationConfigChangeAction, + RelationConfigChange, + RelationConfigChangeset, +) +from dbt.adapters.materialization_config._database import DatabaseConfig +from dbt.adapters.materialization_config._materialization import MaterializationConfig +from dbt.adapters.materialization_config._policy import ( + IncludePolicy, + QuotePolicy, + conform_part, + render_part, + render, +) +from dbt.adapters.materialization_config._schema import SchemaConfig +from dbt.adapters.materialization_config._validation import ( + RelationConfigValidationMixin, + RelationConfigValidationRule, +) diff --git a/core/dbt/adapters/relation_configs/_base.py b/core/dbt/adapters/materialization_config/_base.py similarity index 100% rename from core/dbt/adapters/relation_configs/_base.py rename to core/dbt/adapters/materialization_config/_base.py diff --git a/core/dbt/adapters/relation_configs/_change.py b/core/dbt/adapters/materialization_config/_change.py similarity index 100% rename from core/dbt/adapters/relation_configs/_change.py rename to core/dbt/adapters/materialization_config/_change.py diff --git a/core/dbt/adapters/relation_configs/_database.py b/core/dbt/adapters/materialization_config/_database.py similarity index 91% rename from core/dbt/adapters/relation_configs/_database.py rename to core/dbt/adapters/materialization_config/_database.py index 27dbaf68a50..9803a4a0ce3 100644 --- a/core/dbt/adapters/relation_configs/_database.py +++ b/core/dbt/adapters/materialization_config/_database.py @@ -1,7 +1,7 @@ from abc import ABC from dataclasses import dataclass -from dbt.adapters.relation_configs._base import RelationConfig +from dbt.adapters.materialization_config._base import RelationConfig @dataclass(frozen=True, eq=True, unsafe_hash=True) diff --git a/core/dbt/adapters/relation_configs/_materialization.py b/core/dbt/adapters/materialization_config/_materialization.py similarity index 90% rename from core/dbt/adapters/relation_configs/_materialization.py rename to core/dbt/adapters/materialization_config/_materialization.py index 75983b8392f..1543fb74334 100644 --- a/core/dbt/adapters/relation_configs/_materialization.py +++ b/core/dbt/adapters/materialization_config/_materialization.py @@ -1,9 +1,9 @@ from abc import ABC from dataclasses import dataclass -from dbt.adapters.relation_configs._base import DescribeRelationResults, RelationConfig -from dbt.adapters.relation_configs._database import DatabaseConfig -from dbt.adapters.relation_configs._schema import SchemaConfig +from dbt.adapters.materialization_config._base import DescribeRelationResults, RelationConfig +from dbt.adapters.materialization_config._database import DatabaseConfig +from dbt.adapters.materialization_config._schema import SchemaConfig from dbt.contracts.graph.nodes import ModelNode from dbt.contracts.relation import RelationType diff --git a/core/dbt/adapters/relation_configs/_policy.py b/core/dbt/adapters/materialization_config/_policy.py similarity index 100% rename from core/dbt/adapters/relation_configs/_policy.py rename to core/dbt/adapters/materialization_config/_policy.py diff --git a/core/dbt/adapters/relation_configs/_schema.py b/core/dbt/adapters/materialization_config/_schema.py similarity index 86% rename from core/dbt/adapters/relation_configs/_schema.py rename to core/dbt/adapters/materialization_config/_schema.py index d9676aa7a64..098504b6c07 100644 --- a/core/dbt/adapters/relation_configs/_schema.py +++ b/core/dbt/adapters/materialization_config/_schema.py @@ -1,8 +1,8 @@ from abc import ABC from dataclasses import dataclass -from dbt.adapters.relation_configs._base import RelationConfig -from dbt.adapters.relation_configs._database import DatabaseConfig +from dbt.adapters.materialization_config._base import RelationConfig +from dbt.adapters.materialization_config._database import DatabaseConfig @dataclass(frozen=True, eq=True, unsafe_hash=True) diff --git a/core/dbt/adapters/relation_configs/_validation.py b/core/dbt/adapters/materialization_config/_validation.py similarity index 100% rename from core/dbt/adapters/relation_configs/_validation.py rename to core/dbt/adapters/materialization_config/_validation.py diff --git a/core/dbt/adapters/relation_configs/__init__.py b/core/dbt/adapters/relation_configs/__init__.py deleted file mode 100644 index c06a31a5461..00000000000 --- a/core/dbt/adapters/relation_configs/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -from dbt.adapters.relation_configs._base import RelationConfig, DescribeRelationResults -from dbt.adapters.relation_configs._change import ( - RelationConfigChangeAction, - RelationConfigChange, - RelationConfigChangeset, -) -from dbt.adapters.relation_configs._database import DatabaseConfig -from dbt.adapters.relation_configs._materialization import MaterializationConfig -from dbt.adapters.relation_configs._policy import ( - IncludePolicy, - QuotePolicy, - conform_part, - render_part, - render, -) -from dbt.adapters.relation_configs._schema import SchemaConfig -from dbt.adapters.relation_configs._validation import ( - RelationConfigValidationMixin, - RelationConfigValidationRule, -) diff --git a/core/dbt/context/providers.py b/core/dbt/context/providers.py index c4df007e806..1565cd38db7 100644 --- a/core/dbt/context/providers.py +++ b/core/dbt/context/providers.py @@ -101,6 +101,14 @@ def create(self, *args, **kwargs): return self._relation_type.create(*args, **kwargs) +class MaterializationProxy: + def __init__(self, adapter): + self._materialization = adapter.Materialization + + def __getattr__(self, key): + return getattr(self._materialization, key) + + class BaseDatabaseWrapper: """ Wrapper for runtime database interaction. Applies the runtime quote policy @@ -109,7 +117,9 @@ class BaseDatabaseWrapper: def __init__(self, adapter, namespace: MacroNamespace): self._adapter = adapter + self.Materialization = MaterializationProxy(adapter) self.Relation = RelationProxy(adapter) + self.RelationType = adapter.RelationType self._namespace = namespace def __getattr__(self, name): diff --git a/core/dbt/include/global_project/macros/materializations/materialization.sql b/core/dbt/include/global_project/macros/materializations/materialization.sql index ac60b92c563..794d50ac4a0 100644 --- a/core/dbt/include/global_project/macros/materializations/materialization.sql +++ b/core/dbt/include/global_project/macros/materializations/materialization.sql @@ -11,8 +11,8 @@ -- backup_relation and intermediate_relation should not already exist in the database -- it's possible these exist because of a previous run that exited unexpectedly - {%- set backup_relation = adapter.get_cached_backup_relation_from_materialization_config(materialization_config) -%} - {%- set intermediate_relation = adapter.get_cached_intermediate_relation_from_materialization_config(materialization_config) -%} + {%- set backup_relation = adapter.get_cached_relation(materialization_config, "backup") -%} + {%- set intermediate_relation = adapter.get_cached_relation(materialization_config, "intermediate") -%} -- drop the temp relations if they exist already in the database {{- drop_relation_if_exists(backup_relation) -}} @@ -26,8 +26,8 @@ {%- macro mat_teardown(materialization_config, post_hooks) -%} -- backup_relation and intermediate_relation may exist if the materialized view was replaced - {%- set backup_relation = adapter.get_cached_backup_relation_from_materialization_config(materialization_config) -%} - {%- set intermediate_relation = adapter.get_cached_intermediate_relation_from_materialization_config(materialization_config) -%} + {%- set backup_relation = adapter.get_cached_relation(materialization_config, "backup") -%} + {%- set intermediate_relation = adapter.get_cached_relation(materialization_config, "intermediate") -%} -- drop the temp relations if they exist to leave the database clean for the next run {{- drop_relation_if_exists(backup_relation) -}} diff --git a/core/dbt/include/global_project/macros/materializations/models/all/alter.sql b/core/dbt/include/global_project/macros/materializations/models/all/alter.sql index b3a24894602..619435c6eff 100644 --- a/core/dbt/include/global_project/macros/materializations/models/all/alter.sql +++ b/core/dbt/include/global_project/macros/materializations/models/all/alter.sql @@ -11,7 +11,7 @@ {%- set relation_type = existing_materialization.type -%} - {%- if relation_type == 'materialized_view' -%} + {%- if relation_type == adapter.RelationType.MaterializedView -%} {{ alter_materialized_view_sql(existing_materialization, new_materialization) }} {%- else -%} {{- exceptions.raise_compiler_error("`alter_sql()` has not been implemented for the relation type" ~ relation_type ) -}} diff --git a/core/dbt/include/global_project/macros/materializations/models/all/create.sql b/core/dbt/include/global_project/macros/materializations/models/all/create.sql index 588e3cd9a9f..02ec28409da 100644 --- a/core/dbt/include/global_project/macros/materializations/models/all/create.sql +++ b/core/dbt/include/global_project/macros/materializations/models/all/create.sql @@ -11,7 +11,7 @@ {%- set relation_type = materialization.type -%} - {%- if relation_type == 'materialized_view' -%} + {%- if relation_type == adapter.RelationType.MaterializedView -%} {{ create_materialized_view_sql(materialization) }} {%- else -%} {{- exceptions.raise_compiler_error("`create_sql()` has not been implemented for the relation type" ~ relation_type ) -}} diff --git a/core/dbt/include/global_project/macros/materializations/models/all/describe.sql b/core/dbt/include/global_project/macros/materializations/models/all/describe.sql index e40448f15b1..755bb4653c5 100644 --- a/core/dbt/include/global_project/macros/materializations/models/all/describe.sql +++ b/core/dbt/include/global_project/macros/materializations/models/all/describe.sql @@ -15,7 +15,7 @@ {%- set relation_type = materialization.type -%} - {%- if relation_type == 'materialized_view' -%} + {%- if relation_type == adapter.RelationType.MaterializedView -%} {%- do return(describe_materialized_view_sql(materialization)) -%} {%- else -%} {{- exceptions.raise_compiler_error("`describe_sql()` has not been implemented for the relation type" ~ relation_type ) -}} diff --git a/core/dbt/include/global_project/macros/materializations/models/all/drop.sql b/core/dbt/include/global_project/macros/materializations/models/all/drop.sql index 4ed7b2f0ac3..b7d9ea84169 100644 --- a/core/dbt/include/global_project/macros/materializations/models/all/drop.sql +++ b/core/dbt/include/global_project/macros/materializations/models/all/drop.sql @@ -2,7 +2,7 @@ {{- log('Applying DROP to: ' ~ materialization) -}} - {%- set relation = adapter.base_relation_from_materialization_config(materialization) -%} + {%- set relation = adapter.Relation.from_materialization_config(materialization) -%} {{- adapter.dispatch('drop_sql', 'dbt')(materialization) -}} @@ -15,7 +15,7 @@ {%- set relation_type = materialization.type -%} - {%- if relation_type == 'materialized_view' -%} + {%- if relation_type == adapter.RelationType.MaterializedView -%} {{ drop_materialized_view_sql(materialization) }} {%- else -%} {{- exceptions.raise_compiler_error("`drop_sql()` has not been implemented for the relation type" ~ relation_type ) -}} diff --git a/core/dbt/include/global_project/macros/materializations/models/all/rename.sql b/core/dbt/include/global_project/macros/materializations/models/all/rename.sql index 5d59e114c0c..07b62525ec1 100644 --- a/core/dbt/include/global_project/macros/materializations/models/all/rename.sql +++ b/core/dbt/include/global_project/macros/materializations/models/all/rename.sql @@ -3,7 +3,7 @@ {{- log('Applying RENAME to: ' ~ materialization) -}} {%- if adapter.is_materialization_config(materialization) -%} - {%- set from_relation = adapter.base_relation_from_materialization_config(materialization) -%} + {%- set from_relation = adapter.Relation.from_materialization_config(materialization) -%} {%- else -%} {%- set from_relation = materialization -%} {%- endif-%} @@ -20,11 +20,11 @@ {%- set relation_type = materialization.type -%} - {%- if relation_type == 'view' -%} + {%- if relation_type == adapter.RelationType.View -%} {{ rename_view_sql(materialization, new_name) }} - {%- elif relation_type == 'table' -%} + {%- elif relation_type == adapter.RelationType.Table -%} {{ rename_table_sql(materialization, new_name) }} - {%- elif relation_type == 'materialized_view' -%} + {%- elif relation_type == adapter.RelationType.MaterializedView -%} {{ rename_materialized_view_sql(materialization, new_name) }} {%- else -%} {{- exceptions.raise_compiler_error("`rename_sql()` has not been implemented for the relation type" ~ relation_type ) -}} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/materialized_view.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view/materialized_view.sql index b25e15630fc..74a7a26a56d 100644 --- a/core/dbt/include/global_project/macros/materializations/models/materialized_view/materialized_view.sql +++ b/core/dbt/include/global_project/macros/materializations/models/materialized_view/materialized_view.sql @@ -1,10 +1,10 @@ {%- materialization materialized_view, default -%} -- Try to create a valid materialized view from the config before doing anything else - {%- set new_materialized_view = adapter.materialization_config_from_model_node(config.model) -%} + {%- set new_materialized_view = adapter.Materialization.from_model_node(config.model) -%} -- We still need these because they tie into the existing process (e.g. RelationBase vs. RelationConfigBase) - {%- set existing_relation = adapter.get_cached_relation_from_materialization_config(new_materialized_view) -%} + {%- set existing_relation = adapter.get_cached_relation(new_materialized_view) -%} {{- mat_setup(new_materialized_view, pre_hooks) -}} @@ -18,7 +18,7 @@ {{- mat_teardown(new_materialized_view, post_hooks) -}} - {%- set new_relation = adapter.base_relation_from_materialization_config(new_materialized_view) -%} + {%- set new_relation = adapter.Relation.from_materialization_config(new_materialized_view) -%} {{- return({'relations': [new_relation]}) -}} {%- endmaterialization -%} @@ -43,20 +43,20 @@ {%- macro alter_materialized_view_with_on_configuration_change_option_sql(new_materialized_view) -%} {%- set describe_relation_results = describe_materialized_view_sql(new_materialized_view) -%} - {%- set existing_materialized_view = adapter.materialization_config_from_describe_relation_results( - describe_relation_results, adapter.Relation.MaterializedView + {%- set existing_materialized_view = adapter.Materialization.from_describe_relation_results( + describe_relation_results, adapter.RelationType.MaterializedView ) -%} {%- set on_configuration_change = config.get('on_configuration_change') -%} {%- if existing_materialized_view == new_materialized_view -%} {%- set build_sql = refresh_materialized_view_sql(existing_materialized_view) -%} - {%- elif on_configuration_change == 'apply' -%} + {%- elif on_configuration_change == adapter.Materialization.ChangeOption.Apply -%} {%- set build_sql = alter_sql(existing_materialized_view, new_materialized_view) -%} - {%- elif on_configuration_change == 'continue' -%} + {%- elif on_configuration_change == adapter.Materialization.ChangeOption.Continue -%} {%- set build_sql = '' -%} {{- exceptions.warn("Configuration changes were identified and `on_configuration_change` was set to `continue` for `" ~ new_materialized_view.fully_qualified_path ~ "`") -}} - {%- elif on_configuration_change == 'fail' -%} + {%- elif on_configuration_change == adapter.Materialization.ChangeOption.Fail -%} {{- exceptions.raise_fail_fast_error("Configuration changes were identified and `on_configuration_change` was set to `fail` for `" ~ new_materialized_view.fully_qualified_path ~ "`") -}} {%- else -%} diff --git a/plugins/postgres/dbt/adapters/postgres/impl.py b/plugins/postgres/dbt/adapters/postgres/impl.py index dc0cc4407a4..05a3043f867 100644 --- a/plugins/postgres/dbt/adapters/postgres/impl.py +++ b/plugins/postgres/dbt/adapters/postgres/impl.py @@ -1,13 +1,11 @@ from datetime import datetime from dataclasses import dataclass -from typing import Any, FrozenSet, List, Optional, Set +from typing import Any, List, Optional, Set from dbt.adapters.base.meta import available from dbt.adapters.base.impl import AdapterConfig, ConstraintSupport -from dbt.adapters.relation_configs import MaterializationConfig, RelationConfigChangeAction from dbt.adapters.sql import SQLAdapter from dbt.contracts.graph.nodes import ConstraintType -from dbt.contracts.relation import ComponentName, RelationType from dbt.dataclass_schema import dbtClassMixin, ValidationError from dbt.exceptions import ( CrossDbReferenceProhibitedError, @@ -20,15 +18,7 @@ from dbt.adapters.postgres import PostgresConnectionManager, PostgresRelation from dbt.adapters.postgres.column import PostgresColumn -from dbt.adapters.postgres.relation_configs import ( - PostgresIndexConfig as PostgresIndexConfigMatView, - PostgresIndexConfigChange, - PostgresMaterializedViewConfig, - PostgresMaterializedViewConfigChangeset, - PostgresIncludePolicy, - PostgresQuotePolicy, - postgres_conform_part, -) +from dbt.adapters.postgres.materialization import PostgresMaterialization # note that this isn't an adapter macro, so just a single underscore @@ -72,6 +62,7 @@ class PostgresConfig(AdapterConfig): class PostgresAdapter(SQLAdapter): Relation = PostgresRelation + Materialization = PostgresMaterialization ConnectionManager = PostgresConnectionManager Column = PostgresColumn @@ -84,9 +75,6 @@ class PostgresAdapter(SQLAdapter): ConstraintType.primary_key: ConstraintSupport.ENFORCED, ConstraintType.foreign_key: ConstraintSupport.ENFORCED, } - materialization_configs = {RelationType.MaterializedView: PostgresMaterializedViewConfig} - include_policy = PostgresIncludePolicy() - quote_policy = PostgresQuotePolicy() @classmethod def date_function(cls): @@ -158,89 +146,3 @@ def valid_incremental_strategies(self): def debug_query(self): self.execute("select 1 as id") - - @available - @classmethod - def materialized_view_config_changeset( - cls, - existing_materialized_view: PostgresMaterializedViewConfig, - new_materialized_view: PostgresMaterializedViewConfig, - ) -> PostgresMaterializedViewConfigChangeset: - try: - assert isinstance(existing_materialized_view, PostgresMaterializedViewConfig) - assert isinstance(new_materialized_view, PostgresMaterializedViewConfig) - except AssertionError: - raise DbtRuntimeError( - f"Two materialized view configs were expected, but received:" - f"/n {existing_materialized_view}" - f"/n {new_materialized_view}" - ) - - config_changeset = PostgresMaterializedViewConfigChangeset() - - config_changeset.indexes = cls.index_config_changeset( - existing_materialized_view.indexes, new_materialized_view.indexes - ) - - if config_changeset.is_empty and existing_materialized_view != new_materialized_view: - # we need to force a full refresh if we didn't detect any changes but the objects are not the same - config_changeset.force_full_refresh() - - return config_changeset - - @available - @classmethod - def index_config_changeset( - cls, - existing_indexes: FrozenSet[PostgresIndexConfigMatView], - new_indexes: FrozenSet[PostgresIndexConfigMatView], - ) -> Set[PostgresIndexConfigChange]: - """ - Get the index updates that will occur as a result of a new run - - There are four scenarios: - - 1. Indexes are equal -> don't return these - 2. Index is new -> create these - 3. Index is old -> drop these - 4. Indexes are not equal -> drop old, create new -> two actions - - Returns: a set of index updates in the form {"action": "drop/create", "context": } - """ - drop_changes = set( - PostgresIndexConfigChange(action=RelationConfigChangeAction.drop, context=index) - for index in existing_indexes.difference(new_indexes) - ) - create_changes = set( - PostgresIndexConfigChange(action=RelationConfigChangeAction.create, context=index) - for index in new_indexes.difference(existing_indexes) - ) - return set().union(drop_changes, create_changes) - - @available - @classmethod - def generate_index_name( - cls, - materialization_config: MaterializationConfig, - index_config: PostgresIndexConfigMatView, - ) -> str: - return dbt.utils.md5( - "_".join( - { - postgres_conform_part( - ComponentName.Database, materialization_config.database_name - ), - postgres_conform_part( - ComponentName.Schema, materialization_config.schema_name - ), - postgres_conform_part(ComponentName.Identifier, materialization_config.name), - *sorted( - postgres_conform_part(ComponentName.Identifier, column) - for column in index_config.column_names - ), - str(index_config.unique), - str(index_config.method), - str(datetime.utcnow().isoformat()), - } - ) - ) diff --git a/plugins/postgres/dbt/adapters/postgres/materialization.py b/plugins/postgres/dbt/adapters/postgres/materialization.py new file mode 100644 index 00000000000..726777cc092 --- /dev/null +++ b/plugins/postgres/dbt/adapters/postgres/materialization.py @@ -0,0 +1,110 @@ +from datetime import datetime +from typing import FrozenSet, Set + +from dbt.adapters.base.materialization import Materialization +from dbt.adapters.materialization_config import ( + MaterializationConfig, + RelationConfigChangeAction, +) +from dbt.contracts.relation import ComponentName, RelationType +from dbt.exceptions import DbtRuntimeError +import dbt.utils + +from dbt.adapters.postgres.materialization_config import ( + PostgresIncludePolicy, + PostgresIndexConfig, + PostgresIndexConfigChange, + PostgresMaterializedViewConfig, + PostgresMaterializedViewConfigChangeset, + PostgresQuotePolicy, + postgres_conform_part, +) + + +class PostgresMaterialization(Materialization): + materialization_configs = {RelationType.MaterializedView: PostgresMaterializedViewConfig} + include_policy = PostgresIncludePolicy() + quote_policy = PostgresQuotePolicy() + + @classmethod + def materialized_view_config_changeset( + cls, + existing_materialized_view: PostgresMaterializedViewConfig, + new_materialized_view: PostgresMaterializedViewConfig, + ) -> PostgresMaterializedViewConfigChangeset: + try: + assert isinstance(existing_materialized_view, PostgresMaterializedViewConfig) + assert isinstance(new_materialized_view, PostgresMaterializedViewConfig) + except AssertionError: + raise DbtRuntimeError( + f"Two materialized view configs were expected, but received:" + f"/n {existing_materialized_view}" + f"/n {new_materialized_view}" + ) + + config_changeset = PostgresMaterializedViewConfigChangeset() + + config_changeset.indexes = cls.index_config_changeset( + existing_materialized_view.indexes, new_materialized_view.indexes + ) + + if config_changeset.is_empty and existing_materialized_view != new_materialized_view: + # we need to force a full refresh if we didn't detect any changes but the objects are not the same + config_changeset.force_full_refresh() + + return config_changeset + + @classmethod + def index_config_changeset( + cls, + existing_indexes: FrozenSet[PostgresIndexConfig], + new_indexes: FrozenSet[PostgresIndexConfig], + ) -> Set[PostgresIndexConfigChange]: + """ + Get the index updates that will occur as a result of a new run + + There are four scenarios: + + 1. Indexes are equal -> don't return these + 2. Index is new -> create these + 3. Index is old -> drop these + 4. Indexes are not equal -> drop old, create new -> two actions + + Returns: a set of index updates in the form {"action": "drop/create", "context": } + """ + drop_changes = set( + PostgresIndexConfigChange(action=RelationConfigChangeAction.drop, context=index) + for index in existing_indexes.difference(new_indexes) + ) + create_changes = set( + PostgresIndexConfigChange(action=RelationConfigChangeAction.create, context=index) + for index in new_indexes.difference(existing_indexes) + ) + return set().union(drop_changes, create_changes) + + @classmethod + def generate_index_name( + cls, + materialization_config: MaterializationConfig, + index_config: PostgresIndexConfig, + ) -> str: + return dbt.utils.md5( + "_".join( + { + postgres_conform_part( + ComponentName.Database, materialization_config.database_name + ), + postgres_conform_part( + ComponentName.Schema, materialization_config.schema_name + ), + postgres_conform_part(ComponentName.Identifier, materialization_config.name), + *sorted( + postgres_conform_part(ComponentName.Identifier, column) + for column in index_config.column_names + ), + str(index_config.unique), + str(index_config.method), + str(datetime.utcnow().isoformat()), + } + ) + ) diff --git a/plugins/postgres/dbt/adapters/postgres/relation_configs/__init__.py b/plugins/postgres/dbt/adapters/postgres/materialization_config/__init__.py similarity index 59% rename from plugins/postgres/dbt/adapters/postgres/relation_configs/__init__.py rename to plugins/postgres/dbt/adapters/postgres/materialization_config/__init__.py index f34e59ddc07..baa2e205cd1 100644 --- a/plugins/postgres/dbt/adapters/postgres/relation_configs/__init__.py +++ b/plugins/postgres/dbt/adapters/postgres/materialization_config/__init__.py @@ -1,12 +1,12 @@ -from dbt.adapters.postgres.relation_configs.index import ( +from dbt.adapters.postgres.materialization_config.index import ( PostgresIndexConfig, PostgresIndexConfigChange, ) -from dbt.adapters.postgres.relation_configs.materialized_view import ( +from dbt.adapters.postgres.materialization_config.materialized_view import ( PostgresMaterializedViewConfig, PostgresMaterializedViewConfigChangeset, ) -from dbt.adapters.postgres.relation_configs.policy import ( +from dbt.adapters.postgres.materialization_config.policy import ( MAX_CHARACTERS_IN_IDENTIFIER, PostgresIncludePolicy, PostgresQuotePolicy, diff --git a/plugins/postgres/dbt/adapters/postgres/relation_configs/database.py b/plugins/postgres/dbt/adapters/postgres/materialization_config/database.py similarity index 93% rename from plugins/postgres/dbt/adapters/postgres/relation_configs/database.py rename to plugins/postgres/dbt/adapters/postgres/materialization_config/database.py index ea8509d66ff..b6f42bd9071 100644 --- a/plugins/postgres/dbt/adapters/postgres/relation_configs/database.py +++ b/plugins/postgres/dbt/adapters/postgres/materialization_config/database.py @@ -3,7 +3,7 @@ from typing import Set import agate -from dbt.adapters.relation_configs import ( +from dbt.adapters.materialization_config import ( DatabaseConfig, RelationConfigValidationMixin, RelationConfigValidationRule, @@ -12,7 +12,10 @@ from dbt.contracts.relation import ComponentName from dbt.exceptions import DbtRuntimeError -from dbt.adapters.postgres.relation_configs.policy import postgres_render, postgres_conform_part +from dbt.adapters.postgres.materialization_config.policy import ( + postgres_render, + postgres_conform_part, +) @dataclass(frozen=True, eq=True, unsafe_hash=True) diff --git a/plugins/postgres/dbt/adapters/postgres/relation_configs/index.py b/plugins/postgres/dbt/adapters/postgres/materialization_config/index.py similarity index 97% rename from plugins/postgres/dbt/adapters/postgres/relation_configs/index.py rename to plugins/postgres/dbt/adapters/postgres/materialization_config/index.py index 23c94dba1b9..8fa89f3000b 100644 --- a/plugins/postgres/dbt/adapters/postgres/relation_configs/index.py +++ b/plugins/postgres/dbt/adapters/postgres/materialization_config/index.py @@ -2,7 +2,7 @@ from typing import Set, FrozenSet import agate -from dbt.adapters.relation_configs import ( +from dbt.adapters.materialization_config import ( RelationConfig, RelationConfigValidationMixin, RelationConfigValidationRule, @@ -13,7 +13,7 @@ from dbt.dataclass_schema import StrEnum from dbt.exceptions import DbtRuntimeError -from dbt.adapters.postgres.relation_configs.policy import postgres_conform_part +from dbt.adapters.postgres.materialization_config.policy import postgres_conform_part class PostgresIndexMethod(StrEnum): diff --git a/plugins/postgres/dbt/adapters/postgres/relation_configs/materialized_view.py b/plugins/postgres/dbt/adapters/postgres/materialization_config/materialized_view.py similarity index 96% rename from plugins/postgres/dbt/adapters/postgres/relation_configs/materialized_view.py rename to plugins/postgres/dbt/adapters/postgres/materialization_config/materialized_view.py index 1c0df94d87d..0750029ba4d 100644 --- a/plugins/postgres/dbt/adapters/postgres/relation_configs/materialized_view.py +++ b/plugins/postgres/dbt/adapters/postgres/materialization_config/materialized_view.py @@ -3,7 +3,7 @@ from typing import Set, FrozenSet, List, Dict, Optional import agate -from dbt.adapters.relation_configs import ( +from dbt.adapters.materialization_config import ( MaterializationConfig, RelationConfigChangeset, RelationConfigValidationMixin, @@ -13,16 +13,16 @@ from dbt.contracts.relation import ComponentName, RelationType from dbt.exceptions import DbtRuntimeError -from dbt.adapters.postgres.relation_configs.index import ( +from dbt.adapters.postgres.materialization_config.index import ( PostgresIndexConfig, PostgresIndexConfigChange, ) -from dbt.adapters.postgres.relation_configs.policy import ( +from dbt.adapters.postgres.materialization_config.policy import ( postgres_render, postgres_conform_part, MAX_CHARACTERS_IN_IDENTIFIER, ) -from dbt.adapters.postgres.relation_configs.schema import PostgresSchemaConfig +from dbt.adapters.postgres.materialization_config.schema import PostgresSchemaConfig @dataclass(frozen=True, eq=True, unsafe_hash=True) diff --git a/plugins/postgres/dbt/adapters/postgres/relation_configs/policy.py b/plugins/postgres/dbt/adapters/postgres/materialization_config/policy.py similarity index 95% rename from plugins/postgres/dbt/adapters/postgres/relation_configs/policy.py rename to plugins/postgres/dbt/adapters/postgres/materialization_config/policy.py index 30eada2a42a..963f57742a0 100644 --- a/plugins/postgres/dbt/adapters/postgres/relation_configs/policy.py +++ b/plugins/postgres/dbt/adapters/postgres/materialization_config/policy.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from functools import partial -from dbt.adapters.relation_configs import ( +from dbt.adapters.materialization_config import ( IncludePolicy, QuotePolicy, conform_part, diff --git a/plugins/postgres/dbt/adapters/postgres/relation_configs/schema.py b/plugins/postgres/dbt/adapters/postgres/materialization_config/schema.py similarity index 90% rename from plugins/postgres/dbt/adapters/postgres/relation_configs/schema.py rename to plugins/postgres/dbt/adapters/postgres/materialization_config/schema.py index 093c301fff4..b6d08788637 100644 --- a/plugins/postgres/dbt/adapters/postgres/relation_configs/schema.py +++ b/plugins/postgres/dbt/adapters/postgres/materialization_config/schema.py @@ -3,7 +3,7 @@ from typing import Set import agate -from dbt.adapters.relation_configs import ( +from dbt.adapters.materialization_config import ( SchemaConfig, RelationConfigValidationMixin, RelationConfigValidationRule, @@ -12,8 +12,11 @@ from dbt.contracts.relation import ComponentName from dbt.exceptions import DbtRuntimeError -from dbt.adapters.postgres.relation_configs.database import PostgresDatabaseConfig -from dbt.adapters.postgres.relation_configs.policy import postgres_render, postgres_conform_part +from dbt.adapters.postgres.materialization_config.database import PostgresDatabaseConfig +from dbt.adapters.postgres.materialization_config.policy import ( + postgres_render, + postgres_conform_part, +) @dataclass(frozen=True, eq=True, unsafe_hash=True) diff --git a/plugins/postgres/dbt/adapters/postgres/relation.py b/plugins/postgres/dbt/adapters/postgres/relation.py index b2e0d6d93e8..db85b9e13ba 100644 --- a/plugins/postgres/dbt/adapters/postgres/relation.py +++ b/plugins/postgres/dbt/adapters/postgres/relation.py @@ -3,11 +3,18 @@ from dbt.adapters.base.relation import BaseRelation from dbt.exceptions import DbtRuntimeError -from dbt.adapters.postgres.relation_configs import MAX_CHARACTERS_IN_IDENTIFIER +from dbt.adapters.postgres.materialization_config import ( + PostgresIncludePolicy, + PostgresQuotePolicy, + MAX_CHARACTERS_IN_IDENTIFIER, +) @dataclass(frozen=True, eq=False, repr=False) class PostgresRelation(BaseRelation): + include_policy = PostgresIncludePolicy() + quote_policy = PostgresQuotePolicy() + def __post_init__(self): # Check for length of Postgres table/view names. # Check self.type to exclude test relation identifiers diff --git a/plugins/postgres/dbt/include/postgres/macros/adapters.sql b/plugins/postgres/dbt/include/postgres/macros/adapters.sql index 64520ffb945..816663a0637 100644 --- a/plugins/postgres/dbt/include/postgres/macros/adapters.sql +++ b/plugins/postgres/dbt/include/postgres/macros/adapters.sql @@ -221,32 +221,6 @@ {% endmacro %} -{% macro postgres__get_show_indexes_sql(relation) %} - select - i.relname as name, - m.amname as method, - ix.indisunique as "unique", - array_to_string(array_agg(a.attname), ',') as column_names - from pg_index ix - join pg_class i - on i.oid = ix.indexrelid - join pg_am m - on m.oid=i.relam - join pg_class t - on t.oid = ix.indrelid - join pg_namespace n - on n.oid = t.relnamespace - join pg_attribute a - on a.attrelid = t.oid - and a.attnum = ANY(ix.indkey) - where t.relname ilike '{{ relation.identifier }}' - and n.nspname ilike '{{ relation.schema }}' - and t.relkind in ('r', 'm') - group by 1, 2, 3 - order by 1, 2, 3 -{% endmacro %} - - {%- macro postgres__get_drop_index_sql(relation, index_name) -%} drop index if exists "{{ index_name }}" {%- endmacro -%} diff --git a/plugins/postgres/dbt/include/postgres/macros/index.sql b/plugins/postgres/dbt/include/postgres/macros/index.sql index e19da6052e4..680474a58f1 100644 --- a/plugins/postgres/dbt/include/postgres/macros/index.sql +++ b/plugins/postgres/dbt/include/postgres/macros/index.sql @@ -6,6 +6,7 @@ the basic interactions dbt-postgres requires of indexes in Postgres: - ALTER - CREATE + - DESCRIBE - DROP These macros all take a PostgresIndexConfig instance and/or a MaterializationConfigBase as an input. These classes can be found in the following files, respectively: @@ -26,10 +27,10 @@ {%- for _index_change in index_changeset -%} {%- set _index_config = _index_change.context -%} - {%- if _index_change.action == adapter.relation_config_change_action.drop -%} + {%- if _index_change.action == adapter.Materialization.ChangeAction.drop -%} {{ postgres__drop_index_sql(_index_config) }}; - {%- elif _index_change.action == adapter.relation_config_change_action.create -%} + {%- elif _index_change.action == adapter.Materialization.ChangeAction.create -%} {{ postgres__create_index_sql(materialization_config, _index_config) }}; {%- endif -%} @@ -49,7 +50,7 @@ {% macro postgres__create_index_sql(materialization_config, index_config) -%} - {%- set _index_name = adapter.generate_index_name(materialization_config, index_config) -%} + {%- set _index_name = adapter.Materialization.generate_index_name(materialization_config, index_config) -%} create {% if index_config.unique -%}unique{%- endif %} index if not exists "{{ _index_name }}" on {{ materialization_config.fully_qualified_path }} @@ -61,6 +62,39 @@ {%- endmacro %} +{% macro postgres__describe_indexes_sql(materialization) %} + {%- if adapter.is_materialization_config(materialization) -%} + {%- set _name = materialization.name %} + {%- set _schema = materialization.schema_name %} + {%- else -%} + {%- set _name = materialization.identifier %} + {%- set _schema = materialization.schema %} + {%- endif -%} + select + i.relname as name, + m.amname as method, + ix.indisunique as "unique", + array_to_string(array_agg(a.attname), ',') as column_names + from pg_index ix + join pg_class i + on i.oid = ix.indexrelid + join pg_am m + on m.oid=i.relam + join pg_class t + on t.oid = ix.indrelid + join pg_namespace n + on n.oid = t.relnamespace + join pg_attribute a + on a.attrelid = t.oid + and a.attnum = ANY(ix.indkey) + where t.relname ilike '{{ _name }}' + and n.nspname ilike '{{ _schema }}' + and t.relkind in ('r', 'm') + group by 1, 2, 3 + order by 1, 2, 3 +{% endmacro %} + + {% macro postgres__drop_index_sql(index_config) -%} drop index if exists "{{ index_config.name }}" cascade; {%- endmacro %} diff --git a/plugins/postgres/dbt/include/postgres/macros/materializations/materialized_view.sql b/plugins/postgres/dbt/include/postgres/macros/materializations/materialized_view.sql index d82870597c8..c48a0a956a5 100644 --- a/plugins/postgres/dbt/include/postgres/macros/materializations/materialized_view.sql +++ b/plugins/postgres/dbt/include/postgres/macros/materializations/materialized_view.sql @@ -28,7 +28,7 @@ in the changeset requires a full refresh or if an unmonitored change was detected) or if we can get away with altering the dynamic table in place. */ -#} - {%- set _config_changeset = adapter.materialized_view_config_changeset(existing_materialized_view, new_materialized_view) -%} + {%- set _config_changeset = adapter.Materialization.materialized_view_config_changeset(existing_materialized_view, new_materialized_view) -%} {%- if _config_changeset.requires_full_refresh -%} {{ replace_materialized_view_sql(new_materialized_view) }} @@ -66,30 +66,7 @@ {%- endset -%} {%- set _materialized_view = run_query(_materialized_view_sql) -%} - {%- set _indexes_sql -%} - select - i.relname as name, - m.amname as method, - ix.indisunique as "unique", - array_to_string(array_agg(a.attname), ',') as column_names - from pg_index ix - join pg_class i - on i.oid = ix.indexrelid - join pg_am m - on m.oid=i.relam - join pg_class t - on t.oid = ix.indrelid - join pg_namespace n - on n.oid = t.relnamespace - join pg_attribute a - on a.attrelid = t.oid - and a.attnum = ANY(ix.indkey) - where t.relname ilike '{{ materialized_view.name }}' - and n.nspname ilike '{{ materialized_view.schema_name }}' - and t.relkind = 'm' - group by 1, 2, 3 - order by 1, 2, 3 - {%- endset -%} + {%- set _indexes_sql = postgres__describe_indexes_sql(materialized_view) -%} {%- set _indexes = run_query(_indexes_sql) -%} {%- do return({'materialized_view': _materialized_view, 'indexes': _indexes}) -%} diff --git a/plugins/postgres/dbt/include/postgres/macros/materializations/replace.sql b/plugins/postgres/dbt/include/postgres/macros/materializations/replace.sql index c596c693e03..6725793a92a 100644 --- a/plugins/postgres/dbt/include/postgres/macros/materializations/replace.sql +++ b/plugins/postgres/dbt/include/postgres/macros/materializations/replace.sql @@ -1,7 +1,7 @@ {%- macro postgres__replace_sql(existing_materialization, new_materialization) -%} {#- new_materialization is always a MaterializationConfig instance -#} - {%- set intermediate_materialization = adapter.make_intermediate_materialization_config(new_materialization) -%} + {%- set intermediate_materialization = adapter.Materialization.make_intermediate(new_materialization) -%} {%- set new_name = new_materialization.name -%} From f9bcd8c8c7b7608de4354efef3a44f88c23525df Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 29 Jun 2023 15:21:42 -0400 Subject: [PATCH 09/28] updated docs --- .../adapters/materialization_config/README.md | 43 ++++++++++++++----- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/core/dbt/adapters/materialization_config/README.md b/core/dbt/adapters/materialization_config/README.md index 7550ffe5856..55a7b58a3a4 100644 --- a/core/dbt/adapters/materialization_config/README.md +++ b/core/dbt/adapters/materialization_config/README.md @@ -1,4 +1,4 @@ -# Relation Configs +# Materialization Configs This package serves as an initial abstraction for managing the inspection of existing relations and determining changes on those relations. It arose from the materialized view work and is currently only supporting materialized views for Postgres, Redshift, and BigQuery as well as dynamic tables for Snowflake. There are three main @@ -6,7 +6,7 @@ classes in this package. ## Base (Data)Classes These are the core set of classes required to describe, validate, and monitor changes on database objects. All -other classes in `relation_configs` subclass from one of these classes. +other classes in `materialization_config` subclass from one of these classes. ### RelationConfig This class holds the primary parsing methods required for marshalling data from a user config or a database metadata @@ -56,7 +56,7 @@ object. ### MaterializationConfig This is the pearl in the sand. dbt generally interacts at the materialization level. As an adapter maintainer, you'll -need to subclass from most, if not all, objects in `relation_configs`; however you're likely doing so in order +need to subclass from most, if not all, objects in `materialization_config`; however you're likely doing so in order to work with a `MaterializationConfig` subclass. This class has several helper methods that make it easier to template sql in jinja. @@ -77,14 +77,35 @@ docstrings on these functions for more detail: - `render_part` - `render` +### Materialization +This is a new class that serves as a service layer to expose `MaterializationConfig` functionality in the +jinja context. We're effectively tucking away the modelling of database objects in python and only exposing +class methods to serve as a basic API into `MaterializationConfig`. + +- `make_backup` - create a backup materialization given an existing materialization +- `make_intermediate` - create an intermediate materialization given a target materialization +- `backup_name` - get the name that will be used in `make_backup` +- `intermediate_name` - get the name that will be used in `make_intermediate` +- `from_model_node` - build a `MaterializationConfig` from a `ModelNode` (`config.model` in jinja) +- `from_describe_relation_results` - build a `MaterializationConfig` from the database query results +- `materialization_configs` - a `dict` that registers a `MaterializationConfig` to a `RelationType` + ### BaseRelation -There are also a handful of methods on `BaseRelation` that are meant to be used with this subpackage: +There is a new method on `BaseRelation` that is meant to be used to interact with `MaterializationConfig`: + +- `from_materialization_config` - build a `BaseRelation` from a `MaterializationConfig`; useful for using existing +functionality with the new structure + +### BaseAdapter +The new `Materialization` object is registered on `BaseAdapter` and can be used in a similar fashion in jinja +as `BaseRelation` is used. There are also a few new helper methods: -- `from_model_node` -- `from_describe_relation_results` -- `relation_configs` (a lookup that gets consumed by the two methods above) +- `get_cached_relation` - same as `get_relation`, but for `MaterializationConfig` +- `is_base_relation` - determines if the object is a `BaseRelation` instance +- `is_materialization_config` - determines if the object is a `MaterializationConfig`, usually paired with +`BaseRelation.from_materialization_config` to use existing functionality -These are effectively for creating `MaterializationConfig` subclasses given data from either the `model` -attribute in the global jinja context or from data from the database. Ultimately, we're treating -`BaseAdapter` as a service layer that gets exposed in the jinja context, and tucking everything else into this -subpackage. +It should be noted that `BaseAdapter.is_materialization_config` and `BaseRelation.from_materialization_config` +can be used to "merge" `BaseRelation` instances and `MaterializationConfig` instances into the same signature +in a jinja macro. This makes it so that you only need one macro, and can determine the pieces you need once you +get there. Generally speaking, you only need identifiers, schema names, types, etc. for templating anyway. From 82b54cbf53adc0c93d1d3557f353ea9bc53f1553 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 10 Jul 2023 20:24:38 -0400 Subject: [PATCH 10/28] create relation-materialization framework - final draft --- core/dbt/adapters/base/impl.py | 186 +++++++-- core/dbt/adapters/base/materialization.py | 130 ------- core/dbt/adapters/base/relation.py | 30 +- .../README.md | 0 core/dbt/adapters/materialization/__init__.py | 1 + core/dbt/adapters/materialization/factory.py | 37 ++ .../materialization/models/__init__.py | 6 + .../models/_materialization.py | 143 +++++++ .../models/_materialized_view.py | 55 +++ .../materialization_config/__init__.py | 20 - .../adapters/materialization_config/_base.py | 137 ------- .../materialization_config/_database.py | 23 -- .../_materialization.py | 77 ---- .../materialization_config/_policy.py | 119 ------ .../materialization_config/_schema.py | 29 -- core/dbt/adapters/relation/__init__.py | 1 + core/dbt/adapters/relation/factory.py | 108 ++++++ core/dbt/adapters/relation/models/__init__.py | 22 ++ .../models}/_change.py | 50 ++- .../dbt/adapters/relation/models/_database.py | 82 ++++ .../relation/models/_materialized_view.py | 122 ++++++ core/dbt/adapters/relation/models/_policy.py | 73 ++++ .../dbt/adapters/relation/models/_relation.py | 65 ++++ .../relation/models/_relation_component.py | 156 ++++++++ .../relation/models/_relation_stub.py | 99 +++++ core/dbt/adapters/relation/models/_schema.py | 121 ++++++ .../_validation.py => validation.py} | 19 +- core/dbt/context/providers.py | 11 +- core/dbt/flags.py | 2 +- .../macros/adapters/drop_relation.sql | 35 -- .../macros/adapters/relation.sql | 19 - .../materializations/materialization.sql | 59 +-- .../materializations/materialized_view.sql | 68 ++++ .../materializations/models/all/alter.sql | 20 - .../materializations/models/all/create.sql | 20 - .../materializations/models/all/describe.sql | 24 -- .../materializations/models/all/drop.sql | 24 -- .../materializations/models/all/rename.sql | 33 -- .../materializations/models/all/replace.sql | 16 - .../models/materialized_view/alter.sql | 8 - .../models/materialized_view/create.sql | 8 - .../models/materialized_view/describe.sql | 12 - .../models/materialized_view/drop.sql | 25 -- .../materialized_view/materialized_view.sql | 70 ---- .../models/materialized_view/refresh.sql | 16 - .../models/materialized_view/rename.sql | 8 - .../materializations/models/table/rename.sql | 8 - .../materializations/models/view/rename.sql | 8 - .../index/create.sql} | 18 - .../macros/relation_components/index/drop.sql | 7 + .../macros/relation_components/index/show.sql | 7 + .../schema/create.sql} | 11 - .../relation_components/schema/drop.sql | 9 + .../global_project/macros/relations/README.md | 51 +++ .../macros/relations/atomic/alter.sql | 19 + .../macros/relations/atomic/create.sql | 21 + .../macros/relations/atomic/describe.sql | 23 ++ .../macros/relations/atomic/drop.sql | 61 +++ .../macros/relations/atomic/refresh.sql | 19 + .../macros/relations/atomic/rename.sql | 42 ++ .../macros/relations/composite/backup.sql | 19 + .../relations/composite/deploy_stage.sql | 16 + .../relations/composite/drop_backup.sql | 16 + .../macros/relations/composite/replace.sql | 66 ++++ .../macros/relations/composite/stage.sql | 19 + .../relations/materialized_view/alter.sql | 8 + .../relations/materialized_view/create.sql | 8 + .../relations/materialized_view/describe.sql | 12 + .../relations/materialized_view/drop.sql | 7 + .../relations/materialized_view/refresh.sql | 8 + .../relations/materialized_view/rename.sql | 7 + .../macros/relations/table/drop.sql | 7 + .../macros/relations/table/rename.sql | 7 + .../macros/relations/view/drop.sql | 7 + .../macros/relations/view/rename.sql | 7 + .../dbt/adapters/postgres/__init__.py | 7 +- .../postgres/dbt/adapters/postgres/impl.py | 34 +- .../dbt/adapters/postgres/materialization.py | 110 ------ .../materialization_config/__init__.py | 16 - .../materialization_config/database.py | 71 ---- .../materialized_view.py | 216 ----------- .../postgres/materialization_config/schema.py | 80 ---- .../{relation.py => relation/__init__.py} | 2 +- .../postgres/relation/models/__init__.py | 17 + .../postgres/relation/models/database.py | 46 +++ .../models}/index.py | 126 ++++-- .../relation/models/materialized_view.py | 236 ++++++++++++ .../models}/policy.py | 24 +- .../postgres/relation/models/schema.py | 49 +++ .../dbt/include/postgres/macros/adapters.sql | 57 --- .../macros/{catalog.sql => get_catalog.sql} | 0 .../include/postgres/macros/get_relations.sql | 77 ++++ .../dbt/include/postgres/macros/index.sql | 100 ----- .../macros/materializations/replace.sql | 25 -- .../macros/materializations/table.sql | 8 - .../postgres/macros/materializations/view.sql | 8 - .../macros/relation_components/index.sql | 124 ++++++ .../macros/relation_components/schema.sql | 42 ++ .../dbt/include/postgres/macros/relations.sql | 76 ---- .../materialized_view.sql | 51 +-- .../postgres/macros/relations/table.sql | 8 + .../postgres/macros/relations/view.sql | 8 + .../macros/{ => utils}/timestamps.sql | 0 .../materialized_view_tests/conftest.py | 58 +++ .../materialized_view_tests/files.py | 31 ++ .../materialized_view_tests/fixtures.py | 67 ---- .../test_materialized_view.py | 362 +++++++++++------- .../materialized_view_tests/utils.py | 80 ++++ tests/unit/dbt_postgres_tests/conftest.py | 198 ++++++++++ .../test_materialization.py | 35 ++ .../test_materialization_factory.py | 36 ++ .../model_tests/test_database.py | 24 ++ .../relation_tests/model_tests/test_index.py | 24 ++ .../model_tests/test_materialized_view.py | 112 ++++++ .../relation_tests/model_tests/test_schema.py | 31 ++ .../relation_tests/test_relation_factory.py | 79 ++++ 116 files changed, 3534 insertions(+), 2128 deletions(-) delete mode 100644 core/dbt/adapters/base/materialization.py rename core/dbt/adapters/{materialization_config => materialization}/README.md (100%) create mode 100644 core/dbt/adapters/materialization/__init__.py create mode 100644 core/dbt/adapters/materialization/factory.py create mode 100644 core/dbt/adapters/materialization/models/__init__.py create mode 100644 core/dbt/adapters/materialization/models/_materialization.py create mode 100644 core/dbt/adapters/materialization/models/_materialized_view.py delete mode 100644 core/dbt/adapters/materialization_config/__init__.py delete mode 100644 core/dbt/adapters/materialization_config/_base.py delete mode 100644 core/dbt/adapters/materialization_config/_database.py delete mode 100644 core/dbt/adapters/materialization_config/_materialization.py delete mode 100644 core/dbt/adapters/materialization_config/_policy.py delete mode 100644 core/dbt/adapters/materialization_config/_schema.py create mode 100644 core/dbt/adapters/relation/__init__.py create mode 100644 core/dbt/adapters/relation/factory.py create mode 100644 core/dbt/adapters/relation/models/__init__.py rename core/dbt/adapters/{materialization_config => relation/models}/_change.py (59%) create mode 100644 core/dbt/adapters/relation/models/_database.py create mode 100644 core/dbt/adapters/relation/models/_materialized_view.py create mode 100644 core/dbt/adapters/relation/models/_policy.py create mode 100644 core/dbt/adapters/relation/models/_relation.py create mode 100644 core/dbt/adapters/relation/models/_relation_component.py create mode 100644 core/dbt/adapters/relation/models/_relation_stub.py create mode 100644 core/dbt/adapters/relation/models/_schema.py rename core/dbt/adapters/{materialization_config/_validation.py => validation.py} (76%) delete mode 100644 core/dbt/include/global_project/macros/adapters/drop_relation.sql create mode 100644 core/dbt/include/global_project/macros/materializations/materialized_view.sql delete mode 100644 core/dbt/include/global_project/macros/materializations/models/all/alter.sql delete mode 100644 core/dbt/include/global_project/macros/materializations/models/all/create.sql delete mode 100644 core/dbt/include/global_project/macros/materializations/models/all/describe.sql delete mode 100644 core/dbt/include/global_project/macros/materializations/models/all/drop.sql delete mode 100644 core/dbt/include/global_project/macros/materializations/models/all/rename.sql delete mode 100644 core/dbt/include/global_project/macros/materializations/models/all/replace.sql delete mode 100644 core/dbt/include/global_project/macros/materializations/models/materialized_view/alter.sql delete mode 100644 core/dbt/include/global_project/macros/materializations/models/materialized_view/create.sql delete mode 100644 core/dbt/include/global_project/macros/materializations/models/materialized_view/describe.sql delete mode 100644 core/dbt/include/global_project/macros/materializations/models/materialized_view/drop.sql delete mode 100644 core/dbt/include/global_project/macros/materializations/models/materialized_view/materialized_view.sql delete mode 100644 core/dbt/include/global_project/macros/materializations/models/materialized_view/refresh.sql delete mode 100644 core/dbt/include/global_project/macros/materializations/models/materialized_view/rename.sql delete mode 100644 core/dbt/include/global_project/macros/materializations/models/table/rename.sql delete mode 100644 core/dbt/include/global_project/macros/materializations/models/view/rename.sql rename core/dbt/include/global_project/macros/{adapters/indexes.sql => relation_components/index/create.sql} (52%) create mode 100644 core/dbt/include/global_project/macros/relation_components/index/drop.sql create mode 100644 core/dbt/include/global_project/macros/relation_components/index/show.sql rename core/dbt/include/global_project/macros/{adapters/schema.sql => relation_components/schema/create.sql} (50%) create mode 100644 core/dbt/include/global_project/macros/relation_components/schema/drop.sql create mode 100644 core/dbt/include/global_project/macros/relations/README.md create mode 100644 core/dbt/include/global_project/macros/relations/atomic/alter.sql create mode 100644 core/dbt/include/global_project/macros/relations/atomic/create.sql create mode 100644 core/dbt/include/global_project/macros/relations/atomic/describe.sql create mode 100644 core/dbt/include/global_project/macros/relations/atomic/drop.sql create mode 100644 core/dbt/include/global_project/macros/relations/atomic/refresh.sql create mode 100644 core/dbt/include/global_project/macros/relations/atomic/rename.sql create mode 100644 core/dbt/include/global_project/macros/relations/composite/backup.sql create mode 100644 core/dbt/include/global_project/macros/relations/composite/deploy_stage.sql create mode 100644 core/dbt/include/global_project/macros/relations/composite/drop_backup.sql create mode 100644 core/dbt/include/global_project/macros/relations/composite/replace.sql create mode 100644 core/dbt/include/global_project/macros/relations/composite/stage.sql create mode 100644 core/dbt/include/global_project/macros/relations/materialized_view/alter.sql create mode 100644 core/dbt/include/global_project/macros/relations/materialized_view/create.sql create mode 100644 core/dbt/include/global_project/macros/relations/materialized_view/describe.sql create mode 100644 core/dbt/include/global_project/macros/relations/materialized_view/drop.sql create mode 100644 core/dbt/include/global_project/macros/relations/materialized_view/refresh.sql create mode 100644 core/dbt/include/global_project/macros/relations/materialized_view/rename.sql create mode 100644 core/dbt/include/global_project/macros/relations/table/drop.sql create mode 100644 core/dbt/include/global_project/macros/relations/table/rename.sql create mode 100644 core/dbt/include/global_project/macros/relations/view/drop.sql create mode 100644 core/dbt/include/global_project/macros/relations/view/rename.sql delete mode 100644 plugins/postgres/dbt/adapters/postgres/materialization.py delete mode 100644 plugins/postgres/dbt/adapters/postgres/materialization_config/__init__.py delete mode 100644 plugins/postgres/dbt/adapters/postgres/materialization_config/database.py delete mode 100644 plugins/postgres/dbt/adapters/postgres/materialization_config/materialized_view.py delete mode 100644 plugins/postgres/dbt/adapters/postgres/materialization_config/schema.py rename plugins/postgres/dbt/adapters/postgres/{relation.py => relation/__init__.py} (94%) create mode 100644 plugins/postgres/dbt/adapters/postgres/relation/models/__init__.py create mode 100644 plugins/postgres/dbt/adapters/postgres/relation/models/database.py rename plugins/postgres/dbt/adapters/postgres/{materialization_config => relation/models}/index.py (54%) create mode 100644 plugins/postgres/dbt/adapters/postgres/relation/models/materialized_view.py rename plugins/postgres/dbt/adapters/postgres/{materialization_config => relation/models}/policy.py (57%) create mode 100644 plugins/postgres/dbt/adapters/postgres/relation/models/schema.py rename plugins/postgres/dbt/include/postgres/macros/{catalog.sql => get_catalog.sql} (100%) create mode 100644 plugins/postgres/dbt/include/postgres/macros/get_relations.sql delete mode 100644 plugins/postgres/dbt/include/postgres/macros/index.sql delete mode 100644 plugins/postgres/dbt/include/postgres/macros/materializations/replace.sql delete mode 100644 plugins/postgres/dbt/include/postgres/macros/materializations/table.sql delete mode 100644 plugins/postgres/dbt/include/postgres/macros/materializations/view.sql create mode 100644 plugins/postgres/dbt/include/postgres/macros/relation_components/index.sql create mode 100644 plugins/postgres/dbt/include/postgres/macros/relation_components/schema.sql delete mode 100644 plugins/postgres/dbt/include/postgres/macros/relations.sql rename plugins/postgres/dbt/include/postgres/macros/{materializations => relations}/materialized_view.sql (57%) create mode 100644 plugins/postgres/dbt/include/postgres/macros/relations/table.sql create mode 100644 plugins/postgres/dbt/include/postgres/macros/relations/view.sql rename plugins/postgres/dbt/include/postgres/macros/{ => utils}/timestamps.sql (100%) create mode 100644 tests/functional/materializations/materialized_view_tests/conftest.py create mode 100644 tests/functional/materializations/materialized_view_tests/files.py delete mode 100644 tests/functional/materializations/materialized_view_tests/fixtures.py create mode 100644 tests/functional/materializations/materialized_view_tests/utils.py create mode 100644 tests/unit/dbt_postgres_tests/conftest.py create mode 100644 tests/unit/dbt_postgres_tests/materialization_tests/test_materialization.py create mode 100644 tests/unit/dbt_postgres_tests/materialization_tests/test_materialization_factory.py create mode 100644 tests/unit/dbt_postgres_tests/relation_tests/model_tests/test_database.py create mode 100644 tests/unit/dbt_postgres_tests/relation_tests/model_tests/test_index.py create mode 100644 tests/unit/dbt_postgres_tests/relation_tests/model_tests/test_materialized_view.py create mode 100644 tests/unit/dbt_postgres_tests/relation_tests/model_tests/test_schema.py create mode 100644 tests/unit/dbt_postgres_tests/relation_tests/test_relation_factory.py diff --git a/core/dbt/adapters/base/impl.py b/core/dbt/adapters/base/impl.py index c2602df2787..8d902f042d2 100644 --- a/core/dbt/adapters/base/impl.py +++ b/core/dbt/adapters/base/impl.py @@ -26,7 +26,6 @@ from dbt import deprecations from dbt.adapters.base import Credentials, Column as BaseColumn from dbt.adapters.base.connections import AdapterResponse, Connection -from dbt.adapters.base.materialization import Materialization as _Materialization from dbt.adapters.base.meta import AdapterMeta, available from dbt.adapters.base.relation import ( ComponentName, @@ -35,8 +34,9 @@ SchemaSearchMap, ) from dbt.adapters.cache import RelationsCache, _make_ref_key_dict -from dbt.adapters.materialization_config import MaterializationConfig +from dbt.adapters.materialization import MaterializationFactory, models as materialization_models from dbt.adapters.protocol import AdapterConfig, ConnectionManagerProtocol +from dbt.adapters.relation import RelationFactory, models as relation_models from dbt.clients.agate_helper import empty_table, merge_tables, table_from_rows from dbt.clients.jinja import MacroGenerator from dbt.contracts.graph.manifest import MacroManifest, Manifest @@ -46,7 +46,6 @@ ModelLevelConstraint, ResultNode, ) -from dbt.contracts.relation import RelationType as _RelationType from dbt.events.functions import fire_event, warn_or_error from dbt.events.types import ( CacheMiss, @@ -211,8 +210,6 @@ class BaseAdapter(metaclass=AdapterMeta): """ Relation: Type[BaseRelation] = BaseRelation - RelationType: Type[_RelationType] = _RelationType - Materialization: Type[_Materialization] = _Materialization Column: Type[BaseColumn] = BaseColumn ConnectionManager: Type[ConnectionManagerProtocol] @@ -228,6 +225,14 @@ class BaseAdapter(metaclass=AdapterMeta): ConstraintType.foreign_key: ConstraintSupport.ENFORCED, } + @property + def relation_factory(self): + return RelationFactory(relation_models={}) + + @property + def materialization_factory(self): + return MaterializationFactory(relation_factory=self.relation_factory) + def __init__(self, config): self.config = config self.cache = RelationsCache() @@ -821,33 +826,6 @@ def get_relation(self, database: str, schema: str, identifier: str) -> Optional[ return None - @available - def get_cached_relation( - self, materialization_config: MaterializationConfig, kind: Optional[str] = None - ) -> BaseRelation: - """ - Expose `get_relation()` to `MaterializationConfig` instances. Includes a `kind` argument to - combine `load_cached_relation()` macro with `make_backup_relation()` and `make_intermediate_relation()` - with `get_relation()`. - """ - cached_kind = { - "existing": materialization_config.name, - "backup": self.Materialization.backup_name(materialization_config), - "intermediate": self.Materialization.intermediate_name(materialization_config), - } - - if name := cached_kind.get(kind or "existing"): - return self.get_relation( - database=materialization_config.database_name, - schema=materialization_config.schema_name, - identifier=name, - ) - - raise DbtRuntimeError( - f"Invalid cached relation kind for `get_cached_relation()`: {kind}" - f" Please supply one of: {', '.join(cached_kind.keys())}" - ) - @available.deprecated("get_relation", lambda *a, **k: False) def already_exists(self, schema: str, name: str) -> bool: """DEPRECATED: Return if a model already exists in the database""" @@ -1453,23 +1431,147 @@ def render_model_constraint(cls, constraint: ModelLevelConstraint) -> Optional[s else: return None + """ + Pass-through methods to access `MaterializationFactory` and `RelationFactory` functionality + """ + @available - def is_base_relation( - self, materialization: Union[BaseRelation, MaterializationConfig] - ) -> bool: + def make_materialization_from_runtime_config( + self, runtime_config, materialization_type: materialization_models.MaterializationType + ) -> materialization_models.Materialization: """ - Convenient for templating, given the mix of `BaseRelation` and `MaterializationConfig` + Produce a `Materialization` instance along with whatever associated `Relation` and `RelationStub` + instances are needed. + + Args: + runtime_config: the `config` (`RuntimeConfigObject`) in the global jinja context + materialization_type: the name of the materialization + + Returns: + a `Materialization` instance that contains all the information required to execute the materialization + """ + existing_relation_stub = self._get_existing_relation_stub_from_model_node( + runtime_config.model + ) + + materialization = self.materialization_factory.make_from_runtime_config( + runtime_config, materialization_type, existing_relation_stub + ) + + return materialization + + def _get_existing_relation_stub_from_model_node( + self, model_node + ) -> Optional[relation_models.RelationStub]: """ - return isinstance(materialization, BaseRelation) + We need to get `existing_relation_stub` from `Adapter` because we need access to a bunch of `cache` + things, in particular `get_relations`. + + TODO: if we refactor the interaction between `Adapter` and `cache`, the calculation of `existing_relation_stub` + could be moved here, which is a more intuitive spot (like `target_relation`) for it + (and removes the concern of creating a `RelationStub` from `Adapter` where it doesn't belong + """ + existing_base_relation: BaseRelation = self.get_relation( + database=model_node.database, + schema=model_node.schema, + identifier=model_node.identifier, + ) + + if existing_base_relation: + existing_relation_stub = self.relation_factory.make_stub( + name=existing_base_relation.identifier, + schema_name=existing_base_relation.schema, + database_name=existing_base_relation.database, + relation_type=existing_base_relation.type, + ) + else: + existing_relation_stub = None + + return existing_relation_stub @available - def is_materialization_config( - self, materialization: Union[BaseRelation, MaterializationConfig] - ) -> bool: + def make_changeset( + self, + existing_relation: relation_models.Relation, + target_relation: relation_models.Relation, + ) -> relation_models.RelationChangeset: """ - Convenient for templating, given the mix of `BaseRelation` and `MaterializationConfig` + + Args: + existing_relation: the current implementation of the relation in the database + target_relation: the new implementation that should exist in the database going forward + + Returns: + a `RelationChangeset` instance that collects all the changes required to turn `existing_relation` + into `target_relation` + """ + return self.relation_factory.make_changeset(existing_relation, target_relation) + + """ + Implementation of cache methods for `Relation` instances (versus `BaseRelation` instances) + """ + + @available + def cache_created_relation_model(self, relation: relation_models.Relation) -> str: + base_relation = self.base_relation_from_relation_model(relation) + return self.cache_added(base_relation) + + @available + def cache_dropped_relation_model(self, relation: relation_models.Relation) -> str: + base_relation = self.base_relation_from_relation_model(relation) + return self.cache_dropped(base_relation) + + @available + def cache_renamed_relation_model( + self, relation: relation_models.Relation, new_name: str + ) -> str: + from_relation = self.base_relation_from_relation_model(relation) + to_relation = from_relation.incorporate(path={"identifier": new_name}) + return self.cache_renamed(from_relation, to_relation) + + """ + Methods to swap back and forth between `Relation` and `BaseRelation` instances + """ + + @available + def is_base_relation(self, relation: Union[BaseRelation, relation_models.Relation]) -> bool: + """ + Convenient for templating, given the mix of `BaseRelation` and `Relation` + """ + return isinstance(relation, BaseRelation) + + @available + def is_relation_model(self, relation: Union[BaseRelation, relation_models.Relation]) -> bool: + """ + Convenient for templating, given the mix of `BaseRelation` and `Relation` + """ + return isinstance(relation, relation_models.Relation) + + @available + def base_relation_from_relation_model( + self, relation: relation_models.Relation + ) -> BaseRelation: + """ + Produce a `BaseRelation` instance from a `Relation` instance. This is primarily done to + reuse existing functionality based on `BaseRelation` while working with `Relation` instances. + + Useful in combination with `is_relation_model`/`is_base_relation` + + Args: + relation: a `Relation` instance or subclass to be converted + + Returns: + a converted `BaseRelation` instance """ - return isinstance(materialization, MaterializationConfig) + base_relation: BaseRelation = self.Relation.create( + database=relation.database_name, + schema=relation.schema_name, + identifier=relation.name, + quote_policy=self.Relation.quote_policy, + type=relation.type, + ) + assert isinstance(base_relation, BaseRelation) # mypy + return base_relation COLUMNS_EQUAL_SQL = """ diff --git a/core/dbt/adapters/base/materialization.py b/core/dbt/adapters/base/materialization.py deleted file mode 100644 index 1c9550e3abb..00000000000 --- a/core/dbt/adapters/base/materialization.py +++ /dev/null @@ -1,130 +0,0 @@ -import dataclasses -from typing import Dict, Type - -from dbt.adapters.materialization_config import ( - DescribeRelationResults, - MaterializationConfig, - RelationConfigChangeAction, -) -from dbt.contracts.graph.model_config import OnConfigurationChangeOption -from dbt.contracts.graph.nodes import ModelNode -from dbt.contracts.relation import Policy, RelationType as _RelationType -from dbt.exceptions import DbtRuntimeError - - -class Materialization: - """ - This class is a service layer version of `BaseRelation` that exposes `MaterializationConfig` - functionality on `BaseAdapter`. - """ - - # registers MaterializationConfigs to RelationTypes - materialization_configs: Dict[_RelationType, MaterializationConfig] = dataclasses.field( - default_factory=dict - ) - - include_policy: Policy = Policy() - quote_policy: Policy = Policy() - - # useful Enums for templating - ChangeAction: Type[RelationConfigChangeAction] = RelationConfigChangeAction - ChangeOption: Type[OnConfigurationChangeOption] = OnConfigurationChangeOption - RelationType: Type[_RelationType] = _RelationType - - @classmethod - def make_backup(cls, materialization_config: MaterializationConfig) -> MaterializationConfig: - """ - Return a copy of the materialization config, but with a backup name instead of the original name. - - Args: - materialization_config: the materialization that needs a backup - - Returns: - a renamed copy of the materialization config - """ - return dataclasses.replace( - materialization_config, name=cls.backup_name(materialization_config) - ) - - @classmethod - def make_intermediate( - cls, materialization_config: MaterializationConfig - ) -> MaterializationConfig: - """ - Return a copy of the materialization config, but with a backup name instead of the original name. - - Args: - materialization_config: the materialization that needs a backup - - Returns: - a renamed copy of the materialization config - """ - return dataclasses.replace( - materialization_config, name=cls.intermediate_name(materialization_config) - ) - - @classmethod - def backup_name(cls, materialization_config: MaterializationConfig) -> str: - """ - Mimic the macro `make_backup_relation()` for `MaterializationConfig` instances. - """ - return f"{materialization_config.name}__dbt_backup" - - @classmethod - def intermediate_name(cls, materialization_config: MaterializationConfig) -> str: - """ - Mimic the macro `make_intermediate_relation()` for `MaterializationConfig` instances - """ - return f"{materialization_config.name}__dbt_tmp" - - @classmethod - def from_model_node(cls, model_node: ModelNode) -> MaterializationConfig: - """ - Produce a validated materialization config from the config available in the global jinja context. - - The intention is to remove validation from the jinja context and put it in python. This method gets - called in a jinja template and it's results are used in the jinja template. For an example, please - refer to `dbt/include/global_project/macros/materializations/models/materialized_view/materialization.sql`. - In this file, the relation config is retrieved right away, to ensure that the config is validated before - any sql is executed against the database. - - Args: - model_node: the `model` ModelNode instance that's in the global jinja context - - Returns: a validated adapter-specific, relation_type-specific MaterializationConfig instance - """ - relation_type = _RelationType(model_node.config.materialized) - - if materialization_config := cls.materialization_configs.get(relation_type): - return materialization_config.from_model_node(model_node) - - raise DbtRuntimeError( - f"materialization_config_from_model_node() is not supported" - f" for the provided relation type: {relation_type}" - ) - - @classmethod - def from_describe_relation_results( - cls, describe_relation_results: DescribeRelationResults, relation_type: _RelationType - ) -> MaterializationConfig: - """ - Produce a validated materialization config from a series of "describe "-type queries. - - The intention is to remove validation from the jinja context and put it in python. This method gets - called in a jinja template and it's results are used in the jinja template. For an example, please - refer to `dbt/include/global_project/macros/materializations/models/materialized_view/materialization.sql`. - - Args: - describe_relation_results: the results of one or more queries run against the database - to describe this relation - relation_type: the type of relation associated with the relation results - - Returns: a validated adapter-specific, relation_type-specific MaterializationConfig instance - """ - if materialization_config := cls.materialization_configs.get(relation_type): - return materialization_config.from_describe_relation_results(describe_relation_results) - - raise DbtRuntimeError( - f"materialization_config_from_describe_relation_results() is not" - f" supported for the provided relation type: {relation_type}" - ) diff --git a/core/dbt/adapters/base/relation.py b/core/dbt/adapters/base/relation.py index 87c8159d19e..18a4f6c250e 100644 --- a/core/dbt/adapters/base/relation.py +++ b/core/dbt/adapters/base/relation.py @@ -2,7 +2,6 @@ import dataclasses from typing import Any, Dict, Iterator, Optional, Set, Tuple, Type, TypeVar -from dbt.adapters.materialization_config import MaterializationConfig from dbt.contracts.graph.nodes import ( SourceDefinition, ManifestNode, @@ -38,8 +37,8 @@ class BaseRelation(FakeAPIObject, Hashable): quote_character: str = '"' # Python 3.11 requires that these use default_factory instead of simple default # ValueError: mutable default for field include_policy is not allowed: use default_factory - include_policy: Policy = dataclasses.field(default_factory=lambda: Policy()) - quote_policy: Policy = dataclasses.field(default_factory=lambda: Policy()) + include_policy: Policy = dataclasses.field(default_factory=Policy) + quote_policy: Policy = dataclasses.field(default_factory=Policy) dbt_created: bool = False def _is_exactish_match(self, field: ComponentName, value: str) -> bool: @@ -291,31 +290,6 @@ def create( ) return cls.from_dict(kwargs) - @classmethod - def from_materialization_config( - cls, materialization_config: MaterializationConfig - ) -> "BaseRelation": - """ - Produce a `BaseRelation` instance from a `MaterializationConfig` instance. This is primarily done to - reuse existing functionality based on `BaseRelation` while working with `MaterializationConfig` instances. - - Useful in combination with `is_materialization_config`. - - Args: - materialization_config: a `MaterializationConfig` to be converted - - Returns: - a converted `BaseRelation` instance - """ - relation = cls.create( - database=materialization_config.database_name, - schema=materialization_config.schema_name, - identifier=materialization_config.name, - quote_policy=cls.quote_policy, - type=materialization_config.type, - ) - return relation - def __repr__(self) -> str: return "<{} {}>".format(self.__class__.__name__, self.render()) diff --git a/core/dbt/adapters/materialization_config/README.md b/core/dbt/adapters/materialization/README.md similarity index 100% rename from core/dbt/adapters/materialization_config/README.md rename to core/dbt/adapters/materialization/README.md diff --git a/core/dbt/adapters/materialization/__init__.py b/core/dbt/adapters/materialization/__init__.py new file mode 100644 index 00000000000..9da35f9cde6 --- /dev/null +++ b/core/dbt/adapters/materialization/__init__.py @@ -0,0 +1 @@ +from dbt.adapters.materialization.factory import MaterializationFactory diff --git a/core/dbt/adapters/materialization/factory.py b/core/dbt/adapters/materialization/factory.py new file mode 100644 index 00000000000..bae46937172 --- /dev/null +++ b/core/dbt/adapters/materialization/factory.py @@ -0,0 +1,37 @@ +from typing import Dict, Optional, Type + +from dbt.adapters.materialization import models +from dbt.adapters.relation import RelationFactory +from dbt.adapters.relation.models import RelationStub + + +class MaterializationFactory: + def __init__( + self, + relation_factory: RelationFactory = None, + materialization_map: Dict[models.MaterializationType, Type[models.Materialization]] = None, + ): + self.relation_factory = relation_factory + self.materialization_map = materialization_map or { + models.MaterializationType.MaterializedView: models.MaterializedViewMaterialization + } + + def make_from_runtime_config( + self, + runtime_config, + materialization_type: models.MaterializationType, + existing_relation_stub: Optional[RelationStub] = None, + ) -> Optional[models.Materialization]: + if parser := self._get_parser(materialization_type): + assert self.relation_factory is not None # mypy + return parser.from_runtime_config( + runtime_config=runtime_config, + relation_factory=self.relation_factory, + existing_relation_stub=existing_relation_stub, + ) + return None + + def _get_parser( + self, materialization_type: models.MaterializationType + ) -> Optional[Type[models.Materialization]]: + return self.materialization_map.get(materialization_type) diff --git a/core/dbt/adapters/materialization/models/__init__.py b/core/dbt/adapters/materialization/models/__init__.py new file mode 100644 index 00000000000..dbc0abcacad --- /dev/null +++ b/core/dbt/adapters/materialization/models/__init__.py @@ -0,0 +1,6 @@ +from dbt.adapters.materialization.models._materialization import ( + Materialization, + MaterializationBuildStrategy, + MaterializationType, +) +from dbt.adapters.materialization.models._materialized_view import MaterializedViewMaterialization diff --git a/core/dbt/adapters/materialization/models/_materialization.py b/core/dbt/adapters/materialization/models/_materialization.py new file mode 100644 index 00000000000..6a01050bc45 --- /dev/null +++ b/core/dbt/adapters/materialization/models/_materialization.py @@ -0,0 +1,143 @@ +from abc import ABC +from dataclasses import dataclass, field +from typing import Optional + +from dbt.contracts.graph.model_config import OnConfigurationChangeOption +from dbt.dataclass_schema import StrEnum +from dbt.flags import get_flag_obj +from dbt.utils import filter_null_values + +from dbt.adapters.relation.factory import RelationFactory +from dbt.adapters.relation.models import DescribeRelationResults, Relation, RelationStub + + +class MaterializationType(StrEnum): + """ + This overlaps with `RelationType` for several values (e.g. `View`); however, they are not the same. + For example, a materialization type of `Incremental` would be associated with a relation type of `Table`. + """ + + View = "view" + Table = "table" + Incremental = "incremental" + Seed = "seed" + MaterializedView = "materialized_view" + + +class MaterializationBuildStrategy(StrEnum): + Alter = "alter" + Create = "create" + NoOp = "no_op" + Replace = "replace" + + +@dataclass +class Materialization(ABC): + + type: MaterializationType + relation_factory: RelationFactory + target_relation: Relation + existing_relation_stub: Optional[RelationStub] = None + is_full_refresh: bool = False + grants: dict = field(default_factory=dict) + on_configuration_change: OnConfigurationChangeOption = OnConfigurationChangeOption.default() + + def __str__(self) -> str: + """ + This gets used in some error messages. + + Returns: + A user-friendly name to be used in logging, error messages, etc. + """ + return str(self.target_relation) + + def existing_relation( + self, describe_relation_results: DescribeRelationResults + ) -> Optional[Relation]: + """ + Produce a full-blown `Relation` instance for `self.existing_relation_stub` using metadata from the database + + Args: + describe_relation_results: the results from the macro `describe_sql(self.existing_relation_stub)` + + Returns: + a `Relation` instance that represents `self.existing_relation_stub` in the database + """ + if self.existing_relation_stub: + relation_type = self.existing_relation_stub.type + return self.relation_factory.make_from_describe_relation_results( + describe_relation_results, relation_type + ) + return None + + @property + def intermediate_relation(self) -> Optional[Relation]: + if self.target_relation: + return self.relation_factory.make_intermediate(self.target_relation) + return None + + @property + def backup_relation_stub(self) -> Optional[RelationStub]: + if self.existing_relation_stub: + return self.relation_factory.make_backup_stub(self.existing_relation_stub) + return None + + @property + def build_strategy(self) -> MaterializationBuildStrategy: + return MaterializationBuildStrategy.NoOp + + @property + def should_revoke_grants(self) -> bool: + """ + This attempts to mimic the macro `should_revoke()` + """ + should_revoke = { + MaterializationBuildStrategy.Alter: True, + MaterializationBuildStrategy.Create: False, + MaterializationBuildStrategy.NoOp: False, + MaterializationBuildStrategy.Replace: True, + } + return should_revoke[self.build_strategy] + + @classmethod + def from_dict(cls, config_dict) -> "Materialization": + return cls(**filter_null_values(config_dict)) + + @classmethod + def from_runtime_config( + cls, + runtime_config, + relation_factory: RelationFactory, + existing_relation_stub: Optional[RelationStub] = None, + ) -> "Materialization": + config_dict = cls.parse_runtime_config( + runtime_config, relation_factory, existing_relation_stub + ) + materialization = cls.from_dict(config_dict) + return materialization + + @classmethod + def parse_runtime_config( + cls, + runtime_config, + relation_factory: RelationFactory, + existing_relation_stub: Optional[RelationStub] = None, + ) -> dict: + target_relation = relation_factory.make_from_model_node(runtime_config.model) + # FULL_REFRESH defaults to False, hence the default in runtime_config.get() + is_full_refresh = any( + {get_flag_obj().FULL_REFRESH, runtime_config.get("full_refresh", False)} + ) + grants = runtime_config.get("grants", {}) + on_configuration_change = runtime_config.get( + "on_configuration_change", OnConfigurationChangeOption.default() + ) + + return { + "relation_factory": relation_factory, + "target_relation": target_relation, + "is_full_refresh": is_full_refresh, + "grants": grants, + "on_configuration_change": on_configuration_change, + "existing_relation_stub": existing_relation_stub, + } diff --git a/core/dbt/adapters/materialization/models/_materialized_view.py b/core/dbt/adapters/materialization/models/_materialized_view.py new file mode 100644 index 00000000000..29ccae0eb4e --- /dev/null +++ b/core/dbt/adapters/materialization/models/_materialized_view.py @@ -0,0 +1,55 @@ +from abc import ABC +from dataclasses import dataclass +from typing import Optional + +from dbt.adapters.relation.factory import RelationFactory +from dbt.adapters.relation.models import MaterializedViewRelation, RelationStub + +from dbt.adapters.materialization.models._materialization import ( + Materialization, + MaterializationBuildStrategy, + MaterializationType, +) + + +@dataclass +class MaterializedViewMaterialization(Materialization, ABC): + """ + This config identifies the minimal materialization parameters required for dbt to function as well + as built-ins that make macros more extensible. Additional parameters may be added by subclassing for your adapter. + """ + + target_relation: MaterializedViewRelation = None # type: ignore + existing_relation_stub: RelationStub = None # type: ignore + intermediate_relation: MaterializedViewRelation = None # type: ignore + backup_relation_stub: RelationStub = None # type: ignore + + @property + def build_strategy(self) -> MaterializationBuildStrategy: + + # this is a new relation, so just create it + if self.existing_relation_stub is None: + return MaterializationBuildStrategy.Create + + # there is an existing relation, so check if we are going to replace it before determining changes + elif self.is_full_refresh or ( + self.target_relation.type != self.existing_relation_stub.type + ): + return MaterializationBuildStrategy.Replace + + # `target_relation` and `existing_relation` both exist and are the same type, so we need to determine changes + else: + return MaterializationBuildStrategy.Alter + + @classmethod + def parse_runtime_config( + cls, + runtime_config, + relation_factory: RelationFactory, + existing_relation_stub: Optional[RelationStub] = None, + ) -> dict: + config_dict = super().parse_runtime_config( + runtime_config, relation_factory, existing_relation_stub + ) + config_dict.update({"type": MaterializationType.MaterializedView}) + return config_dict diff --git a/core/dbt/adapters/materialization_config/__init__.py b/core/dbt/adapters/materialization_config/__init__.py deleted file mode 100644 index 1d52ac2dc4d..00000000000 --- a/core/dbt/adapters/materialization_config/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -from dbt.adapters.materialization_config._base import RelationConfig, DescribeRelationResults -from dbt.adapters.materialization_config._change import ( - RelationConfigChangeAction, - RelationConfigChange, - RelationConfigChangeset, -) -from dbt.adapters.materialization_config._database import DatabaseConfig -from dbt.adapters.materialization_config._materialization import MaterializationConfig -from dbt.adapters.materialization_config._policy import ( - IncludePolicy, - QuotePolicy, - conform_part, - render_part, - render, -) -from dbt.adapters.materialization_config._schema import SchemaConfig -from dbt.adapters.materialization_config._validation import ( - RelationConfigValidationMixin, - RelationConfigValidationRule, -) diff --git a/core/dbt/adapters/materialization_config/_base.py b/core/dbt/adapters/materialization_config/_base.py deleted file mode 100644 index 885f02274b2..00000000000 --- a/core/dbt/adapters/materialization_config/_base.py +++ /dev/null @@ -1,137 +0,0 @@ -from abc import ABC, abstractmethod -from dataclasses import dataclass -from typing import Dict, Union - -import agate - -from dbt.contracts.graph.nodes import ModelNode -from dbt.utils import filter_null_values - - -""" -Relation metadata from the database comes in the form of a collection of one or more `agate.Table`s. In order to -reference the tables, they are added to a dictionary. There can be more than one table because there can be -multiple grains of data for a single object. For example, a materialized view in Postgres has base level information, -like name. But it also can have multiple indexes, which needs to be a separate query. It might look like this: - -{ - "materialized_view": agate.Table( - agate.Row({"table_name": "table_abc", "query": "select * from table_def"}) - ), - "indexes": agate.Table("rows": [ - agate.Row({"name": "index_a", "columns": ["column_a"], "type": "hash", "unique": False}), - agate.Row({"name": "index_b", "columns": ["time_dim_a"], "type": "btree", "unique": False}), - ]), -} - -Generally speaking, "primary" `RelationConfig` instances (e.g. materialized view) will be described with -an `agate.Table` and "dependent" `RelationConfig` instances (e.g. index) will be described with an `agate.Row`. -This happens simply because the primary instance is the first step in processing the metadata, but the dependent -instance can be looped when dispatching to it in `parse_describe_relation_results()`. -""" -DescribeRelationResults = Dict[str, Union[agate.Row, agate.Table]] - - -@dataclass(frozen=True) -class RelationConfig(ABC): - @classmethod - def from_dict(cls, kwargs_dict) -> "RelationConfig": - """ - This assumes the subclass of `RelationConfig` is flat, in the sense that no attribute is - itself another subclass of `RelationConfig`. If that's not the case, this should be overriden - to manually manage that complexity. But remember to either call `super().from_dict()` at the end, - or at least use `filter_null_values()` so that defaults get applied properly for the dataclass. - - Args: - kwargs_dict: the dict representation of this instance - - Returns: the `RelationConfig` representation associated with the provided dict - """ - return cls(**filter_null_values(kwargs_dict)) # type: ignore - - @classmethod - def from_model_node(cls, model_node: ModelNode) -> "RelationConfig": - """ - A wrapper around `cls.parse_model_node()` and `cls.from_dict()` that pipes the results of the first into - the second. This shouldn't really need to be overridden; instead, the component methods should be overridden. - - Args: - model_node: the `model` (`ModelNode`) attribute off of `config` (`RuntimeConfigObject`) in the global - jinja context of a materialization - - Returns: - a validated `RelationConfig` instance specific to the adapter and relation type - """ - relation_config = cls.parse_model_node(model_node) - relation = cls.from_dict(relation_config) - return relation - - @classmethod - @abstractmethod - def parse_model_node(cls, model_node: ModelNode) -> dict: - """ - The purpose of this method is to translate the dbt/user generic parlance into the database parlance and - format it for `RelationConfig` consumption. - - In many cases this may be a one-to-one mapping; e.g. dbt calls it "schema_name" and the database calls it - "schema_name". This could also be a renaming, calculation, or dispatch to a lower grain object. - - See `dbt/adapters/postgres/relation_configs/materialized_view.py` to see an example implementation. - - Args: - model_node: the `model` (`ModelNode`) attribute off of `config` (`RuntimeConfigObject`) in the global - jinja context of a materialization - - Returns: - a non-validated dictionary version of a `RelationConfig` instance specific to the adapter and - relation type - """ - raise NotImplementedError( - "`parse_model_node()` needs to be implemented for this relation." - ) - - @classmethod - def from_describe_relation_results( - cls, describe_relation_results: DescribeRelationResults - ) -> "RelationConfig": - """ - A wrapper around `cls.parse_describe_relation_results()` and `cls.from_dict()` that pipes the results of the - first into the second. This shouldn't really need to be overridden; instead, the component methods should - be overridden. - - Args: - describe_relation_results: the results of one or more queries run against the database to gather the - requisite metadata to describe this relation - - Returns: - a validated `RelationConfig` instance specific to the adapter and relation type - """ - relation_config = cls.parse_describe_relation_results(describe_relation_results) - relation = cls.from_dict(relation_config) - return relation - - @classmethod - @abstractmethod - def parse_describe_relation_results( - cls, describe_relation_results: DescribeRelationResults - ) -> dict: - """ - The purpose of this method is to format the database parlance for `RelationConfig` consumption. - - This tends to be one-to-one except for combining grains of data. For example, a single materialized - view could have multiple indexes which would result in multiple queries to the database to build one - materialized view config object. All of these pieces get knit together here. - - See `dbt/adapters/postgres/relation_configs/materialized_view.py` to see an example implementation. - - Args: - describe_relation_results: the results of one or more queries run against the database to gather the - requisite metadata to describe this relation - - Returns: - a non-validated dictionary version of a `RelationConfig` instance specific to the adapter and - relation type - """ - raise NotImplementedError( - "`parse_describe_relation_results()` needs to be implemented for this relation." - ) diff --git a/core/dbt/adapters/materialization_config/_database.py b/core/dbt/adapters/materialization_config/_database.py deleted file mode 100644 index 9803a4a0ce3..00000000000 --- a/core/dbt/adapters/materialization_config/_database.py +++ /dev/null @@ -1,23 +0,0 @@ -from abc import ABC -from dataclasses import dataclass - -from dbt.adapters.materialization_config._base import RelationConfig - - -@dataclass(frozen=True, eq=True, unsafe_hash=True) -class DatabaseConfig(RelationConfig, ABC): - """ - This config identifies the minimal database parameters required for dbt to function as well - as built-ins that make macros more extensible. Additional parameters may be added by subclassing for your adapter. - """ - - name: str - - @property - def fully_qualified_path(self) -> str: - """ - This is sufficient if there is no quote policy or include policy, otherwise override it to apply those policies. - - Returns: a fully qualified path, run through the quote and include policies, for rendering in a template - """ - return self.name diff --git a/core/dbt/adapters/materialization_config/_materialization.py b/core/dbt/adapters/materialization_config/_materialization.py deleted file mode 100644 index 1543fb74334..00000000000 --- a/core/dbt/adapters/materialization_config/_materialization.py +++ /dev/null @@ -1,77 +0,0 @@ -from abc import ABC -from dataclasses import dataclass - -from dbt.adapters.materialization_config._base import DescribeRelationResults, RelationConfig -from dbt.adapters.materialization_config._database import DatabaseConfig -from dbt.adapters.materialization_config._schema import SchemaConfig -from dbt.contracts.graph.nodes import ModelNode -from dbt.contracts.relation import RelationType - - -@dataclass(frozen=True) -class MaterializationConfig(RelationConfig, ABC): - """ - This config identifies the minimal materialization parameters required for dbt to function as well - as built-ins that make macros more extensible. Additional parameters may be added by subclassing for your adapter. - """ - - name: str - schema: "SchemaConfig" - query: str - type: RelationType - - @property - def schema_name(self) -> str: - return self.schema.name - - @property - def database(self) -> "DatabaseConfig": - return self.schema.database - - @property - def database_name(self) -> str: - return self.database.name - - @property - def fully_qualified_path(self) -> str: - """ - This is sufficient if there is no quote policy or include policy, otherwise override it to apply those policies. - - Returns: a fully qualified path, run through the quote and include policies, for rendering in a template - """ - return f"{self.schema.fully_qualified_path}.{self.name}" - - @classmethod - def from_dict(cls, kwargs_dict) -> "MaterializationConfig": - """ - Supports type annotations - """ - config = super().from_dict(kwargs_dict) - assert isinstance(config, MaterializationConfig) - return config - - @classmethod - def from_model_node(cls, model_node: ModelNode) -> "MaterializationConfig": - """ - Supports type annotations - """ - config = super().from_model_node(model_node) - assert isinstance(config, MaterializationConfig) - return config - - @classmethod - def from_describe_relation_results( - cls, describe_relation_results: DescribeRelationResults - ) -> "MaterializationConfig": - """ - Supports type annotations - """ - config = super().from_describe_relation_results(describe_relation_results) - assert isinstance(config, MaterializationConfig) - return config - - def __str__(self): - """ - Useful for template rendering and aligns with BaseRelation so that they are interchangeable - """ - return self.fully_qualified_path diff --git a/core/dbt/adapters/materialization_config/_policy.py b/core/dbt/adapters/materialization_config/_policy.py deleted file mode 100644 index b702e9a922a..00000000000 --- a/core/dbt/adapters/materialization_config/_policy.py +++ /dev/null @@ -1,119 +0,0 @@ -from abc import ABC -from dataclasses import dataclass -from typing import Optional, OrderedDict - -from dbt.contracts.relation import ComponentName, Policy - - -@dataclass -class IncludePolicy(Policy, ABC): - pass - - -@dataclass -class QuotePolicy(Policy, ABC): - @property - def quote_character(self) -> str: - """This is property to appeal to the `Policy` serialization.""" - return '"' - - -""" -It's advised to create your own adapter-specific version of these functions to pipe in the policies -that are specific to your adapter and for easier maintenance. This can be done easily with `functools.partial`. -See `dbt/adapters/postgres/relation_configs/policy.py` for an example. -""" - - -def conform_part( - component: ComponentName, value: str, quote_policy: QuotePolicy = QuotePolicy() -) -> Optional[str]: - """ - Apply the quote policy to the value so that it may be stored on the config object. - - *Note: Parts get quoted as part of methods like list_relations. As a result, a quote policy - of `True` just means "leave it alone", whereas a quote policy of `False` means make it case-insensitive, - which in this case is `str.lower()`. This differs from `render_part` which should only be used - for preparing templates. In that case, the quote character is used. - - It's advised to create your own adapter-specific version to pipe in the policies for easier maintenance. - See `dbt/adapters/postgres/relation_configs/policy.py` for an example. - - Args: - component: the component of the policy to apply - value: the value to which the policies should be applied - quote_policy: the quote policy for the adapter - - Returns: - a policy-compliant value - """ - if quote_policy.get_part(component): - return value - return value.lower() - - -def render_part( - component: ComponentName, - value: str, - quote_policy: QuotePolicy = QuotePolicy(), - include_policy: IncludePolicy = IncludePolicy(), -) -> Optional[str]: - """ - Apply the include and quote policy to the value so that it may be rendered in a template. - - *Note: This differs from `conform_part` in that the quote character actually gets used and - the include policy gets applied. - Additionally, there may be times when `value` shows up already quoted, if that's the case, these - characters are removed so that `value` does not wind up double-quoted. - - It's advised to create your own adapter-specific version to pipe in the policies for easier maintenance. - See `dbt/adapters/postgres/relation_configs/policy.py` for an example. - - Args: - component: the component of the policy to apply - value: the value to which the policies should be applied - quote_policy: the quote policy for the adapter - include_policy: the include policy for the adapter - - Returns: - a policy-compliant value - """ - quote = quote_policy.quote_character - if include_policy.get_part(component): - if quote_policy.get_part(component): - return f"{quote}{value.replace(quote, '')}{quote}" - return value.lower() - return None - - -def render( - parts: OrderedDict[ComponentName, str], - quote_policy: QuotePolicy = QuotePolicy(), - include_policy: IncludePolicy = IncludePolicy(), - delimiter: str = ".", -) -> str: - """ - This does the same thing as `cls.render_part()`, but all at once. - - We need to make sure we join the parts in the correct order, including scenarios where we don't - receive all the components, e.g. someone needs a fully qualified schema (hence no identifier). - - It's advised to create your own adapter-specific version to pipe in the policies for easier maintenance. - See `dbt/adapters/postgres/relation_configs/policy.py` for an example. - - Args: - parts: an ordered dictionary mapping ComponentName to value, provide the dictionary in the order in which - you want the parts joined - quote_policy: the quote policy for the adapter - include_policy: the include policy for the adapter - delimiter: the delimiter to use between parts - - Returns: - a fully rendered path ready for a jinja template - """ - rendered_parts = [ - render_part(component, value, quote_policy, include_policy) - for component, value in parts.items() - ] - rendered_path = delimiter.join(part for part in rendered_parts if part is not None) - return rendered_path diff --git a/core/dbt/adapters/materialization_config/_schema.py b/core/dbt/adapters/materialization_config/_schema.py deleted file mode 100644 index 098504b6c07..00000000000 --- a/core/dbt/adapters/materialization_config/_schema.py +++ /dev/null @@ -1,29 +0,0 @@ -from abc import ABC -from dataclasses import dataclass - -from dbt.adapters.materialization_config._base import RelationConfig -from dbt.adapters.materialization_config._database import DatabaseConfig - - -@dataclass(frozen=True, eq=True, unsafe_hash=True) -class SchemaConfig(RelationConfig, ABC): - """ - This config identifies the minimal schema parameters required for dbt to function as well - as built-ins that make macros more extensible. Additional parameters may be added by subclassing for your adapter. - """ - - name: str - database: DatabaseConfig - - @property - def database_name(self) -> str: - return self.database.name - - @property - def fully_qualified_path(self) -> str: - """ - This is sufficient if there is no quote policy or include policy, otherwise override it to apply those policies. - - Returns: a fully qualified path, run through the quote and include policies, for rendering in a template - """ - return f"{self.database.fully_qualified_path}.{self.name}" diff --git a/core/dbt/adapters/relation/__init__.py b/core/dbt/adapters/relation/__init__.py new file mode 100644 index 00000000000..e23bbd579a8 --- /dev/null +++ b/core/dbt/adapters/relation/__init__.py @@ -0,0 +1 @@ +from dbt.adapters.relation.factory import RelationFactory diff --git a/core/dbt/adapters/relation/factory.py b/core/dbt/adapters/relation/factory.py new file mode 100644 index 00000000000..d20494648d4 --- /dev/null +++ b/core/dbt/adapters/relation/factory.py @@ -0,0 +1,108 @@ +from dataclasses import replace +from typing import Dict, Optional, Set, Type + +from dbt.contracts.graph.nodes import ModelNode +from dbt.contracts.relation import ComponentName, RelationType + +from dbt.adapters.relation import models + + +class RelationFactory: + """ + Unlike other classes that get used by adapters, this class is not intended to be subclassed. Instead, + an instance should be taken from it that takes in all the required configuration (or defaults to + what is here). + """ + + # this configuration should never change + backup_suffix: str = "__dbt_backup" + intermediate_suffix: str = "__dbt_tmp" + + def __init__( + self, + relation_models: Dict[RelationType, Type[models.Relation]], + relation_changesets: Dict[RelationType, Type[models.RelationChangeset]] = None, + relation_can_be_renamed: Set[RelationType] = None, + render_policy: models.RenderPolicy = models.RenderPolicy(), + ): + self.relation_models = relation_models + self.relation_changesets = relation_changesets or {} + self.relation_can_be_renamed = relation_can_be_renamed or set() + self.render_policy = render_policy + + def make_from_model_node(self, model_node: ModelNode) -> Optional[models.Relation]: + relation_type = RelationType(model_node.config.materialized) + if parser := self._get_parser(relation_type): + relation = parser.from_model_node(model_node) + assert isinstance(relation, models.Relation) # mypy + return relation + return None + + def make_from_describe_relation_results( + self, + describe_relation_results: models.DescribeRelationResults, + relation_type: RelationType, + ) -> Optional[models.Relation]: + if parser := self._get_parser(relation_type): + relation = parser.from_describe_relation_results(describe_relation_results) + assert isinstance(relation, models.Relation) # mypy + return relation + return None + + def make_stub( + self, + name: str, + schema_name: str, + database_name: str, + relation_type: RelationType, + ) -> models.RelationStub: + relation_stub = models.RelationStub.from_dict( + { + "name": name, + "schema": { + "name": schema_name, + "database": { + "name": database_name, + "render": self.render_policy, + }, + "render": self.render_policy, + }, + "render": self.render_policy, + "type": relation_type, + "can_be_renamed": relation_type in self.relation_can_be_renamed, + } + ) + return relation_stub + + def make_backup_stub(self, existing_relation: models.Relation) -> models.RelationStub: + backup_name = self.render_policy.part( + ComponentName.Identifier, f"{existing_relation.name}{self.backup_suffix}" + ) + assert isinstance(backup_name, str) # mypy + return self.make_stub( + name=backup_name, + schema_name=existing_relation.schema_name, + database_name=existing_relation.database_name, + relation_type=existing_relation.type, + ) + + def make_intermediate(self, target_relation: models.Relation) -> models.Relation: + intermediate_name = self.render_policy.part( + ComponentName.Identifier, f"{target_relation.name}{self.intermediate_suffix}" + ) + return replace(target_relation, name=intermediate_name) + + def make_changeset( + self, existing_relation: models.Relation, target_relation: models.Relation + ) -> Optional[models.RelationChangeset]: + if changeset := self._get_changeset(existing_relation.type): + return changeset.from_relations(existing_relation, target_relation) + return None + + def _get_parser(self, relation_type: RelationType) -> Optional[Type[models.Relation]]: + return self.relation_models.get(relation_type) + + def _get_changeset( + self, relation_type: RelationType + ) -> Optional[Type[models.RelationChangeset]]: + return self.relation_changesets.get(relation_type) diff --git a/core/dbt/adapters/relation/models/__init__.py b/core/dbt/adapters/relation/models/__init__.py new file mode 100644 index 00000000000..dd1613bc6a5 --- /dev/null +++ b/core/dbt/adapters/relation/models/__init__.py @@ -0,0 +1,22 @@ +from dbt.adapters.relation.models._change import ( + RelationChange, + RelationChangeAction, + RelationChangeset, +) +from dbt.adapters.relation.models._database import DatabaseRelation +from dbt.adapters.relation.models._materialized_view import ( + MaterializedViewRelation, + MaterializedViewRelationChangeset, +) +from dbt.adapters.relation.models._policy import IncludePolicy, QuotePolicy, RenderPolicy +from dbt.adapters.relation.models._relation import Relation +from dbt.adapters.relation.models._relation_component import ( + DescribeRelationResults, + RelationComponent, +) +from dbt.adapters.relation.models._relation_stub import ( + DatabaseRelationStub, + RelationStub, + SchemaRelationStub, +) +from dbt.adapters.relation.models._schema import SchemaRelation diff --git a/core/dbt/adapters/materialization_config/_change.py b/core/dbt/adapters/relation/models/_change.py similarity index 59% rename from core/dbt/adapters/materialization_config/_change.py rename to core/dbt/adapters/relation/models/_change.py index 1bfdf942b98..b5edb814b80 100644 --- a/core/dbt/adapters/materialization_config/_change.py +++ b/core/dbt/adapters/relation/models/_change.py @@ -1,18 +1,23 @@ from abc import ABC, abstractmethod +from copy import deepcopy from dataclasses import dataclass from typing import Hashable from dbt.dataclass_schema import StrEnum +from dbt.exceptions import DbtRuntimeError +from dbt.utils import filter_null_values +from dbt.adapters.relation.models._relation import Relation -class RelationConfigChangeAction(StrEnum): + +class RelationChangeAction(StrEnum): alter = "alter" create = "create" drop = "drop" @dataclass(frozen=True, eq=True, unsafe_hash=True) -class RelationConfigChange(ABC): +class RelationChange(ABC): """ Changes are generally "alter the thing in place" or "drop the old one in favor of the new one". In other words, you will either wind up with a single `alter` or a pair of `drop` and `create`. In the `alter` scenario, @@ -20,7 +25,7 @@ class RelationConfigChange(ABC): `context` tends to be the whole object, in particular for `create`. """ - action: RelationConfigChangeAction + action: RelationChangeAction context: Hashable # this is usually a RelationConfig, e.g. `IndexConfig`, or single value, e.g. `str` @property @@ -40,9 +45,46 @@ def requires_full_refresh(self) -> bool: @dataclass -class RelationConfigChangeset(ABC): +class RelationChangeset(ABC): + existing_relation: Relation + target_relation: Relation _requires_full_refresh_override: bool = False + def __post_init__(self): + if self.is_empty and self.existing_relation != self.target_relation: + # we need to force a full refresh if we didn't detect any changes but the objects are not the same + self.force_full_refresh() + + @classmethod + def from_dict(cls, config_dict) -> "RelationChangeset": + kwargs_dict = deepcopy(config_dict) + try: + changeset = cls(**filter_null_values(kwargs_dict)) + except TypeError: + raise DbtRuntimeError(f"Unexpected configuration received:\n" f" {config_dict}\n") + return changeset + + @classmethod + def from_relations( + cls, existing_relation: Relation, target_relation: Relation + ) -> "RelationChangeset": + kwargs_dict = cls.parse_relations(existing_relation, target_relation) + + # stuff the relations in so that we can do the post init to figure out if we need a full refresh + kwargs_dict.update( + { + "existing_relation": existing_relation, + "target_relation": target_relation, + } + ) + return cls.from_dict(kwargs_dict) + + @classmethod + def parse_relations(cls, existing_relation: Relation, target_relation: Relation) -> dict: + raise NotImplementedError( + "Configuration change management has not been fully configured for this adapter and/or relation type." + ) + @property def requires_full_refresh(self) -> bool: """ diff --git a/core/dbt/adapters/relation/models/_database.py b/core/dbt/adapters/relation/models/_database.py new file mode 100644 index 00000000000..ab69ea009e9 --- /dev/null +++ b/core/dbt/adapters/relation/models/_database.py @@ -0,0 +1,82 @@ +from abc import ABC +from dataclasses import dataclass + +import agate + +from dbt.contracts.graph.nodes import ModelNode +from dbt.contracts.relation import ComponentName + +from dbt.adapters.relation.models._relation import RelationComponent + + +@dataclass(frozen=True) +class DatabaseRelation(RelationComponent, ABC): + """ + This config identifies the minimal materialization parameters required for dbt to function as well + as built-ins that make macros more extensible. Additional parameters may be added by subclassing for your adapter. + """ + + name: str + + def __str__(self) -> str: + return self.fully_qualified_path + + @property + def fully_qualified_path(self) -> str: + path = self.render.part(ComponentName.Database, self.name) + assert isinstance(path, str) + return path + + @classmethod + def from_dict(cls, config_dict) -> "DatabaseRelation": + """ + Parse `config_dict` into a `DatabaseRelation` instance, applying defaults + """ + database = super().from_dict(config_dict) + assert isinstance(database, DatabaseRelation) + return database + + @classmethod + def parse_model_node(cls, model_node: ModelNode) -> dict: + """ + Parse `ModelNode` into a dict representation of a `DatabaseRelation` instance + + This is generally used indirectly by calling `from_model_node()`, but there are times when the dict + version is more useful + + Args: + model_node: the `model` (`ModelNode`) attribute (e.g. `config.model`) in the global jinja context + + Example `model_node`: + + ModelNode({ + "database": "my_database", + ..., + }) + + Returns: a `DatabaseRelation` instance as a dict, can be passed into `from_dict` + """ + config_dict = {"name": model_node.database} + return config_dict + + @classmethod + def parse_describe_relation_results(cls, describe_relation_results: agate.Row) -> dict: # type: ignore + """ + Parse database metadata into a dict representation of a `DatabaseRelation` instance + + This is generally used indirectly by calling `from_describe_relation_results()`, + but there are times when the dict version is more appropriate. + + Args: + describe_relation_results: the results of a set of queries that fully describe an instance of this class + + Example of `describe_relation_results`: + + agate.Row({ + "database_name": "my_database", + }) + + Returns: a `DatabaseRelation` instance as a dict, can be passed into `from_dict` + """ + config_dict = {"name": describe_relation_results["database_name"]} + return config_dict diff --git a/core/dbt/adapters/relation/models/_materialized_view.py b/core/dbt/adapters/relation/models/_materialized_view.py new file mode 100644 index 00000000000..24bdecd0f1a --- /dev/null +++ b/core/dbt/adapters/relation/models/_materialized_view.py @@ -0,0 +1,122 @@ +from abc import ABC +from dataclasses import dataclass +from typing import Dict + +import agate + +from dbt.contracts.graph.nodes import ModelNode +from dbt.contracts.relation import RelationType +from dbt.exceptions import DbtRuntimeError + +from dbt.adapters.relation.models._change import RelationChangeset +from dbt.adapters.relation.models._relation import Relation + + +@dataclass(frozen=True) +class MaterializedViewRelation(Relation, ABC): + """ + This config identifies the minimal materialization parameters required for dbt to function as well + as built-ins that make macros more extensible. Additional parameters may be added by subclassing for your adapter. + """ + + # attribution + query: str + + # configuration + # TODO: move `can_be_renamed` to `Relation`; see `Relation` for more information + can_be_renamed: bool + + @classmethod + def from_dict(cls, config_dict) -> "MaterializedViewRelation": + """ + Parse `config_dict` into a `MaterializationViewRelation` instance, applying defaults + """ + # default configuration + kwargs_dict = { + "type": RelationType.MaterializedView, + "can_be_renamed": cls.can_be_renamed, + } + kwargs_dict.update(config_dict) + + materialized_view = super().from_dict(kwargs_dict) + assert isinstance(materialized_view, MaterializedViewRelation) + return materialized_view + + @classmethod + def parse_model_node(cls, model_node: ModelNode) -> dict: + """ + Parse `ModelNode` into a dict representation of a `MaterializedViewRelation` instance + + This is generally used indirectly by calling `from_model_node()`, but there are times when the dict + version is more useful + + Args: + model_node: the `model` (`ModelNode`) attribute (e.g. `config.model`) in the global jinja context + + Example `model_node`: + + ModelNode({ + "compiled_code": "create view my_view as\n select * from my_table;\n", + "database": "my_database", + "identifier": "my_view", + "schema": "my_schema", + ..., + }) + + Returns: a `MaterializedViewRelation` instance as a dict, can be passed into `from_dict` + """ + config_dict = { + "name": model_node.identifier, + "schema": cls.SchemaParser.parse_model_node(model_node), + "query": (model_node.compiled_code or "").strip(), + } + return config_dict + + @classmethod + def parse_describe_relation_results( # type: ignore + cls, describe_relation_results: Dict[str, agate.Table] + ) -> dict: + """ + Parse database metadata into a dict representation of a `MaterializedViewRelation` instance + + This is generally used indirectly by calling `from_describe_relation_results()`, + but there are times when the dict version is more appropriate. + + Args: + describe_relation_results: the results of a set of queries that fully describe an instance of this class + + Example of `describe_relation_results`: + + { + "materialized_view": agate.Table(agate.Row({ + "table_name": "my_materialized_view", + "query": "create materialized view my_materialized_view as select * from my_table;", + })), + } + + Returns: a `MaterializedViewRelation` instance as a dict, can be passed into `from_dict` + """ + materialization: agate.Row = describe_relation_results["materialized_view"].rows[0] + + config_dict = { + "name": materialization["name"], + "schema": cls.SchemaParser.parse_describe_relation_results(materialization), + "query": materialization["query"].strip(), + } + return config_dict + + +class MaterializedViewRelationChangeset(RelationChangeset): + @classmethod + def parse_relations(cls, existing_relation: Relation, target_relation: Relation) -> dict: + try: + assert isinstance(existing_relation, MaterializedViewRelation) + assert isinstance(target_relation, MaterializedViewRelation) + except AssertionError: + raise DbtRuntimeError( + f"Two materialized view relations were expected, but received:\n" + f" existing: {existing_relation}\n" + f" new: {target_relation}\n" + ) + + return {} diff --git a/core/dbt/adapters/relation/models/_policy.py b/core/dbt/adapters/relation/models/_policy.py new file mode 100644 index 00000000000..617bece3f4d --- /dev/null +++ b/core/dbt/adapters/relation/models/_policy.py @@ -0,0 +1,73 @@ +from abc import ABC +from collections import OrderedDict +from dataclasses import dataclass +from typing import Optional + +from dbt.contracts.relation import Policy, ComponentName + + +@dataclass +class IncludePolicy(Policy, ABC): + pass + + +@dataclass +class QuotePolicy(Policy, ABC): + pass + + +class RenderPolicy: + def __init__( + self, + quote_policy: QuotePolicy = QuotePolicy(), + include_policy: IncludePolicy = IncludePolicy(), + quote_character: str = '"', + delimiter: str = ".", + ): + self.quote_policy = quote_policy + self.include_policy = include_policy + self.quote_character = quote_character + self.delimiter = delimiter + + def part(self, component: ComponentName, value: str) -> Optional[str]: + """ + Apply the include and quote policy to the value so that it may be rendered in a template. + + Args: + component: the component to be referenced in `IncludePolicy` and `QuotePolicy` + value: the value to be rendered + + Returns: + a policy-compliant value + """ + # this is primarily done to make it easy to create the backup and intermediate names, e.g. + # name = "my_view", backup_name = "my_view"__dbt_backup, rendered_name = "my_view__dbt_backup" + unquoted_value = value.replace(self.quote_character, "") + + # if it should be included and quoted, then wrap it in quotes as-is + if self.include_policy.get_part(component) and self.quote_policy.get_part(component): + rendered_value = f"{self.quote_character}{unquoted_value}{self.quote_character}" + + # if it should be included without quotes, then apply `lower()` to make it case-insensitive + elif self.include_policy.get_part(component): + rendered_value = unquoted_value.lower() + + # if it should not be included, return `None`, so it gets excluded in `render` + else: + rendered_value = None + + return rendered_value + + def full(self, parts: OrderedDict[ComponentName, str]) -> str: + """ + Apply `Render.part` to each part and then concatenate in order. + + Args: + parts: an ordered dictionary mapping ComponentName to value + + Returns: + a fully rendered path ready for a jinja template + """ + rendered_parts = [self.part(*part) for part in parts.items()] + rendered_path = self.delimiter.join(part for part in rendered_parts if part is not None) + return rendered_path diff --git a/core/dbt/adapters/relation/models/_relation.py b/core/dbt/adapters/relation/models/_relation.py new file mode 100644 index 00000000000..7dc0243de35 --- /dev/null +++ b/core/dbt/adapters/relation/models/_relation.py @@ -0,0 +1,65 @@ +from abc import ABC +from collections import OrderedDict +from dataclasses import dataclass + +from dbt.contracts.relation import ComponentName, RelationType + +from dbt.adapters.relation.models._relation_component import RelationComponent +from dbt.adapters.relation.models._schema import SchemaRelation + + +@dataclass(frozen=True) +class Relation(RelationComponent, ABC): + + # attribution + name: str + schema: SchemaRelation + + """ + TODO: `can_be_renamed` belongs on `Relation`; however, I get the error below and cannot figure out how to fix it. + + TypeError: non-default argument 'can_be_renamed' follows default argument + + """ + # configuration + type: RelationType + SchemaParser: SchemaRelation + + def __str__(self) -> str: + return self.fully_qualified_path + + @property + def fully_qualified_path(self) -> str: + return self.render.full( + OrderedDict( + { + ComponentName.Database: self.database_name, + ComponentName.Schema: self.schema_name, + ComponentName.Identifier: self.name, + } + ) + ) + + @property + def schema_name(self) -> str: + return self.schema.name + + @property + def database_name(self) -> str: + return self.schema.database_name + + @classmethod + def from_dict(cls, config_dict) -> "Relation": + """ + Parse `config_dict` into a `MaterializationViewRelation` instance, applying defaults + """ + # default configuration + kwargs_dict = {"SchemaParser": cls.SchemaParser} + kwargs_dict.update(config_dict) + + if schema := config_dict.get("schema"): + kwargs_dict.update({"schema": kwargs_dict["SchemaParser"].from_dict(schema)}) + + relation = super().from_dict(kwargs_dict) + assert isinstance(relation, Relation) + return relation diff --git a/core/dbt/adapters/relation/models/_relation_component.py b/core/dbt/adapters/relation/models/_relation_component.py new file mode 100644 index 00000000000..641bb340170 --- /dev/null +++ b/core/dbt/adapters/relation/models/_relation_component.py @@ -0,0 +1,156 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from typing import Dict, Union + +import agate + +from dbt.contracts.graph.nodes import ModelNode +from dbt.exceptions import DbtRuntimeError +from dbt.utils import filter_null_values + +from dbt.adapters.relation.models._policy import RenderPolicy + +""" +`Relation` metadata from the database comes in the form of a collection of one or more `agate.Table`s. In order to +reference the tables, they are added to a dictionary. There can be more than one table because there can be +multiple grains of data for a single object. For example, a materialized view in Postgres has base level information, +like name. But it also can have multiple indexes, which needs to be a separate query. The metadata for +a materialized view might look like this: + +{ + "materialized_view": agate.Table( + agate.Row({"table_name": "table_abc", "query": "select * from table_def"}) + ), + "indexes": agate.Table("rows": [ + agate.Row({"name": "index_a", "columns": ["column_a"], "type": "hash", "unique": False}), + agate.Row({"name": "index_b", "columns": ["time_dim_a"], "type": "btree", "unique": False}), + ]), +} + +whereas the metadata that gets used to create an index (`RelationComponent`) may look like this: + +agate.Row({"name": "index_a", "columns": ["column_a"], "type": "hash", "unique": False}) + +Generally speaking, `Relation` instances (e.g. materialized view) will be described with +an `agate.Table` and `RelationComponent` instances (e.g. index) will be described with an `agate.Row`. +This happens simply because the `Relation` instance is the first step in processing the metadata, but the +`RelationComponent` instance can be looped when dispatching to it in `parse_describe_relation_results()`. +""" +DescribeRelationResults = Union[Dict[str, agate.Table], agate.Row] + + +@dataclass(frozen=True) +class RelationComponent(ABC): + """ + This config identifies the minimal relation parameters required for dbt to function as well + as built-ins that make macros more extensible. Additional parameters may be added by subclassing for your adapter. + """ + + # configuration + render: RenderPolicy = field(compare=False) + + @classmethod + def from_dict(cls, config_dict) -> "RelationComponent": + """ + This assumes the subclass of `Relation` is flat, in the sense that no attribute is + itself another subclass of `Relation`. If that's not the case, this should be overriden + to manually manage that complexity. But remember to either call `super().from_dict()` at the end, + or at least use `filter_null_values()` so that defaults get applied properly for the dataclass. + + Args: + config_dict: the dict representation of this instance + + Returns: the `Relation` representation associated with the provided dict + """ + # default configuration + kwargs_dict = {"render": cls.render} if hasattr(cls, "render") else {} + kwargs_dict.update(config_dict) + + try: + relation_component = cls(**filter_null_values(kwargs_dict)) # type: ignore + except TypeError: + raise DbtRuntimeError(f"Unexpected configuration received:\n" f" {config_dict}\n") + return relation_component + + @classmethod + def from_model_node(cls, model_node: ModelNode) -> "RelationComponent": + """ + A wrapper around `parse_model_node()` and `from_dict()` that pipes the results of the first into + the second. This shouldn't really need to be overridden; instead, the component methods should be overridden. + + Args: + model_node: the `model` (`ModelNode`) attribute (e.g. `config.model`) in the global jinja context + + Returns: + a validated `Relation` instance specific to the adapter and relation type + """ + relation_config = cls.parse_model_node(model_node) + relation = cls.from_dict(relation_config) + return relation + + @classmethod + @abstractmethod + def parse_model_node(cls, model_node: ModelNode) -> dict: + """ + The purpose of this method is to translate the dbt/user generic parlance into the database parlance and + format it for `Relation.from_dict` consumption. + + In many cases this may be a one-to-one mapping; e.g. dbt calls it "schema" and the database calls it + "schema_name". In some cases it could require a calculation or dispatch to a lower grain object. + + See `dbt/adapters/postgres/relation_config/materialized_view.py` to see an example implementation. + + Args: + model_node: the `model` (`ModelNode`) attribute (e.g. `config.model`) in the global jinja context + + Returns: + a non-validated dictionary version of a `Relation` instance specific to the adapter and relation type + """ + raise NotImplementedError( + "`parse_model_node()` needs to be implemented for this relation." + ) + + @classmethod + def from_describe_relation_results( + cls, describe_relation_results: DescribeRelationResults + ) -> "RelationComponent": + """ + A wrapper around `parse_describe_relation_results()` and `from_dict()` that pipes the results of the + first into the second. This shouldn't really need to be overridden; instead, the component methods should + be overridden. + + Args: + describe_relation_results: the results of one or more queries run against the database to gather the + requisite metadata to describe this relation + + Returns: + a validated `Relation` instance specific to the adapter and relation type + """ + config_dict = cls.parse_describe_relation_results(describe_relation_results) + relation = cls.from_dict(config_dict) + return relation + + @classmethod + @abstractmethod + def parse_describe_relation_results( + cls, describe_relation_results: DescribeRelationResults + ) -> dict: + """ + The purpose of this method is to parse the database parlance for `Relation.from_dict` consumption. + + This tends to be one-to-one except for combining grains of data. For example, a single table + could have multiple indexes which would result in multiple queries to the database to build one + `TableRelation` object. All of these pieces get knit together here. + + See `dbt/adapters/postgres/relation_config/materialized_view.py` to see an example implementation. + + Args: + describe_relation_results: the results of one or more queries run against the database to gather the + requisite metadata to describe this relation + + Returns: + a non-validated dictionary version of a `Relation` instance specific to the adapter and relation type + """ + raise NotImplementedError( + "`parse_describe_relation_results()` needs to be implemented for this relation." + ) diff --git a/core/dbt/adapters/relation/models/_relation_stub.py b/core/dbt/adapters/relation/models/_relation_stub.py new file mode 100644 index 00000000000..e2c7531a2dd --- /dev/null +++ b/core/dbt/adapters/relation/models/_relation_stub.py @@ -0,0 +1,99 @@ +""" +This module provides a way to store only the required metadata for a `Relation` without any parsers or actual +relation_type-specific subclasses. It's primarily used to represent a relation that exists in the database +without needing to query the database. This is useful with low attribution macros (e.g. `drop_sql`, `rename_sql`) +where the details are not needed to perform the action. It should be the case that if a macro supports execution +with a `RelationStub` instance, then it should also support execution with a `Relation` instance. The converse +is not true (e.g. `create_sql`). +""" +from dataclasses import dataclass + +from dbt.contracts.graph.nodes import ModelNode + +from dbt.adapters.relation.models._database import DatabaseRelation +from dbt.adapters.relation.models._policy import RenderPolicy +from dbt.adapters.relation.models._relation import Relation +from dbt.adapters.relation.models._relation_component import DescribeRelationResults +from dbt.adapters.relation.models._schema import SchemaRelation + + +@dataclass(frozen=True) +class DatabaseRelationStub(DatabaseRelation): + @classmethod + def from_dict(cls, config_dict) -> "DatabaseRelationStub": + database_stub = cls( + **{ + "name": config_dict["name"], + "render": config_dict["render"], + } + ) + assert isinstance(database_stub, DatabaseRelationStub) + return database_stub + + @classmethod + def parse_model_node(cls, model_node: ModelNode) -> dict: + return {} + + @classmethod + def parse_describe_relation_results( + cls, describe_relation_results: DescribeRelationResults + ) -> dict: + return {} + + +@dataclass(frozen=True) +class SchemaRelationStub(SchemaRelation): + render: RenderPolicy + + @classmethod + def from_dict(cls, config_dict) -> "SchemaRelationStub": + schema_stub = cls( + **{ + "name": config_dict["name"], + "database": DatabaseRelation.from_dict(config_dict["database"]), + "render": config_dict["render"], + "DatabaseParser": DatabaseRelationStub, + } + ) + assert isinstance(schema_stub, SchemaRelationStub) + return schema_stub + + @classmethod + def parse_model_node(cls, model_node: ModelNode) -> dict: + return {} + + @classmethod + def parse_describe_relation_results( + cls, describe_relation_results: DescribeRelationResults + ) -> dict: + return {} + + +@dataclass(frozen=True) +class RelationStub(Relation): + can_be_renamed: bool + + @classmethod + def from_dict(cls, config_dict) -> "RelationStub": + relation_stub = cls( + **{ + "name": config_dict["name"], + "schema": SchemaRelationStub.from_dict(config_dict["schema"]), + "render": config_dict["render"], + "type": config_dict["type"], + "can_be_renamed": config_dict["can_be_renamed"], + "SchemaParser": SchemaRelationStub, + } + ) + assert isinstance(relation_stub, RelationStub) + return relation_stub + + @classmethod + def parse_model_node(cls, model_node: ModelNode) -> dict: + return {} + + @classmethod + def parse_describe_relation_results( + cls, describe_relation_results: DescribeRelationResults + ) -> dict: + return {} diff --git a/core/dbt/adapters/relation/models/_schema.py b/core/dbt/adapters/relation/models/_schema.py new file mode 100644 index 00000000000..22b02533043 --- /dev/null +++ b/core/dbt/adapters/relation/models/_schema.py @@ -0,0 +1,121 @@ +from abc import ABC +from copy import deepcopy +from collections import OrderedDict +from dataclasses import dataclass + +import agate + +from dbt.contracts.graph.nodes import ModelNode +from dbt.contracts.relation import ComponentName + +from dbt.adapters.relation.models._relation import RelationComponent +from dbt.adapters.relation.models._database import DatabaseRelation + + +@dataclass(frozen=True) +class SchemaRelation(RelationComponent, ABC): + """ + This config identifies the minimal materialization parameters required for dbt to function as well + as built-ins that make macros more extensible. Additional parameters may be added by subclassing for your adapter. + """ + + name: str + database: DatabaseRelation + + # configuration of base class + DatabaseParser: DatabaseRelation + + def __str__(self) -> str: + return self.fully_qualified_path + + @property + def fully_qualified_path(self) -> str: + return self.render.full( + OrderedDict( + { + ComponentName.Database: self.database_name, + ComponentName.Schema: self.name, + } + ) + ) + + @property + def database_name(self) -> str: + return self.database.name + + @classmethod + def from_dict(cls, config_dict) -> "SchemaRelation": + """ + Parse `config_dict` into a `SchemaRelation` instance, applying defaults + """ + # don't alter the incoming config + kwargs_dict = deepcopy(config_dict) + + # configuration + kwargs_dict.update( + { + "DatabaseParser": cls.DatabaseParser, + } + ) + + if database := config_dict.get("database"): + kwargs_dict.update({"database": cls.DatabaseParser.from_dict(database)}) + + schema = super().from_dict(kwargs_dict) + assert isinstance(schema, SchemaRelation) + return schema + + @classmethod + def parse_model_node(cls, model_node: ModelNode) -> dict: + """ + Parse `ModelNode` into a dict representation of a `SchemaRelation` instance + + This is generally used indirectly by calling `from_model_node()`, but there are times when the dict + version is more useful + + Args: + model_node: the `model` (`ModelNode`) attribute (e.g. `config.model`) in the global jinja context + + Example `model_node`: + + ModelNode({ + "database": "my_database", + "schema": "my_schema", + ..., + }) + + Returns: a `SchemaRelation` instance as a dict, can be passed into `from_dict` + """ + config_dict = { + "name": model_node.schema, + "database": cls.DatabaseParser.parse_model_node(model_node), + } + return config_dict + + @classmethod + def parse_describe_relation_results(cls, describe_relation_results: agate.Row) -> dict: # type: ignore + """ + Parse database metadata into a dict representation of a `SchemaRelation` instance + + This is generally used indirectly by calling `from_describe_relation_results()`, + but there are times when the dict version is more appropriate. + + Args: + describe_relation_results: the results of a set of queries that fully describe an instance of this class + + Example of `describe_relation_results`: + + agate.Row({ + "schema_name": "my_schema", + "database_name": "my_database", + }) + + Returns: a `SchemaRelation` instance as a dict, can be passed into `from_dict` + """ + config_dict = { + "name": describe_relation_results["schema_name"], + "database": cls.DatabaseParser.parse_describe_relation_results( + describe_relation_results + ), + } + return config_dict diff --git a/core/dbt/adapters/materialization_config/_validation.py b/core/dbt/adapters/validation.py similarity index 76% rename from core/dbt/adapters/materialization_config/_validation.py rename to core/dbt/adapters/validation.py index cc1fe9a9c97..c38379a400d 100644 --- a/core/dbt/adapters/materialization_config/_validation.py +++ b/core/dbt/adapters/validation.py @@ -5,7 +5,7 @@ @dataclass(frozen=True, eq=True, unsafe_hash=True) -class RelationConfigValidationRule: +class ValidationRule: """ A validation rule consists of two parts: - validation_check: the thing that should be True @@ -31,12 +31,12 @@ def default_error(self): @dataclass(frozen=True) -class RelationConfigValidationMixin: +class ValidationMixin: def __post_init__(self): self.run_validation_rules() @property - def validation_rules(self) -> Set[RelationConfigValidationRule]: + def validation_rules(self) -> Set[ValidationRule]: """ A set of validation rules to run against the object upon creation. @@ -45,6 +45,9 @@ def validation_rules(self) -> Set[RelationConfigValidationRule]: This defaults to no validation rules if not implemented. It's recommended to override this with values, but that may not always be necessary. + *Note:* Validation rules for child attributes (e.g. a ViewRelation's SchemaRelation) will run automatically + when they are created; there's no need to call `validation_rules` on child attributes. + Returns: a set of validation rules """ return set() @@ -58,13 +61,3 @@ def run_validation_rules(self): raise validation_rule.validation_error else: raise validation_rule.default_error - self.run_child_validation_rules() - - def run_child_validation_rules(self): - for attr_value in vars(self).values(): - if hasattr(attr_value, "validation_rules"): - attr_value.run_validation_rules() - if isinstance(attr_value, set): - for member in attr_value: - if hasattr(member, "validation_rules"): - member.run_validation_rules() diff --git a/core/dbt/context/providers.py b/core/dbt/context/providers.py index 1565cd38db7..ad58dcfb05a 100644 --- a/core/dbt/context/providers.py +++ b/core/dbt/context/providers.py @@ -101,14 +101,6 @@ def create(self, *args, **kwargs): return self._relation_type.create(*args, **kwargs) -class MaterializationProxy: - def __init__(self, adapter): - self._materialization = adapter.Materialization - - def __getattr__(self, key): - return getattr(self._materialization, key) - - class BaseDatabaseWrapper: """ Wrapper for runtime database interaction. Applies the runtime quote policy @@ -117,9 +109,8 @@ class BaseDatabaseWrapper: def __init__(self, adapter, namespace: MacroNamespace): self._adapter = adapter - self.Materialization = MaterializationProxy(adapter) self.Relation = RelationProxy(adapter) - self.RelationType = adapter.RelationType + self.relation_factory = adapter.relation_factory self._namespace = namespace def __getattr__(self, name): diff --git a/core/dbt/flags.py b/core/dbt/flags.py index 37462659c97..cce29e112f7 100644 --- a/core/dbt/flags.py +++ b/core/dbt/flags.py @@ -25,7 +25,7 @@ def env_set_truthy(key: str) -> Optional[str]: # this roughly follows the patten of EVENT_MANAGER in dbt/events/functions.py -# During de-globlization, we'll need to handle both similarly +# During de-globalization, we'll need to handle both similarly # Match USE_COLORS default with default in dbt.cli.params.use_colors for use in --version GLOBAL_FLAGS = Namespace(USE_COLORS=True) # type: ignore diff --git a/core/dbt/include/global_project/macros/adapters/drop_relation.sql b/core/dbt/include/global_project/macros/adapters/drop_relation.sql deleted file mode 100644 index 06eafcf1d39..00000000000 --- a/core/dbt/include/global_project/macros/adapters/drop_relation.sql +++ /dev/null @@ -1,35 +0,0 @@ -{% macro drop_relation(relation) -%} - {{ return(adapter.dispatch('drop_relation', 'dbt')(relation)) }} -{% endmacro %} - -{% macro default__drop_relation(relation) -%} - {% call statement('drop_relation', auto_begin=False) -%} - {%- if relation.is_table -%} - {{- drop_table(relation) -}} - {%- elif relation.is_view -%} - {{- drop_view(relation) -}} - {%- elif relation.is_materialized_view -%} - {{- drop_materialized_view(relation) -}} - {%- else -%} - drop {{ relation.type }} if exists {{ relation }} cascade - {%- endif -%} - {%- endcall %} -{% endmacro %} - - -{% macro drop_table(relation) -%} - {{ return(adapter.dispatch('drop_table', 'dbt')(relation)) }} -{%- endmacro %} - -{% macro default__drop_table(relation) -%} - drop table if exists {{ relation }} cascade -{%- endmacro %} - - -{% macro drop_view(relation) -%} - {{ return(adapter.dispatch('drop_view', 'dbt')(relation)) }} -{%- endmacro %} - -{% macro default__drop_view(relation) -%} - drop view if exists {{ relation }} cascade -{%- endmacro %} diff --git a/core/dbt/include/global_project/macros/adapters/relation.sql b/core/dbt/include/global_project/macros/adapters/relation.sql index f0dde7f20f0..1c2bd880079 100644 --- a/core/dbt/include/global_project/macros/adapters/relation.sql +++ b/core/dbt/include/global_project/macros/adapters/relation.sql @@ -43,18 +43,6 @@ {% endmacro %} -{% macro rename_relation(from_relation, to_relation) -%} - {{ return(adapter.dispatch('rename_relation', 'dbt')(from_relation, to_relation)) }} -{% endmacro %} - -{% macro default__rename_relation(from_relation, to_relation) -%} - {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %} - {% call statement('rename_relation') -%} - alter table {{ from_relation }} rename to {{ target_name }} - {%- endcall %} -{% endmacro %} - - {% macro get_or_create_relation(database, schema, identifier, type) -%} {{ return(adapter.dispatch('get_or_create_relation', 'dbt')(database, schema, identifier, type)) }} {% endmacro %} @@ -89,10 +77,3 @@ {% macro load_relation(relation) %} {{ return(load_cached_relation(relation)) }} {% endmacro %} - - -{% macro drop_relation_if_exists(relation) %} - {% if relation is not none %} - {{ adapter.drop_relation(relation) }} - {% endif %} -{% endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/materialization.sql b/core/dbt/include/global_project/macros/materializations/materialization.sql index 794d50ac4a0..75485cf8cfa 100644 --- a/core/dbt/include/global_project/macros/materializations/materialization.sql +++ b/core/dbt/include/global_project/macros/materializations/materialization.sql @@ -1,66 +1,23 @@ -{# /* - These are step macros for building out a materialization based on `MaterializationConfig`, which can be - found in `dbt/adapters/relation_configs/materialization.py` - - Note: You cannot start the macro name with "materialization" because then dbt thinks it's a materialization - and not a macro. -*/ #} - - -{%- macro mat_setup(materialization_config, pre_hooks) -%} - - -- backup_relation and intermediate_relation should not already exist in the database - -- it's possible these exist because of a previous run that exited unexpectedly - {%- set backup_relation = adapter.get_cached_relation(materialization_config, "backup") -%} - {%- set intermediate_relation = adapter.get_cached_relation(materialization_config, "intermediate") -%} - - -- drop the temp relations if they exist already in the database - {{- drop_relation_if_exists(backup_relation) -}} - {{- drop_relation_if_exists(intermediate_relation) -}} - - {{- run_hooks(pre_hooks, inside_transaction=False) -}} - -{%- endmacro -%} - - -{%- macro mat_teardown(materialization_config, post_hooks) -%} - - -- backup_relation and intermediate_relation may exist if the materialized view was replaced - {%- set backup_relation = adapter.get_cached_relation(materialization_config, "backup") -%} - {%- set intermediate_relation = adapter.get_cached_relation(materialization_config, "intermediate") -%} - - -- drop the temp relations if they exist to leave the database clean for the next run - {{- drop_relation_if_exists(backup_relation) -}} - {{- drop_relation_if_exists(intermediate_relation) -}} - - {{- run_hooks(post_hooks, inside_transaction=False) -}} - -{%- endmacro -%} - - -{%- macro mat_execute_no_op(materialization_config) -%} +{%- macro execute_no_op(materialization) -%} {%- do store_raw_result( - name="main", - message="skip " ~ materialization_config.fully_qualified_path, - code="skip", - rows_affected="-1" + name='main', + message='skip ' ~ materialization, + code='skip', + rows_affected='-1' ) -%} {%- endmacro -%} -{%- macro mat_execute_build_sql(materialization_config, existing_relation, build_sql, post_hooks) -%} +{%- macro execute_build_sql(materialization, build_sql, pre_hooks, post_hooks) -%} -- `BEGIN` happens here: {{- run_hooks(pre_hooks, inside_transaction=True) -}} - {%- set grant_config = config.get('grants') -%} - - {%- call statement(name="main") -%} + {%- call statement(name='main') -%} {{ build_sql }} {%- endcall -%} - {%- set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) -%} - {%- do apply_grants(materialization_config.fully_qualified_path, grant_config, should_revoke=should_revoke) -%} + {%- do apply_grants(materialization, materialization.grant_config, materialization.should_revoke_grants) -%} {{- run_hooks(post_hooks, inside_transaction=True) -}} diff --git a/core/dbt/include/global_project/macros/materializations/materialized_view.sql b/core/dbt/include/global_project/macros/materializations/materialized_view.sql new file mode 100644 index 00000000000..fc66768a7b8 --- /dev/null +++ b/core/dbt/include/global_project/macros/materializations/materialized_view.sql @@ -0,0 +1,68 @@ +{%- materialization materialized_view, default -%} + + {%- set materialization = adapter.make_materialization_from_runtime_config(config, 'materialized_view') -%} + + {%- set build_sql = materialized_view_build_sql(materialization) -%} + + {{- run_hooks(pre_hooks, inside_transaction=False) -}} + + {%- if build_sql == '' -%} + {{- execute_no_op(materialization) -}} + {%- else -%} + {{- execute_build_sql(materialization, build_sql, pre_hooks, post_hooks) -}} + {%- endif -%} + + {{- run_hooks(post_hooks, inside_transaction=False) -}} + + {%- set new_base_relation = adapter.base_relation_from_relation_model(materialization.target_relation) -%} + {{- return({'relations': [new_base_relation]}) -}} + +{%- endmaterialization -%} + + +{%- macro materialized_view_build_sql(materialization) -%} + + {%- if materialization.build_strategy == 'no_op' -%} + {%- set build_sql = '' -%} + + {%- elif materialization.build_strategy == 'create' -%} + {%- set build_sql = create_template(materialization.target_relation) -%} + + {%- elif materialization.build_strategy == 'replace' -%} + {%- set build_sql = replace_template( + materialization.existing_relation_stub, materialization.target_relation + ) -%} + + {%- elif materialization.build_strategy == 'alter' -%} + + {% set describe_relation_results = describe_template(materialization.existing_relation_stub ) %} + {% set existing_relation = materialization.existing_relation(describe_relation_results) %} + + {%- if materialization.on_configuration_change == 'apply' -%} + {%- set build_sql = alter_template(existing_relation, materialization.target_relation) -%} + + {%- elif materialization.on_configuration_change == 'continue' -%} + {%- set build_sql = '' -%} + {{- exceptions.warn( + "Configuration changes were identified and `on_configuration_change` " + "was set to `continue` for `" ~ materialization.target_relation ~ "`" + ) -}} + + {%- elif materialization.on_configuration_change == 'fail' -%} + {%- set build_sql = '' -%} + {{- exceptions.raise_fail_fast_error( + "Configuration changes were identified and `on_configuration_change` " + "was set to `fail` for `" ~ materialization.target_relation ~ "`" + ) -}} + + {%- endif -%} + + {%- else -%} + + {{- exceptions.raise_compiler_error("This build strategy is not supported for materialized views: " ~ materialization.build_strategy) -}} + + {%- endif -%} + + {%- do return(build_sql) -%} + +{% endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/models/all/alter.sql b/core/dbt/include/global_project/macros/materializations/models/all/alter.sql deleted file mode 100644 index 619435c6eff..00000000000 --- a/core/dbt/include/global_project/macros/materializations/models/all/alter.sql +++ /dev/null @@ -1,20 +0,0 @@ -{%- macro alter_sql(existing_materialization, new_materialization) -%} - - {{- log('Applying ALTER to: ' ~ existing_materialization) -}} - - {{- adapter.dispatch('alter_sql', 'dbt')(existing_materialization, new_materialization) -}} - -{%- endmacro -%} - - -{%- macro default__alter_sql(existing_materialization, new_materialization) -%} - - {%- set relation_type = existing_materialization.type -%} - - {%- if relation_type == adapter.RelationType.MaterializedView -%} - {{ alter_materialized_view_sql(existing_materialization, new_materialization) }} - {%- else -%} - {{- exceptions.raise_compiler_error("`alter_sql()` has not been implemented for the relation type" ~ relation_type ) -}} - {%- endif -%} - -{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/materializations/models/all/create.sql b/core/dbt/include/global_project/macros/materializations/models/all/create.sql deleted file mode 100644 index 02ec28409da..00000000000 --- a/core/dbt/include/global_project/macros/materializations/models/all/create.sql +++ /dev/null @@ -1,20 +0,0 @@ -{%- macro create_sql(materialization) -%} - - {{- log('Applying CREATE to: ' ~ materialization) -}} - - {{- adapter.dispatch('create_sql', 'dbt')(materialization) -}} - -{%- endmacro -%} - - -{%- macro default__create_sql(materialization) -%} - - {%- set relation_type = materialization.type -%} - - {%- if relation_type == adapter.RelationType.MaterializedView -%} - {{ create_materialized_view_sql(materialization) }} - {%- else -%} - {{- exceptions.raise_compiler_error("`create_sql()` has not been implemented for the relation type" ~ relation_type ) -}} - {%- endif -%} - -{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/materializations/models/all/describe.sql b/core/dbt/include/global_project/macros/materializations/models/all/describe.sql deleted file mode 100644 index 755bb4653c5..00000000000 --- a/core/dbt/include/global_project/macros/materializations/models/all/describe.sql +++ /dev/null @@ -1,24 +0,0 @@ -{# /* - This needs to be a {% do return(...) %} because the macro returns a dictionary, not a template. -*/ #} - -{%- macro describe_sql(materialization) -%} - - {{- log('Applying DESCRIBE to: ' ~ materialization) -}} - - {%- do return(adapter.dispatch('describe_sql', 'dbt')(materialization)) -%} - -{%- endmacro -%} - - -{%- macro default__describe_sql(materialization) -%} - - {%- set relation_type = materialization.type -%} - - {%- if relation_type == adapter.RelationType.MaterializedView -%} - {%- do return(describe_materialized_view_sql(materialization)) -%} - {%- else -%} - {{- exceptions.raise_compiler_error("`describe_sql()` has not been implemented for the relation type" ~ relation_type ) -}} - {%- endif -%} - -{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/materializations/models/all/drop.sql b/core/dbt/include/global_project/macros/materializations/models/all/drop.sql deleted file mode 100644 index b7d9ea84169..00000000000 --- a/core/dbt/include/global_project/macros/materializations/models/all/drop.sql +++ /dev/null @@ -1,24 +0,0 @@ -{%- macro drop_sql(materialization) -%} - - {{- log('Applying DROP to: ' ~ materialization) -}} - - {%- set relation = adapter.Relation.from_materialization_config(materialization) -%} - - {{- adapter.dispatch('drop_sql', 'dbt')(materialization) -}} - - {{- adapter.cache_dropped(relation) -}} - -{%- endmacro -%} - - -{%- macro default__drop_sql(materialization) -%} - - {%- set relation_type = materialization.type -%} - - {%- if relation_type == adapter.RelationType.MaterializedView -%} - {{ drop_materialized_view_sql(materialization) }} - {%- else -%} - {{- exceptions.raise_compiler_error("`drop_sql()` has not been implemented for the relation type" ~ relation_type ) -}} - {%- endif -%} - -{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/materializations/models/all/rename.sql b/core/dbt/include/global_project/macros/materializations/models/all/rename.sql deleted file mode 100644 index 07b62525ec1..00000000000 --- a/core/dbt/include/global_project/macros/materializations/models/all/rename.sql +++ /dev/null @@ -1,33 +0,0 @@ -{%- macro rename_sql(materialization, new_name) -%} - - {{- log('Applying RENAME to: ' ~ materialization) -}} - - {%- if adapter.is_materialization_config(materialization) -%} - {%- set from_relation = adapter.Relation.from_materialization_config(materialization) -%} - {%- else -%} - {%- set from_relation = materialization -%} - {%- endif-%} - {%- set to_relation = from_relation.incorporate(path={"identifier": new_name}) -%} - - {{- adapter.dispatch('rename_sql', 'dbt')(materialization, new_name) -}} - - {{- adapter.cache_renamed(from_relation, to_relation) -}} - -{%- endmacro -%} - - -{%- macro default__rename_sql(materialization, new_name) -%} - - {%- set relation_type = materialization.type -%} - - {%- if relation_type == adapter.RelationType.View -%} - {{ rename_view_sql(materialization, new_name) }} - {%- elif relation_type == adapter.RelationType.Table -%} - {{ rename_table_sql(materialization, new_name) }} - {%- elif relation_type == adapter.RelationType.MaterializedView -%} - {{ rename_materialized_view_sql(materialization, new_name) }} - {%- else -%} - {{- exceptions.raise_compiler_error("`rename_sql()` has not been implemented for the relation type" ~ relation_type ) -}} - {%- endif -%} - -{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/materializations/models/all/replace.sql b/core/dbt/include/global_project/macros/materializations/models/all/replace.sql deleted file mode 100644 index 6603d345b8a..00000000000 --- a/core/dbt/include/global_project/macros/materializations/models/all/replace.sql +++ /dev/null @@ -1,16 +0,0 @@ -{# /* - This macro does dispatch to a relation_type-specific macro because it's actually a composite. -*/ #} - -{%- macro replace_sql(existing_materialization, new_materialization) -%} - - {{- log('Applying REPLACE to: ' ~ existing_materialization) -}} - - {{- adapter.dispatch('replace_sql', 'dbt')(existing_materialization, new_materialization) -}} - -{%- endmacro -%} - - -{%- macro default__replace_sql(existing_materialization, new_materialization) -%} - {{- exceptions.raise_compiler_error("`replace_sql` has not been implemented for this adapter.") -}} -{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/alter.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view/alter.sql deleted file mode 100644 index 2eb7ff9e27e..00000000000 --- a/core/dbt/include/global_project/macros/materializations/models/materialized_view/alter.sql +++ /dev/null @@ -1,8 +0,0 @@ -{%- macro alter_materialized_view_sql(existing_materialized_view, new_materialized_view) -%} - {{- adapter.dispatch('alter_materialized_view_sql', 'dbt')(existing_materialized_view, new_materialized_view) -}} -{%- endmacro -%} - - -{%- macro default__alter_materialized_view_sql(existing_materialized_view, new_materialized_view) -%} - {{- exceptions.raise_compiler_error("`alter_materialized_view_sql()` has not been implemented for this adapter.") -}} -{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/create.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view/create.sql deleted file mode 100644 index 44ccf427669..00000000000 --- a/core/dbt/include/global_project/macros/materializations/models/materialized_view/create.sql +++ /dev/null @@ -1,8 +0,0 @@ -{%- macro create_materialized_view_sql(materialized_view) -%} - {{- adapter.dispatch('create_materialized_view_sql', 'dbt')(materialized_view) -}} -{%- endmacro -%} - - -{%- macro default__create_materialized_view_sql(materialized_view) -%} - {{- exceptions.raise_compiler_error("`create_materialized_view_sql()` has not been implemented for this adapter.") -}} -{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/describe.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view/describe.sql deleted file mode 100644 index 0c06cb9e128..00000000000 --- a/core/dbt/include/global_project/macros/materializations/models/materialized_view/describe.sql +++ /dev/null @@ -1,12 +0,0 @@ -{# /* - This needs to be a {% do return(...) %} because the macro returns a dictionary, not a template. -*/ #} - -{%- macro describe_materialized_view_sql(materialized_view) -%} - {%- do return(adapter.dispatch('describe_materialized_view_sql', 'dbt')(materialized_view)) -%} -{%- endmacro -%} - - -{%- macro default__describe_materialized_view_sql(materialized_view) -%} - {{- exceptions.raise_compiler_error("`describe_materialized_view_sql()` has not been implemented for this adapter.") -}} -{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/drop.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view/drop.sql deleted file mode 100644 index 867f1870ad1..00000000000 --- a/core/dbt/include/global_project/macros/materializations/models/materialized_view/drop.sql +++ /dev/null @@ -1,25 +0,0 @@ -{%- macro drop_materialized_view_sql(materialized_view) -%} - {{- adapter.dispatch('drop_materialized_view_sql', 'dbt')(materialized_view) -}} -{%- endmacro -%} - - -{%- macro default__drop_materialized_view_sql(materialized_view) -%} - {{- exceptions.raise_compiler_error("`drop_materialized_view_sql()` has not been implemented for this adapter.") -}} -{%- endmacro -%} - - --- This is the one that gets used when non-materialized-views call drop -{%- macro drop_materialized_view(relation) -%} - - {{- log('Applying DROP to: ' ~ relation) -}} - - {{- return(adapter.dispatch('drop_materialized_view', 'dbt')(relation)) -}} - - {{- adapter.cache.drop(relation) -}} - -{%- endmacro %} - - -{%- macro default__drop_materialized_view(relation) -%} - {{- exceptions.raise_compiler_error("`drop_materialized_view()` has not been implemented for this adapter.") -}} -{%- endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/materialized_view.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view/materialized_view.sql deleted file mode 100644 index 74a7a26a56d..00000000000 --- a/core/dbt/include/global_project/macros/materializations/models/materialized_view/materialized_view.sql +++ /dev/null @@ -1,70 +0,0 @@ -{%- materialization materialized_view, default -%} - - -- Try to create a valid materialized view from the config before doing anything else - {%- set new_materialized_view = adapter.Materialization.from_model_node(config.model) -%} - - -- We still need these because they tie into the existing process (e.g. RelationBase vs. RelationConfigBase) - {%- set existing_relation = adapter.get_cached_relation(new_materialized_view) -%} - - {{- mat_setup(new_materialized_view, pre_hooks) -}} - - {%- set build_sql = materialized_view_build_sql(new_materialized_view, existing_relation) -%} - - {%- if build_sql == '' -%} - {{- mat_execute_no_op(new_materialized_view) -}} - {%- else -%} - {{- mat_execute_build_sql(new_materialized_view, existing_relation, build_sql, post_hooks) -}} - {%- endif -%} - - {{- mat_teardown(new_materialized_view, post_hooks) -}} - - {%- set new_relation = adapter.Relation.from_materialization_config(new_materialized_view) -%} - {{- return({'relations': [new_relation]}) -}} - -{%- endmaterialization -%} - - -{%- macro materialized_view_build_sql(new_materialized_view, existing_relation) -%} - - -- determine the scenario we're in: create, full_refresh, alter - {%- if existing_relation is none -%} - {%- set build_sql = create_sql(new_materialized_view) -%} - {%- elif should_full_refresh() or not existing_relation.is_materialized_view -%} - {%- set build_sql = replace_sql(existing_relation, new_materialized_view) -%} - {%- else -%} - {%- set build_sql = alter_materialized_view_with_on_configuration_change_option_sql(new_materialized_view) -%} - {%- endif -%} - - {%- do return(build_sql) -%} - -{% endmacro %} - - -{%- macro alter_materialized_view_with_on_configuration_change_option_sql(new_materialized_view) -%} - - {%- set describe_relation_results = describe_materialized_view_sql(new_materialized_view) -%} - {%- set existing_materialized_view = adapter.Materialization.from_describe_relation_results( - describe_relation_results, adapter.RelationType.MaterializedView - ) -%} - {%- set on_configuration_change = config.get('on_configuration_change') -%} - - {%- if existing_materialized_view == new_materialized_view -%} - {%- set build_sql = refresh_materialized_view_sql(existing_materialized_view) -%} - - {%- elif on_configuration_change == adapter.Materialization.ChangeOption.Apply -%} - {%- set build_sql = alter_sql(existing_materialized_view, new_materialized_view) -%} - {%- elif on_configuration_change == adapter.Materialization.ChangeOption.Continue -%} - {%- set build_sql = '' -%} - {{- exceptions.warn("Configuration changes were identified and `on_configuration_change` was set to `continue` for `" ~ new_materialized_view.fully_qualified_path ~ "`") -}} - {%- elif on_configuration_change == adapter.Materialization.ChangeOption.Fail -%} - {{- exceptions.raise_fail_fast_error("Configuration changes were identified and `on_configuration_change` was set to `fail` for `" ~ new_materialized_view.fully_qualified_path ~ "`") -}} - - {%- else -%} - -- this only happens if the user provides a value other than `apply`, 'continue', 'fail', which should have already raised an exception - {{- exceptions.raise_compiler_error("Unexpected configuration scenario: `" ~ on_configuration_change ~ "`") -}} - - {%- endif -%} - - {%- do return(build_sql) -%} - -{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/refresh.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view/refresh.sql deleted file mode 100644 index acfd571618e..00000000000 --- a/core/dbt/include/global_project/macros/materializations/models/materialized_view/refresh.sql +++ /dev/null @@ -1,16 +0,0 @@ -{# /* - This macro is elevated to `all` because it only applies to materialized views. -*/ #} - -{%- macro refresh_materialized_view_sql(materialized_view) -%} - - {{- log('Applying REFRESH to: ' ~ materialized_view) -}} - - {{- adapter.dispatch('refresh_materialized_view_sql', 'dbt')(materialized_view) -}} - -{%- endmacro -%} - - -{%- macro default__refresh_materialized_view_sql(materialized_view) -%} - {{- exceptions.raise_compiler_error("`refresh_materialized_view_sql()` has not been implemented for this adapter.") -}} -{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view/rename.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view/rename.sql deleted file mode 100644 index 44abc2a5a9a..00000000000 --- a/core/dbt/include/global_project/macros/materializations/models/materialized_view/rename.sql +++ /dev/null @@ -1,8 +0,0 @@ -{%- macro rename_materialized_view_sql(materialized_view, new_name) -%} - {{- adapter.dispatch('rename_materialized_view_sql', 'dbt')(materialized_view, new_name) -}} -{%- endmacro -%} - - -{%- macro default__rename_materialized_view_sql(materialized_view, new_name) -%} - {{- exceptions.raise_compiler_error("`rename_materialized_view_sql()` has not been implemented for this adapter.") -}} -{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/materializations/models/table/rename.sql b/core/dbt/include/global_project/macros/materializations/models/table/rename.sql deleted file mode 100644 index d3baafa4b1f..00000000000 --- a/core/dbt/include/global_project/macros/materializations/models/table/rename.sql +++ /dev/null @@ -1,8 +0,0 @@ -{%- macro rename_table_sql(table, new_name) -%} - {{- adapter.dispatch('rename_table_sql', 'dbt')(table, new_name) -}} -{%- endmacro -%} - - -{%- macro default__rename_table_sql(table, new_name) -%} - {{- exceptions.raise_compiler_error("`rename_table_sql()` has not been implemented for this adapter.") -}} -{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/materializations/models/view/rename.sql b/core/dbt/include/global_project/macros/materializations/models/view/rename.sql deleted file mode 100644 index cd31333044d..00000000000 --- a/core/dbt/include/global_project/macros/materializations/models/view/rename.sql +++ /dev/null @@ -1,8 +0,0 @@ -{%- macro rename_view_sql(view, new_name) -%} - {{- adapter.dispatch('rename_view_sql', 'dbt')(view, new_name) -}} -{%- endmacro -%} - - -{%- macro default__rename_view_sql(view, new_name) -%} - {{- exceptions.raise_compiler_error("`rename_view_sql()` has not been implemented for this adapter.") -}} -{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/adapters/indexes.sql b/core/dbt/include/global_project/macros/relation_components/index/create.sql similarity index 52% rename from core/dbt/include/global_project/macros/adapters/indexes.sql rename to core/dbt/include/global_project/macros/relation_components/index/create.sql index b8663a7f971..a4fdd9cee1b 100644 --- a/core/dbt/include/global_project/macros/adapters/indexes.sql +++ b/core/dbt/include/global_project/macros/relation_components/index/create.sql @@ -21,21 +21,3 @@ {% endif %} {% endfor %} {% endmacro %} - - -{% macro get_drop_index_sql(relation, index_name) -%} - {{ adapter.dispatch('get_drop_index_sql', 'dbt')(relation, index_name) }} -{%- endmacro %} - -{% macro default__get_drop_index_sql(relation, index_name) -%} - {{ exceptions.raise_compiler_error("`get_drop_index_sql has not been implemented for this adapter.") }} -{%- endmacro %} - - -{% macro get_show_indexes_sql(relation) -%} - {{ adapter.dispatch('get_show_indexes_sql', 'dbt')(relation) }} -{%- endmacro %} - -{% macro default__get_show_indexes_sql(relation) -%} - {{ exceptions.raise_compiler_error("`get_show_indexes_sql has not been implemented for this adapter.") }} -{%- endmacro %} diff --git a/core/dbt/include/global_project/macros/relation_components/index/drop.sql b/core/dbt/include/global_project/macros/relation_components/index/drop.sql new file mode 100644 index 00000000000..5085ed8555d --- /dev/null +++ b/core/dbt/include/global_project/macros/relation_components/index/drop.sql @@ -0,0 +1,7 @@ +{% macro drop_index_sql(relation, index_name) -%} + {{ adapter.dispatch('drop_index_sql', 'dbt')(relation, index_name) }} +{%- endmacro %} + +{% macro default__drop_index_sql(relation, index_name) -%} + {{ exceptions.raise_compiler_error("`drop_index_sql` has not been implemented for this adapter.") }} +{%- endmacro %} diff --git a/core/dbt/include/global_project/macros/relation_components/index/show.sql b/core/dbt/include/global_project/macros/relation_components/index/show.sql new file mode 100644 index 00000000000..2923c742ff7 --- /dev/null +++ b/core/dbt/include/global_project/macros/relation_components/index/show.sql @@ -0,0 +1,7 @@ +{% macro show_indexes_sql(relation) -%} + {{ adapter.dispatch('show_indexes_sql', 'dbt')(relation) }} +{%- endmacro %} + +{% macro default__show_indexes_sql(relation) -%} + {{ exceptions.raise_compiler_error("`show_indexes_sql` has not been implemented for this adapter.") }} +{%- endmacro %} diff --git a/core/dbt/include/global_project/macros/adapters/schema.sql b/core/dbt/include/global_project/macros/relation_components/schema/create.sql similarity index 50% rename from core/dbt/include/global_project/macros/adapters/schema.sql rename to core/dbt/include/global_project/macros/relation_components/schema/create.sql index 9e0c7559286..7b7be3315e7 100644 --- a/core/dbt/include/global_project/macros/adapters/schema.sql +++ b/core/dbt/include/global_project/macros/relation_components/schema/create.sql @@ -7,14 +7,3 @@ create schema if not exists {{ relation.without_identifier() }} {% endcall %} {% endmacro %} - - -{% macro drop_schema(relation) -%} - {{ adapter.dispatch('drop_schema', 'dbt')(relation) }} -{% endmacro %} - -{% macro default__drop_schema(relation) -%} - {%- call statement('drop_schema') -%} - drop schema if exists {{ relation.without_identifier() }} cascade - {% endcall %} -{% endmacro %} diff --git a/core/dbt/include/global_project/macros/relation_components/schema/drop.sql b/core/dbt/include/global_project/macros/relation_components/schema/drop.sql new file mode 100644 index 00000000000..41ae6b3967b --- /dev/null +++ b/core/dbt/include/global_project/macros/relation_components/schema/drop.sql @@ -0,0 +1,9 @@ +{% macro drop_schema(relation) -%} + {{ adapter.dispatch('drop_schema', 'dbt')(relation) }} +{% endmacro %} + +{% macro default__drop_schema(relation) -%} + {%- call statement('drop_schema') -%} + drop schema if exists {{ relation.without_identifier() }} cascade + {% endcall %} +{% endmacro %} diff --git a/core/dbt/include/global_project/macros/relations/README.md b/core/dbt/include/global_project/macros/relations/README.md new file mode 100644 index 00000000000..368be417e38 --- /dev/null +++ b/core/dbt/include/global_project/macros/relations/README.md @@ -0,0 +1,51 @@ +# Relation Macro Templates + +## Composite Macro Templates + +Macros in `/composite/` are composites of atomic macros (e.g. `create_template`, `drop_template`, +`rename_template`, etc. In other words, they don't dispatch directly to a relation_type-specific macro, nor do +they contain sql of their own. They are effectively logic flow to perform transactions that are a combination of +atomic statements. This is done to minimize the amount of sql that is written in jinja and remove redundancy. + +It's unlikely that these macros will need to be overridden; instead, the adapter maintainer is encouraged to +override the atomic components (e.g. `create_template`, `drop_template`, `rename_template`, etc.). Not only will +this minimize the amount of marginal maintenance within an adapter, it will also unlock all of the functionality +in these composite macros as a result. + +## Atomic Macro Templates + +Macros in `/atomic/` represent atomic actions on the database. They aren't necessarily transactions, nor are they +single statements; they are somewhere in between. They should be thought of as atomic at the `Relation` level in +the sense that you can't break down the action any further without losing a part of the relation, or a part of the +action on the relation. For example, the `create` action for a Postgres materialized view is actually a CREATE +statement followed by a series of CREATE INDEX statements. We wouldn't want to create the materialized view +without also creating all of its components, so that's one atomic action. Many actions are straight-forward, +(e.g. `drop` and `rename`) while others are less so (e.g. `alter` and `create`). Another way to think about it +is that all of these actions focus on exactly one relation, hence have a single `relation_type`. Even +`alter_template`, which takes in two `Relation` objects, is really just saying "I want `existing_relation` to +look like `"this"`"; `"this"` just happens to be another `Relation` object that contains all of the same +attributes, some with different values. + +While these actions are atomic, the macros in this directory represent `relation_type`-agnostic actions. +For example, if you want to create a view, execute `create_template(my_view_relation)`. Since `my_view_relation` +has a `relation_type` of `materialized_view`, `create_template` will know to dispatch the call to +`create_materialized_view_template`. If the maintainer looks at any macro in this directory, they will see that +the macro merely dispatches to the `relation_type`-specific version. Hence, there are only two reasons to override +this macro: + +1. The adapter supports more/less `relation-type`s than the default +2. The action can be consolidated into the same statement regardless of `relation_type` + +## Atomic Macro Templates by Relation_Type + +The most likely place that the adapter maintainer should look when overriding macros with adapter-specific +logic is in the relation-specific directories. Those are the directories in `/relations/` that have names +corresponding to `relation_type`s (e.g. `/materialized_view/`, `/view/`, etc.). At the `dbt-core` level, +macros in these directories will default to a version that throws an exception until implemented, much like +an abstract method in python. The intention is to make no assumptions about how databases work to avoid building +dependencies between database platforms within dbt. At the `dbt-` level, each of these files should +correspond to a specific statement (give or take) from that database platform's documentation. For example, +the macro `postgres__create_materialized_view_template` aligns with the documentation found here: +https://www.postgresql.org/docs/current/sql-creatematerializedview.html. Ideally, once this macro is created, +there is not much reason to perform maintenance on it unless the database platform deploys new functionality +and dbt (or the adapter) has chosen to support that functionality. diff --git a/core/dbt/include/global_project/macros/relations/atomic/alter.sql b/core/dbt/include/global_project/macros/relations/atomic/alter.sql new file mode 100644 index 00000000000..9524b82d8bc --- /dev/null +++ b/core/dbt/include/global_project/macros/relations/atomic/alter.sql @@ -0,0 +1,19 @@ +{%- macro alter_template(existing_relation, target_relation, called_directly=True) -%} + {%- if called_directly -%} + {{- log('Applying ALTER to: ' ~ existing_relation) -}} + {%- endif -%} + {{- adapter.dispatch('alter_template', 'dbt')(existing_relation, target_relation) -}} +{%- endmacro -%} + + +{%- macro default__alter_template(existing_relation, target_relation) -%} + + {%- if existing_relation.type == 'materialized_view' -%} + {{ alter_materialized_view_template(existing_relation, target_relation) }} + + {%- else -%} + {{- exceptions.raise_compiler_error("`alter_template` has not been implemented for: " ~ existing_relation.type ) -}} + + {%- endif -%} + +{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/relations/atomic/create.sql b/core/dbt/include/global_project/macros/relations/atomic/create.sql new file mode 100644 index 00000000000..374cc0c00d7 --- /dev/null +++ b/core/dbt/include/global_project/macros/relations/atomic/create.sql @@ -0,0 +1,21 @@ +{%- macro create_template(relation, called_directly=True) -%} + {%- if called_directly -%} + {{- log('Applying CREATE to: ' ~ relation) -}} + {%- endif -%} + {{- adapter.dispatch('create_template', 'dbt')(relation) -}} + + {{- adapter.cache_created_relation_model(relation) -}} +{%- endmacro -%} + + +{%- macro default__create_template(relation) -%} + + {%- if relation.type == 'materialized_view' -%} + {{ create_materialized_view_template(relation) }} + + {%- else -%} + {{- exceptions.raise_compiler_error("`create_template` has not been implemented for: " ~ relation.type ) -}} + + {%- endif -%} + +{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/relations/atomic/describe.sql b/core/dbt/include/global_project/macros/relations/atomic/describe.sql new file mode 100644 index 00000000000..80ae1fa7862 --- /dev/null +++ b/core/dbt/include/global_project/macros/relations/atomic/describe.sql @@ -0,0 +1,23 @@ +{# /* + This needs to be a {% do return(...) %} because the macro returns a dictionary, not a template. +*/ #} + +{%- macro describe_template(relation, called_directly=True) -%} + {%- if called_directly -%} + {{- log('Applying DESCRIBE to: ' ~ relation) -}} + {%- endif -%} + {%- do return(adapter.dispatch('describe_template', 'dbt')(relation)) -%} +{%- endmacro -%} + + +{%- macro default__describe_template(relation) -%} + + {%- if relation.type == 'materialized_view' -%} + {%- do return(describe_materialized_view_template(relation)) -%} + + {%- else -%} + {{- exceptions.raise_compiler_error("`describe_template` has not been implemented for: " ~ relation.type ) -}} + + {%- endif -%} + +{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/relations/atomic/drop.sql b/core/dbt/include/global_project/macros/relations/atomic/drop.sql new file mode 100644 index 00000000000..ef562285de1 --- /dev/null +++ b/core/dbt/include/global_project/macros/relations/atomic/drop.sql @@ -0,0 +1,61 @@ +{%- macro drop_template(relation, called_directly=True) -%} + {%- if called_directly -%} + {{- log('Applying DROP to: ' ~ relation) -}} + {%- endif -%} + {{- adapter.dispatch('drop_template', 'dbt')(relation) -}} + + {{- adapter.cache_dropped_relation_model(relation) -}} +{%- endmacro -%} + + +{%- macro default__drop_template(relation) -%} + + {%- if relation.type == 'view' -%} + {{ drop_view_template(relation) }} + + {%- elif relation.type == 'table' -%} + {{ drop_table_template(relation) }} + + {%- elif relation.type == 'materialized_view' -%} + {{ drop_materialized_view_template(relation) }} + + {%- else -%} + {{- exceptions.raise_compiler_error("`drop_template` has not been implemented for: " ~ relation.type ) -}} + + {%- endif -%} + +{%- endmacro -%} + + +{# /* + These are `BaseRelation` versions. The `BaseRelation` workflows are different. +*/ #} +{% macro drop_relation_if_exists(relation) %} + {% if relation is not none %} + {{ adapter.drop_relation(relation) }} + {% endif %} +{% endmacro %} + + +{% macro drop_relation(relation) -%} + {{ return(adapter.dispatch('drop_relation', 'dbt')(relation)) }} +{% endmacro %} + +{% macro default__drop_relation(relation) -%} + {% call statement('drop_relation', auto_begin=False) -%} + + {%- if relation.is_view -%} + drop view if exists {{ relation }} cascade + + {%- elif relation.is_table -%} + drop table if exists {{ relation }} cascade + + {%- elif relation.is_materialized_view -%} + drop materialized view if exists {{ relation }} cascade + + {%- else -%} + drop {{ relation.type }} if exists {{ relation }} cascade + + {%- endif -%} + {%- endcall %} +{% endmacro %} diff --git a/core/dbt/include/global_project/macros/relations/atomic/refresh.sql b/core/dbt/include/global_project/macros/relations/atomic/refresh.sql new file mode 100644 index 00000000000..925c3d24efc --- /dev/null +++ b/core/dbt/include/global_project/macros/relations/atomic/refresh.sql @@ -0,0 +1,19 @@ +{%- macro refresh_template(relation, called_directly=True) -%} + {%- if called_directly -%} + {{- log('Applying REFRESH to: ' ~ relation) -}} + {%- endif -%} + {{- adapter.dispatch('refresh_template', 'dbt')(relation) -}} +{%- endmacro -%} + + +{%- macro default__refresh_template(relation) -%} + + {%- if relation.type == 'materialized_view' -%} + {{ refresh_materialized_view_template(relation) }} + + {%- else -%} + {{- exceptions.raise_compiler_error("`refresh_template` has not been implemented for: " ~ relation.type ) -}} + + {%- endif -%} + +{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/relations/atomic/rename.sql b/core/dbt/include/global_project/macros/relations/atomic/rename.sql new file mode 100644 index 00000000000..84265912c88 --- /dev/null +++ b/core/dbt/include/global_project/macros/relations/atomic/rename.sql @@ -0,0 +1,42 @@ +{%- macro rename_template(relation, new_name, called_directly=True) -%} + {%- if called_directly -%} + {{- log('Applying RENAME to: ' ~ relation) -}} + {%- endif -%} + {{- adapter.dispatch('rename_template', 'dbt')(relation, new_name) -}} + + {{- adapter.cache_renamed_relation_model(relation, new_name) -}} +{%- endmacro -%} + + +{%- macro default__rename_template(relation, new_name) -%} + + {%- if relation.type == 'view' -%} + {{ rename_view_template(relation, new_name) }} + + {%- elif relation.type == 'table' -%} + {{ rename_table_template(relation, new_name) }} + + {%- elif relation.type == 'materialized_view' -%} + {{ rename_materialized_view_template(relation, new_name) }} + + {%- else -%} + {{- exceptions.raise_compiler_error("`rename_template` has not been implemented for: " ~ relation.type ) -}} + + {%- endif -%} + +{%- endmacro -%} + + +{# /* + These are `BaseRelation` versions. The `BaseRelation` workflows are different. +*/ #} +{% macro rename_relation(from_relation, to_relation) -%} + {{ return(adapter.dispatch('rename_relation', 'dbt')(from_relation, to_relation)) }} +{% endmacro %} + +{% macro default__rename_relation(from_relation, to_relation) -%} + {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %} + {% call statement('rename_relation') -%} + alter table {{ from_relation }} rename to {{ target_name }} + {%- endcall %} +{% endmacro %} diff --git a/core/dbt/include/global_project/macros/relations/composite/backup.sql b/core/dbt/include/global_project/macros/relations/composite/backup.sql new file mode 100644 index 00000000000..a4c24d7de2a --- /dev/null +++ b/core/dbt/include/global_project/macros/relations/composite/backup.sql @@ -0,0 +1,19 @@ +{%- macro backup_template(relation, called_directly=True) -%} + {%- if called_directly -%} + {{- log('Applying BACKUP to: ' ~ relation) -}} + {%- endif -%} + {{- adapter.dispatch('backup_template', 'dbt')(relation) -}} +{%- endmacro -%} + + +{%- macro default__backup_template(relation) -%} + + -- get the standard backup name + {% set backup_relation_stub = adapter.relation_factory.make_backup_stub(relation) -%} + + -- drop any pre-existing backup + {{ drop_template(backup_relation_stub, called_directly=False) }}; + + {{ rename_template(relation, backup_relation_stub.name, called_directly=False) }} + +{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/relations/composite/deploy_stage.sql b/core/dbt/include/global_project/macros/relations/composite/deploy_stage.sql new file mode 100644 index 00000000000..5a78ff10346 --- /dev/null +++ b/core/dbt/include/global_project/macros/relations/composite/deploy_stage.sql @@ -0,0 +1,16 @@ +{%- macro deploy_stage_template(relation, called_directly=True) -%} + {%- if called_directly -%} + {{- log('Applying DEPLOY STAGE to: ' ~ relation) -}} + {%- endif -%} + {{- adapter.dispatch('deploy_stage_template', 'dbt')(relation) -}} +{%- endmacro -%} + + +{%- macro default__deploy_stage_template(relation) -%} + + -- get the standard intermediate name + {% set intermediate_relation = adapter.relation_factory.make_intermediate(relation) -%} + + {{ rename_template(intermediate_relation, relation.name, called_directly=False) }} + +{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/relations/composite/drop_backup.sql b/core/dbt/include/global_project/macros/relations/composite/drop_backup.sql new file mode 100644 index 00000000000..e3d6943afe6 --- /dev/null +++ b/core/dbt/include/global_project/macros/relations/composite/drop_backup.sql @@ -0,0 +1,16 @@ +{%- macro drop_backup_template(relation, called_directly=True) -%} + {%- if called_directly -%} + {{- log('Applying DROP BACKUP to: ' ~ relation) -}} + {%- endif -%} + {{- adapter.dispatch('drop_backup_template', 'dbt')(relation) -}} +{%- endmacro -%} + + +{%- macro default__drop_backup_template(relation) -%} + + -- get the standard backup name + {% set backup_relation_stub = adapter.relation_factory.make_backup_stub(relation) -%} + + {{ drop_template(backup_relation_stub, called_directly=False) }} + +{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/relations/composite/replace.sql b/core/dbt/include/global_project/macros/relations/composite/replace.sql new file mode 100644 index 00000000000..d408b763258 --- /dev/null +++ b/core/dbt/include/global_project/macros/relations/composite/replace.sql @@ -0,0 +1,66 @@ +{%- macro replace_template(existing_relation, target_relation, called_directly=True) -%} + {%- if called_directly -%} + {{- log('Applying REPLACE to: ' ~ target_relation) -}} + {%- endif -%} + {{- adapter.dispatch('replace_template', 'dbt')(existing_relation, target_relation) -}} +{%- endmacro -%} + +{%- macro default__replace_template(existing_relation, target_relation) -%} + + {# /* create target_relation as an intermediate relation, then swap it out with the existing one using a backup */ #} + {%- if target_relation.can_be_renamed and existing_relation.can_be_renamed -%} + {{ stage_template(target_relation, called_directly=False) }}; + {{ backup_template(existing_relation, called_directly=False) }}; + {{ deploy_stage_template(target_relation, called_directly=False) }}; + {{ drop_backup_template(existing_relation, called_directly=False) }} + + {# /* create target_relation as an intermediate relation, then swap it out with the existing one using drop */ #} + {%- elif target_relation.can_be_renamed -%} + {{ stage_template(target_relation, called_directly=False) }}; + {{ drop_template(existing_relation, called_directly=False) }}; + {{ deploy_stage_template(target_relation, called_directly=False) }} + + {# /* create target_relation in place by first backing up the existing relation */ #} + {%- elif existing_relation.can_be_renamed -%} + {{ backup_template(existing_relation, called_directly=False) }}; + {{ create_template(target_relation, called_directly=False) }}; + {{ drop_backup_template(existing_relation, called_directly=False) }} + + {# /* no renaming is allowed, so just drop and create */ #} + {%- else -%} + {{ drop_template(existing_relation, called_directly=False) }}; + {{ create_template(target_relation, called_directly=False) }} + + {%- endif -%} + +{%- endmacro -%} + + +{%- macro default__replace_sql_alt(existing_relation, target_relation) -%} + + {# /* stage the target relation if we can, otherwise we'll create it later */ #} + {%- if target_relation.can_be_renamed -%} + {{ stage_template(target_relation, called_directly=False) }}; + {%- endif -%} + + {# /* backup the existing relation if we can, otherwise just drop it */ #} + {%- if existing_relation.can_be_renamed -%} + {{ backup_template(existing_relation, called_directly=False) }}; + {%- else -%} + {{ drop_template(existing_relation, called_directly=False) }}; + {%- endif -%} + + {# /* create the target relation from the staged relation if we were able to stage it earlier, otherwise create it here */ #} + {%- if target_relation.can_be_renamed -%} + {{ deploy_stage_template(target_relation, called_directly=False) }} + {%- else -%} + {{ create_template(target_relation, called_directly=False) }} + {%- endif -%} + + {# /* drop the backup relation if we were able to create it earlier */ #} + {%- if existing_relation.can_be_renamed -%} + ; -- we need this here because we don't know if the last statement happens in the previous if block until here + {{ drop_backup_template(existing_relation, called_directly=False) }} + {%- endif -%} + +{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/relations/composite/stage.sql b/core/dbt/include/global_project/macros/relations/composite/stage.sql new file mode 100644 index 00000000000..1deb65cf164 --- /dev/null +++ b/core/dbt/include/global_project/macros/relations/composite/stage.sql @@ -0,0 +1,19 @@ +{%- macro stage_template(relation, called_directly=True) -%} + {%- if called_directly -%} + {{- log('Applying STAGE to: ' ~ relation) -}} + {%- endif -%} + {{- adapter.dispatch('stage_template', 'dbt')(relation) -}} +{%- endmacro -%} + + +{%- macro default__stage_template(relation) -%} + + -- get the standard intermediate name + {% set intermediate_relation = adapter.relation_factory.make_intermediate(relation) -%} + + -- drop any pre-existing intermediate + {{ drop_template(intermediate_relation, called_directly=False) }}; + + {{ create_template(intermediate_relation, called_directly=False) }} + +{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/relations/materialized_view/alter.sql b/core/dbt/include/global_project/macros/relations/materialized_view/alter.sql new file mode 100644 index 00000000000..dd4bba88e8b --- /dev/null +++ b/core/dbt/include/global_project/macros/relations/materialized_view/alter.sql @@ -0,0 +1,8 @@ +{%- macro alter_materialized_view_template(existing_materialized_view, target_materialized_view) -%} + {{- adapter.dispatch('alter_materialized_view_template', 'dbt')(existing_materialized_view, target_materialized_view) -}} +{%- endmacro -%} + + +{%- macro default__alter_materialized_view_template(existing_materialized_view, target_materialized_view) -%} + {{- exceptions.raise_compiler_error("`alter_materialized_view_template` has not been implemented for this adapter.") -}} +{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/relations/materialized_view/create.sql b/core/dbt/include/global_project/macros/relations/materialized_view/create.sql new file mode 100644 index 00000000000..d231fc8ac9b --- /dev/null +++ b/core/dbt/include/global_project/macros/relations/materialized_view/create.sql @@ -0,0 +1,8 @@ +{%- macro create_materialized_view_template(materialized_view) -%} + {{- adapter.dispatch('create_materialized_view_template', 'dbt')(materialized_view) -}} +{%- endmacro -%} + + +{%- macro default__create_materialized_view_template(materialized_view) -%} + {{- exceptions.raise_compiler_error("`create_materialized_view_template` has not been implemented for this adapter.") -}} +{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/relations/materialized_view/describe.sql b/core/dbt/include/global_project/macros/relations/materialized_view/describe.sql new file mode 100644 index 00000000000..e4ed1d39c28 --- /dev/null +++ b/core/dbt/include/global_project/macros/relations/materialized_view/describe.sql @@ -0,0 +1,12 @@ +{# /* + This needs to be a {% do return(...) %} because the macro returns a dictionary, not a template. +*/ #} + +{%- macro describe_materialized_view_template(materialized_view) -%} + {%- do return(adapter.dispatch('describe_materialized_view_template', 'dbt')(materialized_view)) -%} +{%- endmacro -%} + + +{%- macro default__describe_materialized_view_template(materialized_view) -%} + {{- exceptions.raise_compiler_error("`describe_materialized_view_template` has not been implemented for this adapter.") -}} +{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/relations/materialized_view/drop.sql b/core/dbt/include/global_project/macros/relations/materialized_view/drop.sql new file mode 100644 index 00000000000..e4873707ba9 --- /dev/null +++ b/core/dbt/include/global_project/macros/relations/materialized_view/drop.sql @@ -0,0 +1,7 @@ +{%- macro drop_materialized_view_template(materialized_view) -%} + {{- adapter.dispatch('drop_materialized_view_template', 'dbt')(materialized_view) -}} +{%- endmacro -%} + +{%- macro default__drop_materialized_view_template(materialized_view) -%} + {{- exceptions.raise_compiler_error("`drop_materialized_view_template` has not been implemented for this adapter.") -}} +{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/relations/materialized_view/refresh.sql b/core/dbt/include/global_project/macros/relations/materialized_view/refresh.sql new file mode 100644 index 00000000000..c892055599e --- /dev/null +++ b/core/dbt/include/global_project/macros/relations/materialized_view/refresh.sql @@ -0,0 +1,8 @@ +{%- macro refresh_materialized_view_template(materialized_view) -%} + {{- adapter.dispatch('refresh_materialized_view_template', 'dbt')(materialized_view) -}} +{%- endmacro -%} + + +{%- macro default__refresh_materialized_view_template(materialized_view) -%} + {{- exceptions.raise_compiler_error("`refresh_materialized_view_template` has not been implemented for this adapter.") -}} +{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/relations/materialized_view/rename.sql b/core/dbt/include/global_project/macros/relations/materialized_view/rename.sql new file mode 100644 index 00000000000..ad75a92cf19 --- /dev/null +++ b/core/dbt/include/global_project/macros/relations/materialized_view/rename.sql @@ -0,0 +1,7 @@ +{%- macro rename_materialized_view_template(materialized_view, new_name) -%} + {{- adapter.dispatch('rename_materialized_view_template', 'dbt')(materialized_view, new_name) -}} +{%- endmacro -%} + +{%- macro default__rename_materialized_view_template(materialized_view, new_name) -%} + {{- exceptions.raise_compiler_error("`rename_materialized_view_template` has not been implemented for this adapter.") -}} +{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/relations/table/drop.sql b/core/dbt/include/global_project/macros/relations/table/drop.sql new file mode 100644 index 00000000000..61c5aae581f --- /dev/null +++ b/core/dbt/include/global_project/macros/relations/table/drop.sql @@ -0,0 +1,7 @@ +{%- macro drop_table_template(table) -%} + {{- adapter.dispatch('drop_table_template', 'dbt')(table) -}} +{%- endmacro -%} + +{%- macro default__drop_table_template(table) -%} + {{- exceptions.raise_compiler_error("`drop_table_template` has not been implemented for this adapter.") -}} +{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/relations/table/rename.sql b/core/dbt/include/global_project/macros/relations/table/rename.sql new file mode 100644 index 00000000000..696c80e6d75 --- /dev/null +++ b/core/dbt/include/global_project/macros/relations/table/rename.sql @@ -0,0 +1,7 @@ +{%- macro rename_table_template(table, new_name) -%} + {{- adapter.dispatch('rename_table_template', 'dbt')(table, new_name) -}} +{%- endmacro -%} + +{%- macro default__rename_table_template(table, new_name) -%} + {{- exceptions.raise_compiler_error("`rename_table_template` has not been implemented for this adapter.") -}} +{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/relations/view/drop.sql b/core/dbt/include/global_project/macros/relations/view/drop.sql new file mode 100644 index 00000000000..91779e2a730 --- /dev/null +++ b/core/dbt/include/global_project/macros/relations/view/drop.sql @@ -0,0 +1,7 @@ +{%- macro drop_view_template(view) -%} + {{- adapter.dispatch('drop_view_template', 'dbt')(view) -}} +{%- endmacro -%} + +{%- macro default__drop_view_template(view) -%} + {{- exceptions.raise_compiler_error("`drop_view_template` has not been implemented for this adapter.") -}} +{%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/relations/view/rename.sql b/core/dbt/include/global_project/macros/relations/view/rename.sql new file mode 100644 index 00000000000..fae68327774 --- /dev/null +++ b/core/dbt/include/global_project/macros/relations/view/rename.sql @@ -0,0 +1,7 @@ +{%- macro rename_view_template(view, new_name) -%} + {{- adapter.dispatch('rename_view_template', 'dbt')(view, new_name) -}} +{%- endmacro -%} + +{%- macro default__rename_view_template(view, new_name) -%} + {{- exceptions.raise_compiler_error("`rename_view_template` has not been implemented for this adapter.") -}} +{%- endmacro -%} diff --git a/plugins/postgres/dbt/adapters/postgres/__init__.py b/plugins/postgres/dbt/adapters/postgres/__init__.py index 38dce8bdb22..6ad0fe0a096 100644 --- a/plugins/postgres/dbt/adapters/postgres/__init__.py +++ b/plugins/postgres/dbt/adapters/postgres/__init__.py @@ -1,8 +1,7 @@ -# these are mostly just exports, #noqa them so flake8 will be happy -from dbt.adapters.postgres.connections import PostgresConnectionManager # noqa +from dbt.adapters.postgres.connections import PostgresConnectionManager from dbt.adapters.postgres.connections import PostgresCredentials -from dbt.adapters.postgres.column import PostgresColumn # noqa -from dbt.adapters.postgres.relation import PostgresRelation # noqa: F401 +from dbt.adapters.postgres.column import PostgresColumn +from dbt.adapters.postgres.relation import PostgresRelation from dbt.adapters.postgres.impl import PostgresAdapter from dbt.adapters.base import AdapterPlugin diff --git a/plugins/postgres/dbt/adapters/postgres/impl.py b/plugins/postgres/dbt/adapters/postgres/impl.py index 05a3043f867..a6c9e8dc3dc 100644 --- a/plugins/postgres/dbt/adapters/postgres/impl.py +++ b/plugins/postgres/dbt/adapters/postgres/impl.py @@ -4,8 +4,10 @@ from dbt.adapters.base.meta import available from dbt.adapters.base.impl import AdapterConfig, ConstraintSupport +from dbt.adapters.relation import RelationFactory from dbt.adapters.sql import SQLAdapter from dbt.contracts.graph.nodes import ConstraintType +from dbt.contracts.relation import RelationType from dbt.dataclass_schema import dbtClassMixin, ValidationError from dbt.exceptions import ( CrossDbReferenceProhibitedError, @@ -18,7 +20,7 @@ from dbt.adapters.postgres import PostgresConnectionManager, PostgresRelation from dbt.adapters.postgres.column import PostgresColumn -from dbt.adapters.postgres.materialization import PostgresMaterialization +from dbt.adapters.postgres.relation import models as relation_models # note that this isn't an adapter macro, so just a single underscore @@ -62,7 +64,6 @@ class PostgresConfig(AdapterConfig): class PostgresAdapter(SQLAdapter): Relation = PostgresRelation - Materialization = PostgresMaterialization ConnectionManager = PostgresConnectionManager Column = PostgresColumn @@ -76,6 +77,19 @@ class PostgresAdapter(SQLAdapter): ConstraintType.foreign_key: ConstraintSupport.ENFORCED, } + @property + def relation_factory(self): + return RelationFactory( + relation_models={ + RelationType.MaterializedView: relation_models.PostgresMaterializedViewRelation, + }, + relation_changesets={ + RelationType.MaterializedView: relation_models.PostgresMaterializedViewRelationChangeset, + }, + relation_can_be_renamed={RelationType.MaterializedView}, + render_policy=relation_models.PostgresRenderPolicy, + ) + @classmethod def date_function(cls): return "now()" @@ -146,3 +160,19 @@ def valid_incremental_strategies(self): def debug_query(self): self.execute("select 1 as id") + + @available + def generate_index_name( + self, + relation: relation_models.PostgresMaterializedViewRelation, + index: relation_models.PostgresIndexRelation, + ) -> str: + return dbt.utils.md5( + "_".join( + { + relation.fully_qualified_path, + index.fully_qualified_path, + str(datetime.utcnow().isoformat()), + } + ) + ) diff --git a/plugins/postgres/dbt/adapters/postgres/materialization.py b/plugins/postgres/dbt/adapters/postgres/materialization.py deleted file mode 100644 index 726777cc092..00000000000 --- a/plugins/postgres/dbt/adapters/postgres/materialization.py +++ /dev/null @@ -1,110 +0,0 @@ -from datetime import datetime -from typing import FrozenSet, Set - -from dbt.adapters.base.materialization import Materialization -from dbt.adapters.materialization_config import ( - MaterializationConfig, - RelationConfigChangeAction, -) -from dbt.contracts.relation import ComponentName, RelationType -from dbt.exceptions import DbtRuntimeError -import dbt.utils - -from dbt.adapters.postgres.materialization_config import ( - PostgresIncludePolicy, - PostgresIndexConfig, - PostgresIndexConfigChange, - PostgresMaterializedViewConfig, - PostgresMaterializedViewConfigChangeset, - PostgresQuotePolicy, - postgres_conform_part, -) - - -class PostgresMaterialization(Materialization): - materialization_configs = {RelationType.MaterializedView: PostgresMaterializedViewConfig} - include_policy = PostgresIncludePolicy() - quote_policy = PostgresQuotePolicy() - - @classmethod - def materialized_view_config_changeset( - cls, - existing_materialized_view: PostgresMaterializedViewConfig, - new_materialized_view: PostgresMaterializedViewConfig, - ) -> PostgresMaterializedViewConfigChangeset: - try: - assert isinstance(existing_materialized_view, PostgresMaterializedViewConfig) - assert isinstance(new_materialized_view, PostgresMaterializedViewConfig) - except AssertionError: - raise DbtRuntimeError( - f"Two materialized view configs were expected, but received:" - f"/n {existing_materialized_view}" - f"/n {new_materialized_view}" - ) - - config_changeset = PostgresMaterializedViewConfigChangeset() - - config_changeset.indexes = cls.index_config_changeset( - existing_materialized_view.indexes, new_materialized_view.indexes - ) - - if config_changeset.is_empty and existing_materialized_view != new_materialized_view: - # we need to force a full refresh if we didn't detect any changes but the objects are not the same - config_changeset.force_full_refresh() - - return config_changeset - - @classmethod - def index_config_changeset( - cls, - existing_indexes: FrozenSet[PostgresIndexConfig], - new_indexes: FrozenSet[PostgresIndexConfig], - ) -> Set[PostgresIndexConfigChange]: - """ - Get the index updates that will occur as a result of a new run - - There are four scenarios: - - 1. Indexes are equal -> don't return these - 2. Index is new -> create these - 3. Index is old -> drop these - 4. Indexes are not equal -> drop old, create new -> two actions - - Returns: a set of index updates in the form {"action": "drop/create", "context": } - """ - drop_changes = set( - PostgresIndexConfigChange(action=RelationConfigChangeAction.drop, context=index) - for index in existing_indexes.difference(new_indexes) - ) - create_changes = set( - PostgresIndexConfigChange(action=RelationConfigChangeAction.create, context=index) - for index in new_indexes.difference(existing_indexes) - ) - return set().union(drop_changes, create_changes) - - @classmethod - def generate_index_name( - cls, - materialization_config: MaterializationConfig, - index_config: PostgresIndexConfig, - ) -> str: - return dbt.utils.md5( - "_".join( - { - postgres_conform_part( - ComponentName.Database, materialization_config.database_name - ), - postgres_conform_part( - ComponentName.Schema, materialization_config.schema_name - ), - postgres_conform_part(ComponentName.Identifier, materialization_config.name), - *sorted( - postgres_conform_part(ComponentName.Identifier, column) - for column in index_config.column_names - ), - str(index_config.unique), - str(index_config.method), - str(datetime.utcnow().isoformat()), - } - ) - ) diff --git a/plugins/postgres/dbt/adapters/postgres/materialization_config/__init__.py b/plugins/postgres/dbt/adapters/postgres/materialization_config/__init__.py deleted file mode 100644 index baa2e205cd1..00000000000 --- a/plugins/postgres/dbt/adapters/postgres/materialization_config/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -from dbt.adapters.postgres.materialization_config.index import ( - PostgresIndexConfig, - PostgresIndexConfigChange, -) -from dbt.adapters.postgres.materialization_config.materialized_view import ( - PostgresMaterializedViewConfig, - PostgresMaterializedViewConfigChangeset, -) -from dbt.adapters.postgres.materialization_config.policy import ( - MAX_CHARACTERS_IN_IDENTIFIER, - PostgresIncludePolicy, - PostgresQuotePolicy, - postgres_conform_part, - postgres_render_part, - postgres_render, -) diff --git a/plugins/postgres/dbt/adapters/postgres/materialization_config/database.py b/plugins/postgres/dbt/adapters/postgres/materialization_config/database.py deleted file mode 100644 index b6f42bd9071..00000000000 --- a/plugins/postgres/dbt/adapters/postgres/materialization_config/database.py +++ /dev/null @@ -1,71 +0,0 @@ -from collections import OrderedDict -from dataclasses import dataclass -from typing import Set - -import agate -from dbt.adapters.materialization_config import ( - DatabaseConfig, - RelationConfigValidationMixin, - RelationConfigValidationRule, -) -from dbt.contracts.graph.nodes import ModelNode -from dbt.contracts.relation import ComponentName -from dbt.exceptions import DbtRuntimeError - -from dbt.adapters.postgres.materialization_config.policy import ( - postgres_render, - postgres_conform_part, -) - - -@dataclass(frozen=True, eq=True, unsafe_hash=True) -class PostgresDatabaseConfig(DatabaseConfig, RelationConfigValidationMixin): - """ - This config follow the specs found here: - https://www.postgresql.org/docs/current/sql-createdatabase.html - - The following parameters are configurable by dbt: - - name: name of the database - """ - - name: str - - @property - def fully_qualified_path(self) -> str: - return postgres_render(OrderedDict({ComponentName.Database: self.name})) - - @property - def validation_rules(self) -> Set[RelationConfigValidationRule]: - return { - RelationConfigValidationRule( - validation_check=len(self.name or "") > 0, - validation_error=DbtRuntimeError( - f"dbt-postgres requires a name for a database, received: {self.name}" - ), - ) - } - - @classmethod - def from_dict(cls, config_dict: dict) -> "DatabaseConfig": - """ - Because this returns a frozen dataclass, this method should be overridden if additional parameters are supplied. - """ - kwargs_dict = {"name": postgres_conform_part(ComponentName.Database, config_dict["name"])} - database: "DatabaseConfig" = super().from_dict(kwargs_dict) # type: ignore - return database - - @classmethod - def parse_model_node(cls, model_node: ModelNode) -> dict: - """ - Because this returns a `dict`, this method should be extended if additional parameters are supplied. - """ - config_dict = {"name": model_node.database} - return config_dict - - @classmethod - def parse_describe_relation_results(cls, describe_relation_results: agate.Row) -> dict: - """ - Because this returns a `dict`, this method should be extended if additional parameters are supplied. - """ - config_dict = {"name": describe_relation_results["databasename"]} - return config_dict diff --git a/plugins/postgres/dbt/adapters/postgres/materialization_config/materialized_view.py b/plugins/postgres/dbt/adapters/postgres/materialization_config/materialized_view.py deleted file mode 100644 index 0750029ba4d..00000000000 --- a/plugins/postgres/dbt/adapters/postgres/materialization_config/materialized_view.py +++ /dev/null @@ -1,216 +0,0 @@ -from collections import OrderedDict -from dataclasses import dataclass, field -from typing import Set, FrozenSet, List, Dict, Optional - -import agate -from dbt.adapters.materialization_config import ( - MaterializationConfig, - RelationConfigChangeset, - RelationConfigValidationMixin, - RelationConfigValidationRule, -) -from dbt.contracts.graph.nodes import ModelNode -from dbt.contracts.relation import ComponentName, RelationType -from dbt.exceptions import DbtRuntimeError - -from dbt.adapters.postgres.materialization_config.index import ( - PostgresIndexConfig, - PostgresIndexConfigChange, -) -from dbt.adapters.postgres.materialization_config.policy import ( - postgres_render, - postgres_conform_part, - MAX_CHARACTERS_IN_IDENTIFIER, -) -from dbt.adapters.postgres.materialization_config.schema import PostgresSchemaConfig - - -@dataclass(frozen=True, eq=True, unsafe_hash=True) -class PostgresMaterializedViewConfig(MaterializationConfig, RelationConfigValidationMixin): - """ - This config follows the specs found here: - https://www.postgresql.org/docs/current/sql-creatematerializedview.html - - The following parameters are configurable by dbt: - - name: name of the materialized view - - schema: schema that contains the materialized view - - query: the query that defines the view - - indexes: the collection (set) of indexes on the materialized view - - Applicable defaults for non-configurable parameters: - - method: `heap` - - tablespace_name: `default_tablespace` - - with_data: `True` - """ - - name: str - schema: PostgresSchemaConfig - query: str = field(hash=False, compare=False) - indexes: FrozenSet[PostgresIndexConfig] = field(default_factory=frozenset) - type: Optional[RelationType] = RelationType.MaterializedView - - @property - def fully_qualified_path(self) -> str: - return postgres_render( - OrderedDict( - { - ComponentName.Database: self.database_name, - ComponentName.Schema: self.schema_name, - ComponentName.Identifier: self.name, - } - ) - ) - - @property - def validation_rules(self) -> Set[RelationConfigValidationRule]: - """ - Validation rules at the materialized view level. All attribute level rules get run as a result of - `RelationConfigValidationMixin`. - - Returns: a set of rules that should evaluate to `True` (i.e. False == validation failure) - """ - return { - RelationConfigValidationRule( - validation_check=self.name is None - or len(self.name) <= MAX_CHARACTERS_IN_IDENTIFIER, - validation_error=DbtRuntimeError( - f"The materialized view name is more than {MAX_CHARACTERS_IN_IDENTIFIER} " - f"characters: {self.name}" - ), - ), - RelationConfigValidationRule( - validation_check=all({self.database_name, self.schema_name, self.name}), - validation_error=DbtRuntimeError( - f"dbt-snowflake requires all three parts of an object's path, received:/n" - f" database: {self.database_name}/n" - f" schema: {self.schema_name}/n" - f" identifier: {self.name}/n" - ), - ), - } - - @classmethod - def from_dict(cls, config_dict: dict) -> "PostgresMaterializedViewConfig": - """ - Creates an instance of this class given the dict representation - - This is generally used indirectly by calling either `from_model_node()` or `from_relation_results()` - - Args: - config_dict: a dict that aligns with the structure of this class, and it's attribute classes (e.g. indexes) - - Returns: an instance of this class - """ - kwargs_dict = { - "name": postgres_conform_part(ComponentName.Identifier, config_dict["name"]), - "schema": PostgresSchemaConfig.from_dict(config_dict["schema"]), - "query": config_dict["query"], - "indexes": frozenset( - PostgresIndexConfig.from_dict(index) for index in config_dict.get("indexes", {}) - ), - } - materialized_view: "PostgresMaterializedViewConfig" = super().from_dict(kwargs_dict) # type: ignore - return materialized_view - - @classmethod - def parse_model_node(cls, model_node: ModelNode) -> dict: - """ - Parse `RuntimeConfigObject.model` into a dict representation of a `PostgresMaterializedViewConfig` instance - - This is generally used indirectly by calling `from_model_node()`, but there are times when the dict - version is more appropriate. - - Args: - model_node: the `model` attribute from `config` in the jinja context - - Example `model_node`: - - ModelNode({ - ..., - "compiled_code": "create materialized view my_materialized_view as select * from my_ref_table;", - ..., - "config" { - ..., - "extra": { - ..., - "indexes": [ - {"columns": ["id"], "type": "hash", "unique": True}, - ], - ..., - }, - ..., - }, - ..., - "identifier": "my_materialized_view", - ..., - }) - - Returns: a dict representation of an instance of this class that can be passed into `from_dict()` - """ - indexes: List[dict] = model_node.config.extra.get("indexes", []) - config_dict = { - "name": model_node.identifier, - "schema": PostgresSchemaConfig.parse_model_node(model_node), - "query": (model_node.compiled_code or "").strip(), - "indexes": [PostgresIndexConfig.parse_model_node(index) for index in indexes], - } - return config_dict - - @classmethod - def parse_describe_relation_results( - cls, describe_relation_results: Dict[str, agate.Table] - ) -> dict: - """ - Parse `RelationResults` into a dict representation of a `PostgresMaterializedViewConfig` instance - - This is generally used indirectly by calling `from_relation_results()`, but there are times when the dict - version is more appropriate. - - Args: - describe_relation_results: the results of a set of queries that fully describe an instance of this class - - Example of `relation_results`: - - { - "materialized_view": agate.Table(agate.Row({ - "table_name": "my_materialized_view", - "query": "create materialized view my_materialized_view as select * from my_ref_table;", - })), - "indexes": agate.Table([ - agate.Row({"columns": ["id"], "type": "hash", "unique": True}), - ..., - ], - } - - Returns: a dict representation of an instance of this class that can be passed into `from_dict()` - """ - materialized_view_config: agate.Table = describe_relation_results.get("materialized_view") - materialized_view = materialized_view_config.rows[0] - indexes: agate.Table = describe_relation_results["indexes"] - - config_dict = { - "name": materialized_view["matviewname"], - "schema": PostgresSchemaConfig.parse_describe_relation_results(materialized_view), - "query": materialized_view["definition"].strip(), - "indexes": [ - PostgresIndexConfig.parse_describe_relation_results(index) - for index in indexes.rows - ], - } - return config_dict - - -@dataclass -class PostgresMaterializedViewConfigChangeset(RelationConfigChangeset): - indexes: Set[PostgresIndexConfigChange] = field(default_factory=set) - - @property - def requires_full_refresh(self) -> bool: - return ( - any(index.requires_full_refresh for index in self.indexes) - or super().requires_full_refresh - ) - - @property - def is_empty(self) -> bool: - return self.indexes == set() and super().is_empty diff --git a/plugins/postgres/dbt/adapters/postgres/materialization_config/schema.py b/plugins/postgres/dbt/adapters/postgres/materialization_config/schema.py deleted file mode 100644 index b6d08788637..00000000000 --- a/plugins/postgres/dbt/adapters/postgres/materialization_config/schema.py +++ /dev/null @@ -1,80 +0,0 @@ -from collections import OrderedDict -from dataclasses import dataclass -from typing import Set - -import agate -from dbt.adapters.materialization_config import ( - SchemaConfig, - RelationConfigValidationMixin, - RelationConfigValidationRule, -) -from dbt.contracts.graph.nodes import ModelNode -from dbt.contracts.relation import ComponentName -from dbt.exceptions import DbtRuntimeError - -from dbt.adapters.postgres.materialization_config.database import PostgresDatabaseConfig -from dbt.adapters.postgres.materialization_config.policy import ( - postgres_render, - postgres_conform_part, -) - - -@dataclass(frozen=True, eq=True, unsafe_hash=True) -class PostgresSchemaConfig(SchemaConfig, RelationConfigValidationMixin): - """ - This config follow the specs found here: - https://www.postgresql.org/docs/15/sql-createschema.html - - The following parameters are configurable by dbt: - - name: name of the schema - - database_name: name of the database - """ - - name: str - database: PostgresDatabaseConfig - - @property - def fully_qualified_path(self) -> str: - return postgres_render( - OrderedDict( - {ComponentName.Database: self.database_name, ComponentName.Schema: self.name} - ) - ) - - @property - def validation_rules(self) -> Set[RelationConfigValidationRule]: - return { - RelationConfigValidationRule( - validation_check=len(self.name or "") > 0, - validation_error=DbtRuntimeError( - f"dbt-postgres requires a name for a schema, received: {self.name}" - ), - ) - } - - @classmethod - def from_dict(cls, config_dict: dict) -> "SchemaConfig": - kwargs_dict = { - "name": postgres_conform_part(ComponentName.Schema, config_dict["name"]), - "database": PostgresDatabaseConfig.from_dict(config_dict["database"]), - } - schema: "SchemaConfig" = super().from_dict(kwargs_dict) # type: ignore - return schema - - @classmethod - def parse_model_node(cls, model_node: ModelNode) -> dict: - config_dict = { - "name": model_node.schema, - "database": PostgresDatabaseConfig.parse_model_node(model_node), - } - return config_dict - - @classmethod - def parse_describe_relation_results(cls, describe_relation_results: agate.Row) -> dict: - config_dict = { - "name": describe_relation_results["schemaname"], - "database": PostgresDatabaseConfig.parse_describe_relation_results( - describe_relation_results - ), - } - return config_dict diff --git a/plugins/postgres/dbt/adapters/postgres/relation.py b/plugins/postgres/dbt/adapters/postgres/relation/__init__.py similarity index 94% rename from plugins/postgres/dbt/adapters/postgres/relation.py rename to plugins/postgres/dbt/adapters/postgres/relation/__init__.py index db85b9e13ba..24a37960dcb 100644 --- a/plugins/postgres/dbt/adapters/postgres/relation.py +++ b/plugins/postgres/dbt/adapters/postgres/relation/__init__.py @@ -3,7 +3,7 @@ from dbt.adapters.base.relation import BaseRelation from dbt.exceptions import DbtRuntimeError -from dbt.adapters.postgres.materialization_config import ( +from dbt.adapters.postgres.relation.models import ( PostgresIncludePolicy, PostgresQuotePolicy, MAX_CHARACTERS_IN_IDENTIFIER, diff --git a/plugins/postgres/dbt/adapters/postgres/relation/models/__init__.py b/plugins/postgres/dbt/adapters/postgres/relation/models/__init__.py new file mode 100644 index 00000000000..d4c4ea771bd --- /dev/null +++ b/plugins/postgres/dbt/adapters/postgres/relation/models/__init__.py @@ -0,0 +1,17 @@ +from dbt.adapters.postgres.relation.models.database import PostgresDatabaseRelation +from dbt.adapters.postgres.relation.models.index import ( + PostgresIndexMethod, + PostgresIndexRelation, + PostgresIndexRelationChange, +) +from dbt.adapters.postgres.relation.models.materialized_view import ( + PostgresMaterializedViewRelation, + PostgresMaterializedViewRelationChangeset, +) +from dbt.adapters.postgres.relation.models.policy import ( + PostgresIncludePolicy, + PostgresQuotePolicy, + PostgresRenderPolicy, + MAX_CHARACTERS_IN_IDENTIFIER, +) +from dbt.adapters.postgres.relation.models.schema import PostgresSchemaRelation diff --git a/plugins/postgres/dbt/adapters/postgres/relation/models/database.py b/plugins/postgres/dbt/adapters/postgres/relation/models/database.py new file mode 100644 index 00000000000..d652be3e947 --- /dev/null +++ b/plugins/postgres/dbt/adapters/postgres/relation/models/database.py @@ -0,0 +1,46 @@ +from dataclasses import dataclass +from typing import Set + +from dbt.adapters.relation.models import DatabaseRelation +from dbt.adapters.validation import ValidationMixin, ValidationRule +from dbt.exceptions import DbtRuntimeError + +from dbt.adapters.postgres.relation.models.policy import PostgresRenderPolicy + + +@dataclass(frozen=True, eq=True, unsafe_hash=True) +class PostgresDatabaseRelation(DatabaseRelation, ValidationMixin): + """ + This config follow the specs found here: + https://www.postgresql.org/docs/current/sql-createdatabase.html + + The following parameters are configurable by dbt: + - name: name of the database + """ + + # attribution + name: str + + # configuration + render = PostgresRenderPolicy + + @classmethod + def from_dict(cls, config_dict) -> "PostgresDatabaseRelation": + database = super().from_dict(config_dict) + assert isinstance(database, PostgresDatabaseRelation) + return database + + @property + def validation_rules(self) -> Set[ValidationRule]: + """ + Returns: a set of rules that should evaluate to `True` (i.e. False == validation failure) + """ + return { + ValidationRule( + validation_check=len(self.name or "") > 0, + validation_error=DbtRuntimeError( + f"dbt-postgres requires a name to reference a database, received:\n" + f" database: {self.name}\n" + ), + ), + } diff --git a/plugins/postgres/dbt/adapters/postgres/materialization_config/index.py b/plugins/postgres/dbt/adapters/postgres/relation/models/index.py similarity index 54% rename from plugins/postgres/dbt/adapters/postgres/materialization_config/index.py rename to plugins/postgres/dbt/adapters/postgres/relation/models/index.py index 8fa89f3000b..54d683092d7 100644 --- a/plugins/postgres/dbt/adapters/postgres/materialization_config/index.py +++ b/plugins/postgres/dbt/adapters/postgres/relation/models/index.py @@ -1,19 +1,19 @@ +from copy import deepcopy from dataclasses import dataclass, field from typing import Set, FrozenSet import agate -from dbt.adapters.materialization_config import ( - RelationConfig, - RelationConfigValidationMixin, - RelationConfigValidationRule, - RelationConfigChangeAction, - RelationConfigChange, +from dbt.adapters.relation.models import ( + RelationComponent, + RelationChangeAction, + RelationChange, ) +from dbt.adapters.validation import ValidationMixin, ValidationRule from dbt.contracts.relation import ComponentName from dbt.dataclass_schema import StrEnum from dbt.exceptions import DbtRuntimeError -from dbt.adapters.postgres.materialization_config.policy import postgres_conform_part +from dbt.adapters.postgres.relation.models.policy import PostgresRenderPolicy class PostgresIndexMethod(StrEnum): @@ -30,7 +30,7 @@ def default(cls) -> "PostgresIndexMethod": @dataclass(frozen=True, eq=True, unsafe_hash=True) -class PostgresIndexConfig(RelationConfig, RelationConfigValidationMixin): +class PostgresIndexRelation(RelationComponent, ValidationMixin): """ This config fallows the specs found here: https://www.postgresql.org/docs/current/sql-createindex.html @@ -57,11 +57,27 @@ class PostgresIndexConfig(RelationConfig, RelationConfigValidationMixin): unique: bool = field(default=False, hash=True) method: PostgresIndexMethod = field(default=PostgresIndexMethod.default(), hash=True) + # configuration + render = PostgresRenderPolicy + + @property + def fully_qualified_path(self) -> str: + return "_".join( + { + *sorted( + self.render.part(ComponentName.Identifier, column) + for column in self.column_names + ), + str(self.unique), + str(self.method), + } + ).replace(self.render.quote_character, "") + @property - def validation_rules(self) -> Set[RelationConfigValidationRule]: + def validation_rules(self) -> Set[ValidationRule]: return { - RelationConfigValidationRule( - validation_check=self.column_names is not None, + ValidationRule( + validation_check=self.column_names != frozenset(), validation_error=DbtRuntimeError( "Indexes require at least one column, but none were provided" ), @@ -69,24 +85,47 @@ def validation_rules(self) -> Set[RelationConfigValidationRule]: } @classmethod - def from_dict(cls, config_dict) -> "PostgresIndexConfig": - kwargs_dict = { - "name": config_dict.get("name"), - "column_names": frozenset( - postgres_conform_part(ComponentName.Identifier, column) - for column in config_dict.get("column_names", set()) - ), - "unique": config_dict.get("unique"), - } + def from_dict(cls, config_dict) -> "PostgresIndexRelation": + # don't alter the incoming config + kwargs_dict = deepcopy(config_dict) + + # component-specific attributes + if column_names := config_dict.get("column_names"): + kwargs_dict.update({"column_names": frozenset(column_names)}) if method := config_dict.get("method"): kwargs_dict.update({"method": PostgresIndexMethod(method)}) - index: "PostgresIndexConfig" = super().from_dict(kwargs_dict) # type: ignore + index = super().from_dict(kwargs_dict) + assert isinstance(index, PostgresIndexRelation) return index @classmethod def parse_model_node(cls, model_node_entry: dict) -> dict: + """ + Parse a `ModelNode` instance into a `PostgresIndexRelation` instance as a dict + + This is generally used indirectly by calling `from_model_node()`, but there are times when the dict + version is more appropriate. + + Args: + model_node_entry: an entry from the `model` attribute (e.g. `config.model`) in the jinja context + + Example `model_node`: + + ModelNode({ + "config" { + "extra": { + "indexes": [{"columns": ["id"], "type": "hash", "unique": True},...], + ..., + }, + ..., + }, + ..., + }) + + Returns: a `PostgresIndexRelation` instance as a dict, can be passed into `from_dict` + """ config_dict = { "column_names": set(model_node_entry.get("columns", set())), "unique": model_node_entry.get("unique"), @@ -106,7 +145,7 @@ def parse_describe_relation_results(cls, describe_relation_results: agate.Row) - @dataclass(frozen=True, eq=True, unsafe_hash=True) -class PostgresIndexConfigChange(RelationConfigChange, RelationConfigValidationMixin): +class PostgresIndexRelationChange(RelationChange, ValidationMixin): """ Example of an index change: { @@ -129,33 +168,33 @@ class PostgresIndexConfigChange(RelationConfigChange, RelationConfigValidationMi } """ - context: PostgresIndexConfig + context: PostgresIndexRelation @property def requires_full_refresh(self) -> bool: return False @property - def validation_rules(self) -> Set[RelationConfigValidationRule]: + def validation_rules(self) -> Set[ValidationRule]: return { - RelationConfigValidationRule( + ValidationRule( validation_check=self.action - in {RelationConfigChangeAction.create, RelationConfigChangeAction.drop}, + in {RelationChangeAction.create, RelationChangeAction.drop}, validation_error=DbtRuntimeError( "Invalid operation, only `drop` and `create` changes are supported for indexes." ), ), - RelationConfigValidationRule( + ValidationRule( validation_check=not ( - self.action == RelationConfigChangeAction.drop and self.context.name is None + self.action == RelationChangeAction.drop and self.context.name is None ), validation_error=DbtRuntimeError( "Invalid operation, attempting to drop an index with no name." ), ), - RelationConfigValidationRule( + ValidationRule( validation_check=not ( - self.action == RelationConfigChangeAction.create + self.action == RelationChangeAction.create and self.context.column_names == set() ), validation_error=DbtRuntimeError( @@ -163,3 +202,30 @@ def validation_rules(self) -> Set[RelationConfigValidationRule]: ), ), } + + +def index_config_changes( + existing_indexes: FrozenSet[PostgresIndexRelation], + new_indexes: FrozenSet[PostgresIndexRelation], +) -> Set[PostgresIndexRelationChange]: + """ + Get the index updates that will occur as a result of a new run + + There are four scenarios: + + 1. Indexes are equal -> don't return these + 2. Index is new -> create these + 3. Index is old -> drop these + 4. Indexes are not equal -> drop old, create new -> two actions + + Returns: a set of index updates in the form {"action": "drop/create", "context": } + """ + drop_changes = set( + PostgresIndexRelationChange(action=RelationChangeAction.drop, context=index) + for index in existing_indexes.difference(new_indexes) + ) + create_changes = set( + PostgresIndexRelationChange(action=RelationChangeAction.create, context=index) + for index in new_indexes.difference(existing_indexes) + ) + return set().union(drop_changes, create_changes) diff --git a/plugins/postgres/dbt/adapters/postgres/relation/models/materialized_view.py b/plugins/postgres/dbt/adapters/postgres/relation/models/materialized_view.py new file mode 100644 index 00000000000..cf6c8f08b8d --- /dev/null +++ b/plugins/postgres/dbt/adapters/postgres/relation/models/materialized_view.py @@ -0,0 +1,236 @@ +from copy import deepcopy +from dataclasses import dataclass, field +from typing import Dict, FrozenSet, Optional, Set + +import agate +from dbt.adapters.relation.models import ( + MaterializedViewRelation, + MaterializedViewRelationChangeset, + Relation, +) +from dbt.adapters.validation import ValidationMixin, ValidationRule +from dbt.contracts.graph.nodes import ModelNode +from dbt.exceptions import DbtRuntimeError + +from dbt.adapters.postgres.relation.models.index import ( + index_config_changes, + PostgresIndexRelation, + PostgresIndexRelationChange, +) +from dbt.adapters.postgres.relation.models.policy import ( + PostgresRenderPolicy, + MAX_CHARACTERS_IN_IDENTIFIER, +) +from dbt.adapters.postgres.relation.models.schema import PostgresSchemaRelation + + +@dataclass(frozen=True, eq=True, unsafe_hash=True) +class PostgresMaterializedViewRelation(MaterializedViewRelation, ValidationMixin): + """ + This config follows the specs found here: + https://www.postgresql.org/docs/current/sql-creatematerializedview.html + + The following parameters are configurable by dbt: + - name: name of the materialized view + - schema: schema that contains the materialized view + - query: the query that defines the view + - indexes: the collection (set) of indexes on the materialized view + + Applicable defaults for non-configurable parameters: + - method: `heap` + - tablespace_name: `default_tablespace` + - with_data: `True` + """ + + # attribution + name: str + schema: PostgresSchemaRelation + query: str = field(hash=False, compare=False) + indexes: Optional[FrozenSet[PostgresIndexRelation]] = field(default_factory=frozenset) + + # configuration + render = PostgresRenderPolicy + SchemaParser = PostgresSchemaRelation + can_be_renamed = True + + @property + def validation_rules(self) -> Set[ValidationRule]: + """ + Validation rules at the materialized view level. All attribute level rules get run as a result of + `ValidationMixin`. + + Returns: a set of rules that should evaluate to `True` (i.e. False == validation failure) + """ + return { + ValidationRule( + validation_check=self.name is None + or len(self.name) <= MAX_CHARACTERS_IN_IDENTIFIER, + validation_error=DbtRuntimeError( + f"The materialized view name is more than the max allowed length" + f"of {MAX_CHARACTERS_IN_IDENTIFIER} characters.\n" + f" name: {self.name}\n" + f" characters: {len(self.name)}\n" + ), + ), + ValidationRule( + validation_check=all({self.database_name, self.schema_name, self.name}), + validation_error=DbtRuntimeError( + f"dbt-postgres requires all three parts of an object's path, received:\n" + f" database: {self.database_name}\n" + f" schema: {self.schema_name}\n" + f" identifier: {self.name}\n" + ), + ), + } + + @classmethod + def from_dict(cls, config_dict: dict) -> "PostgresMaterializedViewRelation": + """ + Creates an instance of this class given the dict representation + + This is generally used indirectly by calling either `from_model_node()` or `from_relation_results()` + + Args: + config_dict: a dict that aligns with the structure of this class, and it's attribute classes (e.g. indexes) + + Returns: an instance of this class + """ + # don't alter the incoming config + kwargs_dict = deepcopy(config_dict) + + # adapter-specific attributes + if indexes := config_dict.get("indexes"): + kwargs_dict.update( + { + "indexes": frozenset( + PostgresIndexRelation.from_dict(index) for index in indexes + ), + } + ) + + materialized_view = super().from_dict(kwargs_dict) + assert isinstance(materialized_view, PostgresMaterializedViewRelation) + return materialized_view + + @classmethod + def parse_model_node(cls, model_node: ModelNode) -> dict: + """ + Parse a `ModelNode` instance into a `PostgresMaterializedViewRelation` instance as a dict + + This is generally used indirectly by calling `from_model_node()`, but there are times when the dict + version is more appropriate. + + Args: + model_node: the `model` attribute (e.g. `config.model`) in the jinja context + + Example `model_node`: + + ModelNode({ + "compiled_code": "create materialized view my_materialized_view as select * from my_table;", + "config" { + "extra": { + "indexes": [{"columns": ["id"], "type": "hash", "unique": True},...], + ..., + }, + ..., + }, + "database": "my_database", + "identifier": "my_materialized_view", + "schema": "my_schema", + ..., + }) + + Returns: a `PostgresMaterializedViewRelation` instance as a dict, can be passed into `from_dict` + """ + config_dict = super().parse_model_node(model_node) + + if indexes := model_node.config.extra.get("indexes"): + config_dict.update( + { + "indexes": [ + PostgresIndexRelation.parse_model_node(index) for index in indexes + ], + } + ) + + return config_dict + + @classmethod + def parse_describe_relation_results( + cls, describe_relation_results: Dict[str, agate.Table] + ) -> dict: + """ + Parse `RelationResults` into a dict representation of a `PostgresMaterializedViewConfig` instance + + This is generally used indirectly by calling `from_relation_results()`, but there are times when the dict + version is more appropriate. + + Args: + describe_relation_results: the results of a set of queries that fully describe an instance of this class + + Example of `relation_results`: + + { + "materialization": agate.Table(agate.Row({ + "database": "my_database", + "name": "my_materialized_view", + "query": "create materialized view my_materialized_view as select * from my_ref_table;", + "schema": "my_schema", + })), + "indexes": agate.Table([ + agate.Row({"columns": ["id"], "type": "hash", "unique": True}), + ..., + ], + } + + Returns: a dict representation of an instance of this class that can be passed into `from_dict()` + """ + config_dict = super().parse_describe_relation_results(describe_relation_results) + + if indexes := describe_relation_results.get("indexes"): + config_dict.update( + { + "indexes": [ + PostgresIndexRelation.parse_describe_relation_results(index) + for index in indexes.rows + ], + } + ) + + return config_dict + + +@dataclass +class PostgresMaterializedViewRelationChangeset(MaterializedViewRelationChangeset): + indexes: Set[PostgresIndexRelationChange] = field(default_factory=set) + + @classmethod + def parse_relations(cls, existing_relation: Relation, target_relation: Relation) -> dict: + try: + assert isinstance(existing_relation, PostgresMaterializedViewRelation) + assert isinstance(target_relation, PostgresMaterializedViewRelation) + except AssertionError: + raise DbtRuntimeError( + f"Two Postgres materialized view relations were expected, but received:\n" + f" existing: {existing_relation}\n" + f" new: {target_relation}\n" + ) + + config_dict = super().parse_relations(existing_relation, target_relation) + + config_dict.update( + {"indexes": index_config_changes(existing_relation.indexes, target_relation.indexes)} + ) + + return config_dict + + @property + def requires_full_refresh(self) -> bool: + return ( + any(index.requires_full_refresh for index in self.indexes) + or super().requires_full_refresh + ) + + @property + def is_empty(self) -> bool: + return self.indexes == set() and super().is_empty diff --git a/plugins/postgres/dbt/adapters/postgres/materialization_config/policy.py b/plugins/postgres/dbt/adapters/postgres/relation/models/policy.py similarity index 57% rename from plugins/postgres/dbt/adapters/postgres/materialization_config/policy.py rename to plugins/postgres/dbt/adapters/postgres/relation/models/policy.py index 963f57742a0..4e30fa9bd26 100644 --- a/plugins/postgres/dbt/adapters/postgres/materialization_config/policy.py +++ b/plugins/postgres/dbt/adapters/postgres/relation/models/policy.py @@ -1,13 +1,6 @@ from dataclasses import dataclass -from functools import partial - -from dbt.adapters.materialization_config import ( - IncludePolicy, - QuotePolicy, - conform_part, - render_part, - render, -) + +from dbt.adapters.relation.models import IncludePolicy, QuotePolicy, RenderPolicy MAX_CHARACTERS_IN_IDENTIFIER = 63 @@ -31,18 +24,9 @@ def quote_character(self) -> str: return '"' -postgres_conform_part = partial( - conform_part, - quote_policy=PostgresQuotePolicy(), -) -postgres_render_part = partial( - render_part, - quote_policy=PostgresQuotePolicy(), - include_policy=PostgresIncludePolicy(), -) -postgres_render = partial( - render, +PostgresRenderPolicy = RenderPolicy( quote_policy=PostgresQuotePolicy(), include_policy=PostgresIncludePolicy(), + quote_character='"', delimiter=".", ) diff --git a/plugins/postgres/dbt/adapters/postgres/relation/models/schema.py b/plugins/postgres/dbt/adapters/postgres/relation/models/schema.py new file mode 100644 index 00000000000..41ac6af204c --- /dev/null +++ b/plugins/postgres/dbt/adapters/postgres/relation/models/schema.py @@ -0,0 +1,49 @@ +from dataclasses import dataclass +from typing import Set + +from dbt.adapters.relation.models import SchemaRelation +from dbt.adapters.validation import ValidationMixin, ValidationRule +from dbt.exceptions import DbtRuntimeError + +from dbt.adapters.postgres.relation.models.database import PostgresDatabaseRelation +from dbt.adapters.postgres.relation.models.policy import PostgresRenderPolicy + + +@dataclass(frozen=True, eq=True, unsafe_hash=True) +class PostgresSchemaRelation(SchemaRelation, ValidationMixin): + """ + This config follow the specs found here: + https://www.postgresql.org/docs/15/sql-createschema.html + + The following parameters are configurable by dbt: + - name: name of the schema + - database_name: name of the database + """ + + # attribution + name: str + + # configuration + render = PostgresRenderPolicy + DatabaseParser = PostgresDatabaseRelation + + @classmethod + def from_dict(cls, config_dict) -> "PostgresSchemaRelation": + schema = super().from_dict(config_dict) + assert isinstance(schema, PostgresSchemaRelation) + return schema + + @property + def validation_rules(self) -> Set[ValidationRule]: + """ + Returns: a set of rules that should evaluate to `True` (i.e. False == validation failure) + """ + return { + ValidationRule( + validation_check=len(self.name or "") > 0, + validation_error=DbtRuntimeError( + f"dbt-postgres requires a name to reference a schema, received:\n" + f" schema: {self.name}\n" + ), + ), + } diff --git a/plugins/postgres/dbt/include/postgres/macros/adapters.sql b/plugins/postgres/dbt/include/postgres/macros/adapters.sql index 816663a0637..71ff9f4e517 100644 --- a/plugins/postgres/dbt/include/postgres/macros/adapters.sql +++ b/plugins/postgres/dbt/include/postgres/macros/adapters.sql @@ -25,38 +25,6 @@ ); {%- endmacro %} -{% macro postgres__get_create_index_sql(relation, index_dict) -%} - {%- set index_config = adapter.parse_index(index_dict) -%} - {%- set comma_separated_columns = ", ".join(index_config.columns) -%} - {%- set index_name = index_config.render(relation) -%} - - create {% if index_config.unique -%} - unique - {%- endif %} index if not exists - "{{ index_name }}" - on {{ relation }} {% if index_config.type -%} - using {{ index_config.type }} - {%- endif %} - ({{ comma_separated_columns }}); -{%- endmacro %} - -{% macro postgres__create_schema(relation) -%} - {% if relation.database -%} - {{ adapter.verify_database(relation.database) }} - {%- endif -%} - {%- call statement('create_schema') -%} - create schema if not exists {{ relation.without_identifier().include(database=False) }} - {%- endcall -%} -{% endmacro %} - -{% macro postgres__drop_schema(relation) -%} - {% if relation.database -%} - {{ adapter.verify_database(relation.database) }} - {%- endif -%} - {%- call statement('drop_schema') -%} - drop schema if exists {{ relation.without_identifier().include(database=False) }} cascade - {%- endcall -%} -{% endmacro %} {% macro postgres__get_columns_in_relation(relation) -%} {% call statement('get_columns_in_relation', fetch_result=True) %} @@ -116,26 +84,6 @@ information_schema {%- endmacro %} -{% macro postgres__list_schemas(database) %} - {% if database -%} - {{ adapter.verify_database(database) }} - {%- endif -%} - {% call statement('list_schemas', fetch_result=True, auto_begin=False) %} - select distinct nspname from pg_namespace - {% endcall %} - {{ return(load_result('list_schemas').table) }} -{% endmacro %} - -{% macro postgres__check_schema_exists(information_schema, schema) -%} - {% if information_schema.database -%} - {{ adapter.verify_database(information_schema.database) }} - {%- endif -%} - {% call statement('check_schema_exists', fetch_result=True, auto_begin=False) %} - select count(*) from pg_namespace where nspname = '{{ schema }}' - {% endcall %} - {{ return(load_result('check_schema_exists').table) }} -{% endmacro %} - {# Postgres tables have a maximum length of 63 characters, anything longer is silently truncated. Temp and backup relations add a lot of extra characters to the end of table names to ensure uniqueness. @@ -219,8 +167,3 @@ {% macro postgres__copy_grants() %} {{ return(False) }} {% endmacro %} - - -{%- macro postgres__get_drop_index_sql(relation, index_name) -%} - drop index if exists "{{ index_name }}" -{%- endmacro -%} diff --git a/plugins/postgres/dbt/include/postgres/macros/catalog.sql b/plugins/postgres/dbt/include/postgres/macros/get_catalog.sql similarity index 100% rename from plugins/postgres/dbt/include/postgres/macros/catalog.sql rename to plugins/postgres/dbt/include/postgres/macros/get_catalog.sql diff --git a/plugins/postgres/dbt/include/postgres/macros/get_relations.sql b/plugins/postgres/dbt/include/postgres/macros/get_relations.sql new file mode 100644 index 00000000000..4e20940e37a --- /dev/null +++ b/plugins/postgres/dbt/include/postgres/macros/get_relations.sql @@ -0,0 +1,77 @@ +{% macro postgres_get_relations () -%} + + {# /* + -- in pg_depend, objid is the dependent, refobjid is the referenced object + -- > a pg_depend entry indicates that the referenced object cannot be + -- > dropped without also dropping the dependent object. + */ #} + + {%- call statement('relations', fetch_result=True) -%} + with relation as ( + select + pg_rewrite.ev_class as class, + pg_rewrite.oid as id + from pg_rewrite + ), + class as ( + select + oid as id, + relname as name, + relnamespace as schema, + relkind as kind + from pg_class + ), + dependency as ( + select distinct + pg_depend.objid as id, + pg_depend.refobjid as ref + from pg_depend + ), + schema as ( + select + pg_namespace.oid as id, + pg_namespace.nspname as name + from pg_namespace + where nspname != 'information_schema' and nspname not like 'pg\_%' + ), + referenced as ( + select + relation.id AS id, + referenced_class.name , + referenced_class.schema , + referenced_class.kind + from relation + join class as referenced_class on relation.class=referenced_class.id + where referenced_class.kind in ('r', 'v', 'm') + ), + relationships as ( + select + referenced.name as referenced_name, + referenced.schema as referenced_schema_id, + dependent_class.name as dependent_name, + dependent_class.schema as dependent_schema_id, + referenced.kind as kind + from referenced + join dependency on referenced.id=dependency.id + join class as dependent_class on dependency.ref=dependent_class.id + where + (referenced.name != dependent_class.name or + referenced.schema != dependent_class.schema) + ) + + select + referenced_schema.name as referenced_schema, + relationships.referenced_name as referenced_name, + dependent_schema.name as dependent_schema, + relationships.dependent_name as dependent_name + from relationships + join schema as dependent_schema on relationships.dependent_schema_id=dependent_schema.id + join schema as referenced_schema on relationships.referenced_schema_id=referenced_schema.id + group by referenced_schema, referenced_name, dependent_schema, dependent_name + order by referenced_schema, referenced_name, dependent_schema, dependent_name; + + {%- endcall -%} + + {{ return(load_result('relations').table) }} + +{% endmacro %} diff --git a/plugins/postgres/dbt/include/postgres/macros/index.sql b/plugins/postgres/dbt/include/postgres/macros/index.sql deleted file mode 100644 index 680474a58f1..00000000000 --- a/plugins/postgres/dbt/include/postgres/macros/index.sql +++ /dev/null @@ -1,100 +0,0 @@ -{#- /* - This file contains DDL that gets consumed in the Postgres implementation of the materialized view materialization - in `dbt/include/postgres/macros/materializations/materialized_view.sql`. - These macros could be used elsewhere as they do not care that they are being called by a materialization; - but the original intention was to support the materialization of materialized views. These macros represent - the basic interactions dbt-postgres requires of indexes in Postgres: - - ALTER - - CREATE - - DESCRIBE - - DROP - These macros all take a PostgresIndexConfig instance and/or a MaterializationConfigBase as an input. - These classes can be found in the following files, respectively: - `dbt/adapters/postgres/relation_configs/index.py` - `dbt/adapters/relation_configs/materialization.py` - - Used in: - `dbt/include/postgres/macros/materializations/materialized_view.sql` - Uses: - `dbt/adapters/postgres/relation_configs/index.py` - `dbt/adapters/postgres/relation_configs/materialized_view.py` -*/ -#} - - -{% macro postgres__alter_indexes_sql(materialization_config, index_changeset) -%} - {{- log('Applying UPDATE INDEXES to: ' ~ materialization_config.fully_qualified_path) -}} - - {%- for _index_change in index_changeset -%} - {%- set _index_config = _index_change.context -%} - - {%- if _index_change.action == adapter.Materialization.ChangeAction.drop -%} - {{ postgres__drop_index_sql(_index_config) }}; - - {%- elif _index_change.action == adapter.Materialization.ChangeAction.create -%} - {{ postgres__create_index_sql(materialization_config, _index_config) }}; - - {%- endif -%} - {%- endfor -%} - -{%- endmacro %} - - -{% macro postgres__create_indexes_sql(materialization_config) -%} - - {% for _index_config in materialization_config.indexes -%} - {{- postgres__create_index_sql(materialization_config, _index_config) -}}; - {%- endfor -%} - -{%- endmacro %} - - -{% macro postgres__create_index_sql(materialization_config, index_config) -%} - - {%- set _index_name = adapter.Materialization.generate_index_name(materialization_config, index_config) -%} - - create {% if index_config.unique -%}unique{%- endif %} index if not exists "{{ _index_name }}" - on {{ materialization_config.fully_qualified_path }} - using {{ index_config.method }} - ( - {{ ", ".join(index_config.column_names) }} - ) - -{%- endmacro %} - - -{% macro postgres__describe_indexes_sql(materialization) %} - {%- if adapter.is_materialization_config(materialization) -%} - {%- set _name = materialization.name %} - {%- set _schema = materialization.schema_name %} - {%- else -%} - {%- set _name = materialization.identifier %} - {%- set _schema = materialization.schema %} - {%- endif -%} - select - i.relname as name, - m.amname as method, - ix.indisunique as "unique", - array_to_string(array_agg(a.attname), ',') as column_names - from pg_index ix - join pg_class i - on i.oid = ix.indexrelid - join pg_am m - on m.oid=i.relam - join pg_class t - on t.oid = ix.indrelid - join pg_namespace n - on n.oid = t.relnamespace - join pg_attribute a - on a.attrelid = t.oid - and a.attnum = ANY(ix.indkey) - where t.relname ilike '{{ _name }}' - and n.nspname ilike '{{ _schema }}' - and t.relkind in ('r', 'm') - group by 1, 2, 3 - order by 1, 2, 3 -{% endmacro %} - - -{% macro postgres__drop_index_sql(index_config) -%} - drop index if exists "{{ index_config.name }}" cascade; -{%- endmacro %} diff --git a/plugins/postgres/dbt/include/postgres/macros/materializations/replace.sql b/plugins/postgres/dbt/include/postgres/macros/materializations/replace.sql deleted file mode 100644 index 6725793a92a..00000000000 --- a/plugins/postgres/dbt/include/postgres/macros/materializations/replace.sql +++ /dev/null @@ -1,25 +0,0 @@ -{%- macro postgres__replace_sql(existing_materialization, new_materialization) -%} - - {#- new_materialization is always a MaterializationConfig instance -#} - {%- set intermediate_materialization = adapter.Materialization.make_intermediate(new_materialization) -%} - {%- set new_name = new_materialization.name -%} - - - {%- if adapter.is_materialization_config(existing_materialization) -%} - {%- set backup_name = adapter.make_backup_name(existing_materialization) -%} - - {%- elif adapter.is_base_relation(existing_materialization) -%} - {%- set backup_relation = make_backup_relation(existing_materialization, existing_materialization.type) -%} - {%- set backup_name = backup_relation.identifier -%} - - {%- else -%} - {{- exceptions.raise_compiler_error("Unexpected materialization type: " ~ existing_materialization) -}} - - {%- endif -%} - - - {{ create_sql(intermediate_materialization) }}; - {{ rename_sql(existing_materialization, backup_name) }}; - {{ rename_sql(intermediate_materialization, new_name) }} - -{%- endmacro -%} diff --git a/plugins/postgres/dbt/include/postgres/macros/materializations/table.sql b/plugins/postgres/dbt/include/postgres/macros/materializations/table.sql deleted file mode 100644 index eb70eb1c1d5..00000000000 --- a/plugins/postgres/dbt/include/postgres/macros/materializations/table.sql +++ /dev/null @@ -1,8 +0,0 @@ -{# /* - These are BaseRelation instances, not MaterializationConfig instances -*/ #} - - -{%- macro postgres__rename_table_sql(table, new_name) -%} - alter table {{ table }} rename to {{ new_name }} -{%- endmacro -%} diff --git a/plugins/postgres/dbt/include/postgres/macros/materializations/view.sql b/plugins/postgres/dbt/include/postgres/macros/materializations/view.sql deleted file mode 100644 index ffbd24f3325..00000000000 --- a/plugins/postgres/dbt/include/postgres/macros/materializations/view.sql +++ /dev/null @@ -1,8 +0,0 @@ -{# /* - These are BaseRelation instances, not MaterializationConfig instances -*/ #} - - -{%- macro postgres__rename_view_sql(view, new_name) -%} - alter view {{ view }} rename to {{ new_name }} -{%- endmacro -%} diff --git a/plugins/postgres/dbt/include/postgres/macros/relation_components/index.sql b/plugins/postgres/dbt/include/postgres/macros/relation_components/index.sql new file mode 100644 index 00000000000..d4f12cba0e0 --- /dev/null +++ b/plugins/postgres/dbt/include/postgres/macros/relation_components/index.sql @@ -0,0 +1,124 @@ +{#- /* + This file contains DDL that gets consumed in the Postgres implementation of the materialized view materialization. + These macros could be used elsewhere as they do not care that they are being called by a materialization; + but the original intention was to support the materialization of materialized views. These macros represent + the basic interactions dbt-postgres requires of indexes in Postgres: + - ALTER + - CREATE + - DESCRIBE + - DROP + These macros all take a `PostgresIndexRelation` instance and/or a `Relation` instance as an input. + These classes can be found in the following files, respectively: + `dbt/adapters/postgres/relation_configs/index.py` + `dbt/adapters/relation/models/_relation.py` + + Used in: + `dbt/include/postgres/macros/relations/materialized_view.sql` + Uses: + `dbt/adapters/postgres/relation/models/index.py` + `dbt/adapters/postgres/relation/models/materialized_view.py` +*/ -#} + + +{% macro postgres__alter_indexes_template(relation, index_changeset) -%} + {{- log('Applying ALTER INDEXES to: ' ~ relation) -}} + + {%- for _change in index_changeset -%} + {%- set _index = _change.context -%} + + {% if _change.action == 'drop' -%} + {{ postgres__drop_index_template(relation, _index) }}; + + {% elif _change.action == 'create' -%} + {{ postgres__create_index_template(relation, _index) }}; + + {%- endif -%} + {%- endfor -%} + +{%- endmacro %} + + +{% macro postgres__create_indexes_template(relation) -%} + + {% for _index in relation.indexes -%} + {{- postgres__create_index_template(relation, _index) -}} + {%- if not loop.last %};{% endif -%} + {%- endfor -%} + +{%- endmacro %} + + +{% macro postgres__create_index_template(relation, index) -%} + + {%- set _index_name = adapter.generate_index_name(relation, index) -%} + + create {% if index.unique -%}unique{%- endif %} index if not exists "{{ _index_name }}" + on {{ relation.fully_qualified_path }} + using {{ index.method }} + ( + {{ ", ".join(index.column_names) }} + ) + +{%- endmacro %} + + +{% macro postgres__describe_indexes_template(relation) %} + {%- if adapter.is_relation_model(relation) -%} + {%- set _name = relation.name %} + {%- set _schema = relation.schema_name %} + {%- else -%} + {%- set _name = relation.identifier %} + {%- set _schema = relation.schema %} + {%- endif -%} + select + i.relname as name, + m.amname as method, + ix.indisunique as "unique", + array_to_string(array_agg(a.attname), ',') as column_names + from pg_index ix + join pg_class i + on i.oid = ix.indexrelid + join pg_am m + on m.oid=i.relam + join pg_class t + on t.oid = ix.indrelid + join pg_namespace n + on n.oid = t.relnamespace + join pg_attribute a + on a.attrelid = t.oid + and a.attnum = ANY(ix.indkey) + where t.relname ilike '{{ _name }}' + and n.nspname ilike '{{ _schema }}' + and t.relkind in ('r', 'm') + group by 1, 2, 3 + order by 1, 2, 3 +{% endmacro %} + + +{% macro postgres__drop_index_template(relation, index) -%} + drop index if exists "{{ relation.schema_name }}"."{{ index.name }}" cascade +{%- endmacro %} + + +{# /* + These are `BaseRelation` versions. The `BaseRelation` workflows are different. +*/ #} +{% macro postgres__get_create_index_sql(relation, index_dict) -%} + {%- set index_config = adapter.parse_index(index_dict) -%} + {%- set comma_separated_columns = ", ".join(index_config.columns) -%} + {%- set index_name = index_config.render(relation) -%} + + create {% if index_config.unique -%} + unique + {%- endif %} index if not exists + "{{ index_name }}" + on {{ relation }} {% if index_config.type -%} + using {{ index_config.type }} + {%- endif %} + ({{ comma_separated_columns }}); +{%- endmacro %} + + +{%- macro postgres__get_drop_index_sql(relation, index_name) -%} + drop index if exists "{{ index_name }}" +{%- endmacro -%} diff --git a/plugins/postgres/dbt/include/postgres/macros/relation_components/schema.sql b/plugins/postgres/dbt/include/postgres/macros/relation_components/schema.sql new file mode 100644 index 00000000000..04ec7a216aa --- /dev/null +++ b/plugins/postgres/dbt/include/postgres/macros/relation_components/schema.sql @@ -0,0 +1,42 @@ +{# /* + These are `BaseRelation` versions. The `BaseRelation` workflows are different. +*/ #} +{% macro postgres__create_schema(relation) -%} + {% if relation.database -%} + {{ adapter.verify_database(relation.database) }} + {%- endif -%} + {%- call statement('create_schema') -%} + create schema if not exists {{ relation.without_identifier().include(database=False) }} + {%- endcall -%} +{% endmacro %} + + +{% macro postgres__drop_schema(relation) -%} + {% if relation.database -%} + {{ adapter.verify_database(relation.database) }} + {%- endif -%} + {%- call statement('drop_schema') -%} + drop schema if exists {{ relation.without_identifier().include(database=False) }} cascade + {%- endcall -%} +{% endmacro %} + + +{% macro postgres__list_schemas(database) %} + {% if database -%} + {{ adapter.verify_database(database) }} + {%- endif -%} + {% call statement('list_schemas', fetch_result=True, auto_begin=False) %} + select distinct nspname from pg_namespace + {% endcall %} + {{ return(load_result('list_schemas').table) }} +{% endmacro %} + +{% macro postgres__check_schema_exists(information_schema, schema) -%} + {% if information_schema.database -%} + {{ adapter.verify_database(information_schema.database) }} + {%- endif -%} + {% call statement('check_schema_exists', fetch_result=True, auto_begin=False) %} + select count(*) from pg_namespace where nspname = '{{ schema }}' + {% endcall %} + {{ return(load_result('check_schema_exists').table) }} +{% endmacro %} diff --git a/plugins/postgres/dbt/include/postgres/macros/relations.sql b/plugins/postgres/dbt/include/postgres/macros/relations.sql deleted file mode 100644 index 9966c5db2e8..00000000000 --- a/plugins/postgres/dbt/include/postgres/macros/relations.sql +++ /dev/null @@ -1,76 +0,0 @@ -{% macro postgres_get_relations () -%} - - {# - -- in pg_depend, objid is the dependent, refobjid is the referenced object - -- > a pg_depend entry indicates that the referenced object cannot be - -- > dropped without also dropping the dependent object. - #} - - {%- call statement('relations', fetch_result=True) -%} - with relation as ( - select - pg_rewrite.ev_class as class, - pg_rewrite.oid as id - from pg_rewrite - ), - class as ( - select - oid as id, - relname as name, - relnamespace as schema, - relkind as kind - from pg_class - ), - dependency as ( - select distinct - pg_depend.objid as id, - pg_depend.refobjid as ref - from pg_depend - ), - schema as ( - select - pg_namespace.oid as id, - pg_namespace.nspname as name - from pg_namespace - where nspname != 'information_schema' and nspname not like 'pg\_%' - ), - referenced as ( - select - relation.id AS id, - referenced_class.name , - referenced_class.schema , - referenced_class.kind - from relation - join class as referenced_class on relation.class=referenced_class.id - where referenced_class.kind in ('r', 'v', 'm') - ), - relationships as ( - select - referenced.name as referenced_name, - referenced.schema as referenced_schema_id, - dependent_class.name as dependent_name, - dependent_class.schema as dependent_schema_id, - referenced.kind as kind - from referenced - join dependency on referenced.id=dependency.id - join class as dependent_class on dependency.ref=dependent_class.id - where - (referenced.name != dependent_class.name or - referenced.schema != dependent_class.schema) - ) - - select - referenced_schema.name as referenced_schema, - relationships.referenced_name as referenced_name, - dependent_schema.name as dependent_schema, - relationships.dependent_name as dependent_name - from relationships - join schema as dependent_schema on relationships.dependent_schema_id=dependent_schema.id - join schema as referenced_schema on relationships.referenced_schema_id=referenced_schema.id - group by referenced_schema, referenced_name, dependent_schema, dependent_name - order by referenced_schema, referenced_name, dependent_schema, dependent_name; - - {%- endcall -%} - - {{ return(load_result('relations').table) }} -{% endmacro %} diff --git a/plugins/postgres/dbt/include/postgres/macros/materializations/materialized_view.sql b/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view.sql similarity index 57% rename from plugins/postgres/dbt/include/postgres/macros/materializations/materialized_view.sql rename to plugins/postgres/dbt/include/postgres/macros/relations/materialized_view.sql index c48a0a956a5..c26a3c1089f 100644 --- a/plugins/postgres/dbt/include/postgres/macros/materializations/materialized_view.sql +++ b/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view.sql @@ -21,78 +21,85 @@ */ -#} -{%- macro postgres__alter_materialized_view_sql(existing_materialized_view, new_materialized_view) -%} +{%- macro postgres__alter_materialized_view_template(existing_materialized_view, target_materialized_view) -%} {#- /* We need to get the config changeset to determine if we require a full refresh (happens if any change in the changeset requires a full refresh or if an unmonitored change was detected) or if we can get away with altering the dynamic table in place. */ -#} - {%- set _config_changeset = adapter.Materialization.materialized_view_config_changeset(existing_materialized_view, new_materialized_view) -%} - {%- if _config_changeset.requires_full_refresh -%} - {{ replace_materialized_view_sql(new_materialized_view) }} + {%- if target_materialized_view == existing_materialized_view -%} + {{- exceptions.warn("No changes were identified for: " ~ existing_materialized_view) -}} {%- else -%} - {{ postgres__alter_indexes_sql(new_materialized_view, _config_changeset.indexes) }} + {%- set _changeset = adapter.make_changeset(existing_materialized_view, target_materialized_view) -%} + {%- if _changeset.requires_full_refresh -%} + {{ replace_template(existing_materialized_view, target_materialized_view) }} + + {%- else -%} + {{ postgres__alter_indexes_template(existing_materialized_view, _changeset.indexes) }} + + {%- endif -%} {%- endif -%} {%- endmacro -%} -{%- macro postgres__create_materialized_view_sql(materialized_view) -%} +{%- macro postgres__create_materialized_view_template(materialized_view) -%} create materialized view {{ materialized_view.fully_qualified_path }} as {{ materialized_view.query }} ; - - {{ postgres__create_indexes_sql(materialized_view) -}} + {{ postgres__create_indexes_template(materialized_view) -}} {%- endmacro -%} -{%- macro postgres__describe_materialized_view_sql(materialized_view) -%} +{%- macro postgres__describe_materialized_view_template(materialized_view) -%} - {%- set _materialized_view_sql -%} + {%- set _materialized_view_template -%} select - v.matviewname, - v.schemaname, - '{{ this.database }}' as databasename, - v.definition + v.matviewname as name, + v.schemaname as schema_name, + '{{ this.database }}' as database_name, + v.definition as query from pg_matviews v where v.matviewname ilike '{{ materialized_view.name }}' and v.schemaname ilike '{{ materialized_view.schema_name }}' {%- endset -%} - {%- set _materialized_view = run_query(_materialized_view_sql) -%} + {%- set _materialized_view = run_query(_materialized_view_template) -%} - {%- set _indexes_sql = postgres__describe_indexes_sql(materialized_view) -%} - {%- set _indexes = run_query(_indexes_sql) -%} + {%- set _indexes_template = postgres__describe_indexes_template(materialized_view) -%} + {%- set _indexes = run_query(_indexes_template) -%} {%- do return({'materialized_view': _materialized_view, 'indexes': _indexes}) -%} {%- endmacro -%} -{%- macro postgres__drop_materialized_view_sql(materialized_view) -%} +{%- macro postgres__drop_materialized_view_template(materialized_view) -%} drop materialized view if exists {{ materialized_view.fully_qualified_path }} cascade {%- endmacro -%} --- This is the old one that gets used with BaseRelation instances +{# /* + These are `BaseRelation` versions. The `BaseRelation` workflows are different. +*/ #} {%- macro postgres__drop_materialized_view(relation) -%} drop materialized view if exists {{ relation }} cascade {%- endmacro -%} -{%- macro postgres__refresh_materialized_view_sql(materialized_view) -%} +{%- macro postgres__refresh_materialized_view_template(materialized_view) -%} refresh materialized view {{ materialized_view.fully_qualified_path }} {%- endmacro -%} -{%- macro postgres__rename_materialized_view_sql(materialized_view, new_name) -%} +{%- macro postgres__rename_materialized_view_template(materialized_view, new_name) -%} - {%- if adapter.is_materialization_config(materialized_view) -%} + {%- if adapter.is_relation_model(materialized_view) -%} {%- set fully_qualified_path = materialized_view.fully_qualified_path -%} {%- else -%} {%- set fully_qualified_path = materialized_view -%} diff --git a/plugins/postgres/dbt/include/postgres/macros/relations/table.sql b/plugins/postgres/dbt/include/postgres/macros/relations/table.sql new file mode 100644 index 00000000000..11c992e68d1 --- /dev/null +++ b/plugins/postgres/dbt/include/postgres/macros/relations/table.sql @@ -0,0 +1,8 @@ +{%- macro postgres__drop_table_template(table) -%} + drop table if exists {{ table.fully_qualified_path }} cascade +{%- endmacro -%} + + +{%- macro postgres__rename_table_template(table, new_name) -%} + alter table {{ table.fully_qualified_path }} rename to {{ new_name }} +{%- endmacro -%} diff --git a/plugins/postgres/dbt/include/postgres/macros/relations/view.sql b/plugins/postgres/dbt/include/postgres/macros/relations/view.sql new file mode 100644 index 00000000000..96b9303f445 --- /dev/null +++ b/plugins/postgres/dbt/include/postgres/macros/relations/view.sql @@ -0,0 +1,8 @@ +{%- macro postgres__drop_view_template(view) -%} + drop view if exists {{ view.fully_qualified_path }} cascade +{%- endmacro -%} + + +{%- macro postgres__rename_view_template(view, new_name) -%} + alter view {{ view.fully_qualified_path }} rename to {{ new_name }} +{%- endmacro -%} diff --git a/plugins/postgres/dbt/include/postgres/macros/timestamps.sql b/plugins/postgres/dbt/include/postgres/macros/utils/timestamps.sql similarity index 100% rename from plugins/postgres/dbt/include/postgres/macros/timestamps.sql rename to plugins/postgres/dbt/include/postgres/macros/utils/timestamps.sql diff --git a/tests/functional/materializations/materialized_view_tests/conftest.py b/tests/functional/materializations/materialized_view_tests/conftest.py new file mode 100644 index 00000000000..fc27b781a9d --- /dev/null +++ b/tests/functional/materializations/materialized_view_tests/conftest.py @@ -0,0 +1,58 @@ +import pytest + +from dbt.adapters.relation.models import RelationStub +from dbt.adapters.relation.factory import RelationFactory +from dbt.contracts.relation import RelationType + +from dbt.adapters.postgres.relation import models as relation_models + + +@pytest.fixture(scope="class") +def relation_factory(): + return RelationFactory( + relation_models={ + RelationType.MaterializedView: relation_models.PostgresMaterializedViewRelation, + }, + relation_can_be_renamed={RelationType.MaterializedView}, + render_policy=relation_models.PostgresRenderPolicy, + ) + + +@pytest.fixture(scope="class") +def my_materialized_view(project, relation_factory) -> RelationStub: + return relation_factory.make_stub( + name="my_materialized_view", + schema_name=project.test_schema, + database_name=project.database, + relation_type=RelationType.MaterializedView, + ) + + +@pytest.fixture(scope="class") +def my_view(project, relation_factory) -> RelationStub: + return relation_factory.make_stub( + name="my_view", + schema_name=project.test_schema, + database_name=project.database, + relation_type=RelationType.View, + ) + + +@pytest.fixture(scope="class") +def my_table(project, relation_factory) -> RelationStub: + return relation_factory.make_stub( + name="my_table", + schema_name=project.test_schema, + database_name=project.database, + relation_type=RelationType.Table, + ) + + +@pytest.fixture(scope="class") +def my_seed(project, relation_factory) -> RelationStub: + return relation_factory.make_stub( + name="my_seed", + schema_name=project.test_schema, + database_name=project.database, + relation_type=RelationType.Table, + ) diff --git a/tests/functional/materializations/materialized_view_tests/files.py b/tests/functional/materializations/materialized_view_tests/files.py new file mode 100644 index 00000000000..9bf881ef970 --- /dev/null +++ b/tests/functional/materializations/materialized_view_tests/files.py @@ -0,0 +1,31 @@ +MY_SEED = """ +id,value +1,100 +2,200 +3,300 +""".strip() + + +MY_TABLE = """ +{{ config( + materialized='table', +) }} +select * from {{ ref('my_seed') }} +""" + + +MY_VIEW = """ +{{ config( + materialized='view', +) }} +select * from {{ ref('my_seed') }} +""" + + +MY_MATERIALIZED_VIEW = """ +{{ config( + materialized='materialized_view', + indexes=[{'columns': ['id']}], +) }} +select * from {{ ref('my_seed') }} +""" diff --git a/tests/functional/materializations/materialized_view_tests/fixtures.py b/tests/functional/materializations/materialized_view_tests/fixtures.py deleted file mode 100644 index 0250152376f..00000000000 --- a/tests/functional/materializations/materialized_view_tests/fixtures.py +++ /dev/null @@ -1,67 +0,0 @@ -import pytest - -from dbt.tests.util import relation_from_name -from tests.adapter.dbt.tests.adapter.materialized_view.base import Base -from tests.adapter.dbt.tests.adapter.materialized_view.on_configuration_change import ( - OnConfigurationChangeBase, - get_model_file, - set_model_file, -) - - -class PostgresBasicBase(Base): - @pytest.fixture(scope="class") - def models(self): - base_table = """ - {{ config(materialized='table') }} - select 1 as base_column - """ - base_materialized_view = """ - {{ config(materialized='materialized_view') }} - select * from {{ ref('base_table') }} - """ - return {"base_table.sql": base_table, "base_materialized_view.sql": base_materialized_view} - - -class PostgresOnConfigurationChangeBase(OnConfigurationChangeBase): - @pytest.fixture(scope="class") - def models(self): - base_table = """ - {{ config( - materialized='table', - indexes=[{'columns': ['id', 'value']}] - ) }} - select - 1 as id, - 100 as value, - 42 as new_id, - 4242 as new_value - """ - base_materialized_view = """ - {{ config( - materialized='materialized_view', - indexes=[{'columns': ['id', 'value']}] - ) }} - select * from {{ ref('base_table') }} - """ - return {"base_table.sql": base_table, "base_materialized_view.sql": base_materialized_view} - - @pytest.fixture(scope="function") - def configuration_changes(self, project): - initial_model = get_model_file(project, "base_materialized_view") - - # change the index from [`id`, `value`] to [`new_id`, `new_value`] - new_model = initial_model.replace( - "indexes=[{'columns': ['id', 'value']}]", - "indexes=[{'columns': ['new_id', 'new_value']}]", - ) - set_model_file(project, "base_materialized_view", new_model) - - yield - - # set this back for the next test - set_model_file(project, "base_materialized_view", initial_model) - - @pytest.fixture(scope="function") - def update_index_message(self, project): - return f"Applying UPDATE INDEXES to: {relation_from_name(project.adapter, 'base_materialized_view')}" diff --git a/tests/functional/materializations/materialized_view_tests/test_materialized_view.py b/tests/functional/materializations/materialized_view_tests/test_materialized_view.py index b332c0fc57b..c5040aea92a 100644 --- a/tests/functional/materializations/materialized_view_tests/test_materialized_view.py +++ b/tests/functional/materializations/materialized_view_tests/test_materialized_view.py @@ -1,188 +1,254 @@ import pytest + from dbt.contracts.graph.model_config import OnConfigurationChangeOption -from dbt.contracts.results import RunStatus -from dbt.contracts.relation import RelationType -from tests.adapter.dbt.tests.adapter.materialized_view.base import ( - run_model, - assert_model_exists_and_is_correct_type, - insert_record, - get_row_count, +from dbt.tests.util import run_dbt, run_dbt_and_capture + +from tests.functional.materializations.materialized_view_tests.files import ( + MY_SEED, + MY_TABLE, + MY_MATERIALIZED_VIEW, + MY_VIEW, ) -from tests.adapter.dbt.tests.adapter.materialized_view.on_configuration_change import ( - assert_proper_scenario, +from tests.functional.materializations.materialized_view_tests.utils import ( + get_model_file, + query_indexes, + query_relation_type, + query_row_count, + set_model_file, ) -from tests.functional.materializations.materialized_view_tests.fixtures import ( - PostgresOnConfigurationChangeBase, - PostgresBasicBase, -) +@pytest.fixture(scope="class", autouse=True) +def seeds(): + return {"my_seed.csv": MY_SEED} -class TestBasic(PostgresBasicBase): - def test_relation_is_materialized_view_on_initial_creation(self, project): - assert_model_exists_and_is_correct_type( - project, "base_materialized_view", RelationType.MaterializedView - ) - assert_model_exists_and_is_correct_type(project, "base_table", RelationType.Table) - def test_relation_is_materialized_view_when_rerun(self, project): - run_model("base_materialized_view") - assert_model_exists_and_is_correct_type( - project, "base_materialized_view", RelationType.MaterializedView - ) +@pytest.fixture(scope="class", autouse=True) +def models(): + yield { + "my_table.sql": MY_TABLE, + "my_view.sql": MY_VIEW, + "my_materialized_view.sql": MY_MATERIALIZED_VIEW, + } - def test_relation_is_materialized_view_on_full_refresh(self, project): - run_model("base_materialized_view", full_refresh=True) - assert_model_exists_and_is_correct_type( - project, "base_materialized_view", RelationType.MaterializedView - ) - def test_relation_is_materialized_view_on_update(self, project): - run_model("base_materialized_view", run_args=["--vars", "quoting: {identifier: True}"]) - assert_model_exists_and_is_correct_type( - project, "base_materialized_view", RelationType.MaterializedView - ) +@pytest.fixture(scope="class", autouse=True) +def setup(project): + run_dbt(["seed"]) + yield + run_dbt(["seed"]) - def test_updated_base_table_data_only_shows_in_materialized_view_after_rerun(self, project): - # poll database - table_start = get_row_count(project, "base_table") - view_start = get_row_count(project, "base_materialized_view") - # insert new record in table - new_record = (2,) - insert_record(project, new_record, "base_table", ["base_column"]) +def create_materialized_view_and_assert(project, my_materialized_view): + _, logs = run_dbt_and_capture(["--debug", "run", "--models", my_materialized_view.name]) + assert query_relation_type(project, my_materialized_view) == "materialized_view" + assert f"Applying CREATE to: {my_materialized_view.fully_qualified_path}" in logs - # poll database - table_mid = get_row_count(project, "base_table") - view_mid = get_row_count(project, "base_materialized_view") - # refresh the materialized view - run_model("base_materialized_view") +def swap_indexes(project, my_materialized_view): + initial_model = get_model_file(project, my_materialized_view) + new_model = initial_model.replace( + "indexes=[{'columns': ['id']}]", + "indexes=[{'columns': ['value']}]", + ) + set_model_file(project, my_materialized_view, new_model) - # poll database - table_end = get_row_count(project, "base_table") - view_end = get_row_count(project, "base_materialized_view") - # new records were inserted in the table but didn't show up in the view until it was refreshed - assert table_start < table_mid == table_end - assert view_start == view_mid < view_end +def test_materialized_view_create(project, my_materialized_view): + assert query_relation_type(project, my_materialized_view) is None + create_materialized_view_and_assert(project, my_materialized_view) -class TestOnConfigurationChangeApply(PostgresOnConfigurationChangeBase): - # we don't need to specify OnConfigurationChangeOption.Apply because it's the default - # this is part of the test +def test_materialized_view_create_idempotent(project, my_materialized_view): + create_materialized_view_and_assert(project, my_materialized_view) - def test_full_refresh_takes_precedence_over_any_configuration_changes( - self, configuration_changes, replace_message - ): - results, logs = run_model("base_materialized_view", full_refresh=True) - assert_proper_scenario( - OnConfigurationChangeOption.Apply, - results, - logs, - RunStatus.Success, - messages_in_logs=[replace_message], - ) + _, logs = run_dbt_and_capture(["--debug", "run", "--models", my_materialized_view.name]) + assert query_relation_type(project, my_materialized_view) == "materialized_view" + assert f"Applying ALTER to: {my_materialized_view.fully_qualified_path}" in logs + assert f"No changes were identified for: {my_materialized_view.fully_qualified_path}" in logs + + +def test_materialized_view_replaces_table(project, my_materialized_view, my_table): + run_dbt(["run", "--models", my_table.name]) + project.run_sql( + f"alter table {my_table.fully_qualified_path} rename to {my_materialized_view.name}" + ) + run_dbt(["run", "--models", my_materialized_view.name]) + assert query_relation_type(project, my_materialized_view) == "materialized_view" + + +def test_materialized_view_replaces_view(project, my_materialized_view, my_view): + run_dbt(["run", "--models", my_view.name]) + project.run_sql( + f"alter view {my_view.fully_qualified_path} rename to {my_materialized_view.name}" + ) + run_dbt(["run", "--models", my_materialized_view.name]) + assert query_relation_type(project, my_materialized_view) == "materialized_view" + + +def test_view_replaces_materialized_table(project, my_materialized_view, my_table): + run_dbt(["run", "--models", my_materialized_view.name]) + project.run_sql( + f"alter materialized view {my_materialized_view.fully_qualified_path} rename to {my_table.name}" + ) + run_dbt(["run", "--models", my_table.name]) + assert query_relation_type(project, my_table) == "table" + + +def test_view_replaces_materialized_view(project, my_materialized_view, my_view): + run_dbt(["run", "--models", my_materialized_view.name]) + project.run_sql( + f"alter materialized view {my_materialized_view.fully_qualified_path} rename to {my_view.name}" + ) + run_dbt(["run", "--models", my_view.name]) + assert query_relation_type(project, my_view) == "view" + + +def test_materialized_view_only_updates_after_refresh(project, my_materialized_view, my_seed): + create_materialized_view_and_assert(project, my_materialized_view) + + # poll database + table_start = query_row_count(project, my_seed) + view_start = query_row_count(project, my_materialized_view) + + # insert new record in table + project.run_sql(f"insert into {my_seed.fully_qualified_path} (id, value) values (4, 400);") + + # poll database + table_mid = query_row_count(project, my_seed) + view_mid = query_row_count(project, my_materialized_view) + + # refresh the materialized view + project.run_sql(f"refresh materialized view {my_materialized_view.fully_qualified_path};") + + # poll database + table_end = query_row_count(project, my_seed) + view_end = query_row_count(project, my_materialized_view) + + # new records were inserted in the table but didn't show up in the view until it was refreshed + assert table_start < table_mid == table_end + assert view_start == view_mid < view_end + + +def test_materialized_view_full_refresh(project, my_materialized_view): + create_materialized_view_and_assert(project, my_materialized_view) + + _, logs = run_dbt_and_capture( + ["--debug", "run", "--models", my_materialized_view.name, "--full-refresh"] + ) + assert query_relation_type(project, my_materialized_view) == "materialized_view" + assert f"Applying REPLACE to: {my_materialized_view.fully_qualified_path}" in logs + + +def test_indexes_are_updated_with_apply(project, my_materialized_view): + create_materialized_view_and_assert(project, my_materialized_view) + indexes = query_indexes(project, my_materialized_view) + assert len(indexes) == 1 + assert indexes[0]["column_names"] == "id" - def test_model_is_refreshed_with_no_configuration_changes(self, refresh_message): - results, logs = run_model("base_materialized_view") - assert_proper_scenario( - OnConfigurationChangeOption.Apply, - results, - logs, - RunStatus.Success, - messages_in_logs=[refresh_message], + swap_indexes(project, my_materialized_view) + run_dbt(["run", "--models", my_materialized_view.name]) + + indexes = query_indexes(project, my_materialized_view) + assert len(indexes) == 1 + assert indexes[0]["column_names"] == "value" + + +class OnConfigurationChangeBase: + @pytest.fixture(scope="class") + def project_config_update(self): + return {"models": {"on_configuration_change": OnConfigurationChangeOption.Continue.value}} + + @pytest.fixture(scope="class", autouse=True) + def models(self): + yield { + "my_table.sql": MY_TABLE, + "my_view.sql": MY_VIEW, + "my_materialized_view.sql": MY_MATERIALIZED_VIEW, + } + + @pytest.fixture(scope="function", autouse=True) + def setup(self, project, my_materialized_view): + run_dbt(["seed"]) + run_dbt(["run", "--models", my_materialized_view.name, "--full-refresh"]) + + initial_model = get_model_file(project, my_materialized_view) + new_model = initial_model.replace( + "indexes=[{'columns': ['id']}]", + "indexes=[{'columns': ['value']}]", ) + set_model_file(project, my_materialized_view, new_model) + + yield - def test_model_applies_changes_with_configuration_changes( - self, configuration_changes, alter_message, update_index_message + set_model_file(project, my_materialized_view, initial_model) + + run_dbt(["seed"]) + run_dbt(["run", "--models", my_materialized_view.name, "--full-refresh"]) + + +class TestOnConfigurationChangeApply(OnConfigurationChangeBase): + @pytest.fixture(scope="class") + def project_config_update(self): + return {"models": {"on_configuration_change": OnConfigurationChangeOption.Apply.value}} + + def test_materialized_view_full_refresh_with_changes_apply( + self, project, my_materialized_view ): - results, logs = run_model("base_materialized_view") - assert_proper_scenario( - OnConfigurationChangeOption.Apply, - results, - logs, - RunStatus.Success, - messages_in_logs=[alter_message, update_index_message], - ) + run_dbt(["run", "--models", my_materialized_view.name, "--full-refresh"]) + assert query_relation_type(project, my_materialized_view) == "materialized_view" + def test_materialized_view_with_changes_apply(self, project, my_materialized_view): + indexes = query_indexes(project, my_materialized_view) + assert len(indexes) == 1 + assert indexes[0]["column_names"] == "id" -class TestOnConfigurationChangeContinue(PostgresOnConfigurationChangeBase): + run_dbt(["run", "--models", my_materialized_view.name]) + + indexes = query_indexes(project, my_materialized_view) + assert len(indexes) == 1 + assert indexes[0]["column_names"] == "value" + + +class TestOnConfigurationChangeContinue(OnConfigurationChangeBase): @pytest.fixture(scope="class") def project_config_update(self): return {"models": {"on_configuration_change": OnConfigurationChangeOption.Continue.value}} - def test_full_refresh_takes_precedence_over_any_configuration_changes( - self, configuration_changes, replace_message + def test_materialized_view_full_refresh_with_changes_continue( + self, project, my_materialized_view ): - results, logs = run_model("base_materialized_view", full_refresh=True) - assert_proper_scenario( - OnConfigurationChangeOption.Continue, - results, - logs, - RunStatus.Success, - messages_in_logs=[replace_message], - ) + run_dbt(["run", "--models", my_materialized_view.name, "--full-refresh"]) + assert query_relation_type(project, my_materialized_view) == "materialized_view" - def test_model_is_refreshed_with_no_configuration_changes(self, refresh_message): - results, logs = run_model("base_materialized_view") - assert_proper_scenario( - OnConfigurationChangeOption.Continue, - results, - logs, - RunStatus.Success, - messages_in_logs=[refresh_message], - ) + def test_materialized_view_with_changes_continue(self, project, my_materialized_view): + indexes = query_indexes(project, my_materialized_view) + assert len(indexes) == 1 + assert indexes[0]["column_names"] == "id" - def test_model_is_not_refreshed_with_configuration_changes( - self, configuration_changes, configuration_change_continue_message, refresh_message - ): - results, logs = run_model("base_materialized_view") - assert_proper_scenario( - OnConfigurationChangeOption.Continue, - results, - logs, - RunStatus.Success, - messages_in_logs=[configuration_change_continue_message], - messages_not_in_logs=[refresh_message], - ) + run_dbt(["run", "--models", my_materialized_view.name]) + indexes = query_indexes(project, my_materialized_view) + assert len(indexes) == 1 + assert indexes[0]["column_names"] == "id" -class TestOnConfigurationChangeFail(PostgresOnConfigurationChangeBase): + +class TestOnConfigurationChangeFail(OnConfigurationChangeBase): @pytest.fixture(scope="class") def project_config_update(self): return {"models": {"on_configuration_change": OnConfigurationChangeOption.Fail.value}} - def test_full_refresh_takes_precedence_over_any_configuration_changes( - self, configuration_changes, replace_message - ): - results, logs = run_model("base_materialized_view", full_refresh=True) - assert_proper_scenario( - OnConfigurationChangeOption.Fail, - results, - logs, - RunStatus.Success, - messages_in_logs=[replace_message], - ) + def test_materialized_view_full_refresh_with_changes_fail(self, project, my_materialized_view): + run_dbt(["run", "--models", my_materialized_view.name, "--full-refresh"]) + assert query_relation_type(project, my_materialized_view) == "materialized_view" - def test_model_is_refreshed_with_no_configuration_changes(self, refresh_message): - results, logs = run_model("base_materialized_view") - assert_proper_scenario( - OnConfigurationChangeOption.Fail, - results, - logs, - RunStatus.Success, - messages_in_logs=[refresh_message], - ) + def test_materialized_view_with_changes_fail(self, project, my_materialized_view): + indexes = query_indexes(project, my_materialized_view) + assert len(indexes) == 1 + assert indexes[0]["column_names"] == "id" - def test_run_fails_with_configuration_changes( - self, configuration_changes, configuration_change_fail_message - ): - results, logs = run_model("base_materialized_view", expect_pass=False) - assert_proper_scenario( - OnConfigurationChangeOption.Fail, - results, - logs, - RunStatus.Error, - messages_in_logs=[configuration_change_fail_message], - ) + run_dbt(["run", "--models", my_materialized_view.name], expect_pass=False) + + indexes = query_indexes(project, my_materialized_view) + assert len(indexes) == 1 + assert indexes[0]["column_names"] == "id" diff --git a/tests/functional/materializations/materialized_view_tests/utils.py b/tests/functional/materializations/materialized_view_tests/utils.py new file mode 100644 index 00000000000..bc10505188a --- /dev/null +++ b/tests/functional/materializations/materialized_view_tests/utils.py @@ -0,0 +1,80 @@ +from typing import Dict, List, Optional + +from dbt.adapters.relation.models import Relation, RelationStub +from dbt.tests.util import read_file, write_file + + +def query_relation_type(project, relation: RelationStub) -> Optional[str]: + sql = f""" + select 'table' as relation_type + from pg_tables + where schemaname = '{relation.schema_name}' + and tablename = '{relation.name}' + union all + select 'view' as relation_type + from pg_views + where schemaname = '{relation.schema_name}' + and viewname = '{relation.name}' + union all + select 'materialized_view' as relation_type + from pg_matviews + where schemaname = '{relation.schema_name}' + and matviewname = '{relation.name}' + """ + results = project.run_sql(sql, fetch="all") + if len(results) == 0: + return None + elif len(results) > 1: + raise ValueError(f"More than one instance of {relation.name} found!") + else: + return results[0][0] + + +def query_row_count(project, relation: RelationStub) -> int: + sql = f"select count(*) from {relation.fully_qualified_path};" + return project.run_sql(sql, fetch="one")[0] + + +def query_indexes(project, relation: RelationStub) -> List[Dict[str, str]]: + # pulled directly from `postgres__describe_indexes_template` and manually verified + sql = f""" + select + i.relname as name, + m.amname as method, + ix.indisunique as "unique", + array_to_string(array_agg(a.attname), ',') as column_names + from pg_index ix + join pg_class i + on i.oid = ix.indexrelid + join pg_am m + on m.oid=i.relam + join pg_class t + on t.oid = ix.indrelid + join pg_namespace n + on n.oid = t.relnamespace + join pg_attribute a + on a.attrelid = t.oid + and a.attnum = ANY(ix.indkey) + where t.relname ilike '{ relation.name }' + and n.nspname ilike '{ relation.schema_name }' + and t.relkind in ('r', 'm') + group by 1, 2, 3 + order by 1, 2, 3 + """ + raw_indexes = project.run_sql(sql, fetch="all") + indexes = [ + { + header: value + for header, value in zip(["name", "method", "unique", "column_names"], index) + } + for index in raw_indexes + ] + return indexes + + +def get_model_file(project, relation: Relation) -> str: + return read_file(project.project_root, "models", f"{relation.name}.sql") + + +def set_model_file(project, relation: Relation, model_sql: str): + write_file(model_sql, project.project_root, "models", f"{relation.name}.sql") diff --git a/tests/unit/dbt_postgres_tests/conftest.py b/tests/unit/dbt_postgres_tests/conftest.py new file mode 100644 index 00000000000..0b80878fb83 --- /dev/null +++ b/tests/unit/dbt_postgres_tests/conftest.py @@ -0,0 +1,198 @@ +from dataclasses import dataclass + +import agate +import pytest + +from dbt.adapters.materialization.factory import MaterializationFactory +from dbt.adapters.materialization.models import ( + MaterializationType, + MaterializedViewMaterialization, +) +from dbt.adapters.relation.factory import RelationFactory +from dbt.contracts.files import FileHash +from dbt.contracts.graph.model_config import OnConfigurationChangeOption +from dbt.contracts.graph.nodes import DependsOn, ModelNode, NodeConfig +from dbt.contracts.relation import RelationType +from dbt.node_types import NodeType + +from dbt.adapters.postgres.relation import models + + +@pytest.fixture +def relation_factory(): + return RelationFactory( + relation_models={ + RelationType.MaterializedView: models.PostgresMaterializedViewRelation, + }, + relation_changesets={ + RelationType.MaterializedView: models.PostgresMaterializedViewRelationChangeset, + }, + relation_can_be_renamed={RelationType.MaterializedView}, + render_policy=models.PostgresRenderPolicy, + ) + + +@pytest.fixture +def materialization_factory(relation_factory): + return MaterializationFactory( + relation_factory=relation_factory, + materialization_map={ + MaterializationType.MaterializedView: MaterializedViewMaterialization + }, + ) + + +@pytest.fixture +def materialized_view_stub(relation_factory): + return relation_factory.make_stub( + name="my_materialized_view", + schema_name="my_schema", + database_name="my_database", + relation_type=RelationType.MaterializedView, + ) + + +@pytest.fixture +def view_stub(relation_factory): + return relation_factory.make_stub( + name="my_view", + schema_name="my_schema", + database_name="my_database", + relation_type=RelationType.View, + ) + + +@pytest.fixture +def materialized_view_describe_relation_results(): + materialized_view_agate = agate.Table.from_object( + [ + { + "name": "my_materialized_view", + "schema_name": "my_schema", + "database_name": "my_database", + "query": "select 42 from meaning_of_life", + } + ] + ) + indexes_agate = agate.Table.from_object( + [ + {"name": "index_1", "column_names": "id,value", "method": "hash", "unique": None}, + {"name": "index_2", "column_names": "id", "method": None, "unique": True}, + ] + ) + return {"materialized_view": materialized_view_agate, "indexes": indexes_agate} + + +@pytest.fixture +def materialized_view_model_node(): + return ModelNode( + alias="my_materialized_view", + name="my_materialized_view", + database="my_database", + schema="my_schema", + resource_type=NodeType.Model, + unique_id="model.root.my_materialized_view", + fqn=["root", "my_materialized_view"], + package_name="root", + original_file_path="my_materialized_view.sql", + refs=[], + sources=[], + depends_on=DependsOn(), + config=NodeConfig.from_dict( + { + "enabled": True, + "materialized": "materialized_view", + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "vars": {}, + "quoting": {}, + "column_types": {}, + "tags": [], + "indexes": [ + {"columns": ["id", "value"], "type": "hash"}, + {"columns": ["id"], "unique": True}, + ], + } + ), + tags=[], + path="my_materialized_view.sql", + language="sql", + raw_code="select 42 from meaning_of_life", + compiled_code="select 42 from meaning_of_life", + description="", + columns={}, + checksum=FileHash.from_contents(""), + ) + + +@pytest.fixture +def materialized_view_relation(relation_factory, materialized_view_describe_relation_results): + return relation_factory.make_from_describe_relation_results( + materialized_view_describe_relation_results, RelationType.MaterializedView + ) + + +@pytest.fixture +def materialized_view_runtime_config(materialized_view_model_node): + """ + This is not actually a `RuntimeConfigObject`. It's an object that has attribution that looks like + a boiled down version of a RuntimeConfigObject. + + TODO: replace this with an actual `RuntimeConfigObject` + """ + + @dataclass() + class RuntimeConfigObject: + model: ModelNode + full_refresh: bool + grants: dict + on_configuration_change: OnConfigurationChangeOption + + def get(self, attribute: str, default=None): + return getattr(self, attribute, default) + + return RuntimeConfigObject( + model=materialized_view_model_node, + full_refresh=False, + grants={}, + on_configuration_change=OnConfigurationChangeOption.Continue, + ) + + +""" +Make sure the fixtures at least work, more thorough testing is done elsewhere +""" + + +def test_relation_factory(relation_factory): + assert ( + relation_factory._get_parser(RelationType.MaterializedView) + == models.PostgresMaterializedViewRelation + ) + + +def test_materialization_factory(materialization_factory): + postgres_parser = materialization_factory.relation_factory._get_parser( + RelationType.MaterializedView + ) + assert postgres_parser == models.PostgresMaterializedViewRelation + + +def test_materialized_view_stub(materialized_view_stub): + assert materialized_view_stub.name == "my_materialized_view" + + +def test_materialized_view_model_node(materialized_view_model_node): + assert materialized_view_model_node.name == "my_materialized_view" + + +def test_materialized_view_runtime_config(materialized_view_runtime_config): + assert materialized_view_runtime_config.get("full_refresh", False) is False + assert materialized_view_runtime_config.get("on_configuration_change", "apply") == "continue" + assert materialized_view_runtime_config.model.name == "my_materialized_view" + + +def test_materialized_view_relation(materialized_view_relation): + assert materialized_view_relation.type == RelationType.MaterializedView + assert materialized_view_relation.name == "my_materialized_view" diff --git a/tests/unit/dbt_postgres_tests/materialization_tests/test_materialization.py b/tests/unit/dbt_postgres_tests/materialization_tests/test_materialization.py new file mode 100644 index 00000000000..53a9491ae80 --- /dev/null +++ b/tests/unit/dbt_postgres_tests/materialization_tests/test_materialization.py @@ -0,0 +1,35 @@ +from dataclasses import replace + +from dbt.adapters.materialization.models import ( + MaterializedViewMaterialization, + MaterializationBuildStrategy, +) + + +def test_materialized_view_create(materialized_view_runtime_config, relation_factory): + materialization = MaterializedViewMaterialization.from_runtime_config( + materialized_view_runtime_config, relation_factory + ) + assert materialization.build_strategy == MaterializationBuildStrategy.Create + assert materialization.should_revoke_grants is False + + +def test_materialized_view_replace(materialized_view_runtime_config, relation_factory, view_stub): + + materialization = MaterializedViewMaterialization.from_runtime_config( + materialized_view_runtime_config, relation_factory, view_stub + ) + assert materialization.build_strategy == MaterializationBuildStrategy.Replace + assert materialization.should_revoke_grants is True + + +def test_materialized_view_alter( + materialized_view_runtime_config, relation_factory, materialized_view_relation +): + altered_materialized_view = replace(materialized_view_relation, indexes={}) + + materialization = MaterializedViewMaterialization.from_runtime_config( + materialized_view_runtime_config, relation_factory, altered_materialized_view + ) + assert materialization.build_strategy == MaterializationBuildStrategy.Alter + assert materialization.should_revoke_grants is True diff --git a/tests/unit/dbt_postgres_tests/materialization_tests/test_materialization_factory.py b/tests/unit/dbt_postgres_tests/materialization_tests/test_materialization_factory.py new file mode 100644 index 00000000000..4cc28896a72 --- /dev/null +++ b/tests/unit/dbt_postgres_tests/materialization_tests/test_materialization_factory.py @@ -0,0 +1,36 @@ +from dbt.adapters.materialization.models import MaterializationType +from dbt.contracts.relation import RelationType + +from dbt.adapters.postgres.relation import models as relation_models + + +def test_make_from_runtime_config(materialization_factory, materialized_view_runtime_config): + materialization = materialization_factory.make_from_runtime_config( + runtime_config=materialized_view_runtime_config, + materialization_type=MaterializationType.MaterializedView, + existing_relation_stub=None, + ) + assert materialization.type == MaterializationType.MaterializedView + + materialized_view = materialization.target_relation + assert materialized_view.type == RelationType.MaterializedView + + assert materialized_view.name == "my_materialized_view" + assert materialized_view.schema_name == "my_schema" + assert materialized_view.database_name == "my_database" + assert materialized_view.query == "select 42 from meaning_of_life" + + index_1 = relation_models.PostgresIndexRelation( + column_names=frozenset({"id", "value"}), + method=relation_models.PostgresIndexMethod.hash, + unique=False, + render=relation_models.PostgresRenderPolicy, + ) + index_2 = relation_models.PostgresIndexRelation( + column_names=frozenset({"id"}), + method=relation_models.PostgresIndexMethod.btree, + unique=True, + render=relation_models.PostgresRenderPolicy, + ) + assert index_1 in materialized_view.indexes + assert index_2 in materialized_view.indexes diff --git a/tests/unit/dbt_postgres_tests/relation_tests/model_tests/test_database.py b/tests/unit/dbt_postgres_tests/relation_tests/model_tests/test_database.py new file mode 100644 index 00000000000..ce4792b2f31 --- /dev/null +++ b/tests/unit/dbt_postgres_tests/relation_tests/model_tests/test_database.py @@ -0,0 +1,24 @@ +from typing import Type + +import pytest +from dbt.exceptions import DbtRuntimeError + +from dbt.adapters.postgres.relation.models import PostgresDatabaseRelation + + +@pytest.mark.parametrize( + "config_dict,exception", + [ + ({"name": "my_database"}, None), + ({"name": ""}, DbtRuntimeError), + ({"wrong_name": "my_database"}, DbtRuntimeError), + ({}, DbtRuntimeError), + ], +) +def test_make_database(config_dict: dict, exception: Type[Exception]): + if exception: + with pytest.raises(exception): + PostgresDatabaseRelation.from_dict(config_dict) + else: + my_database = PostgresDatabaseRelation.from_dict(config_dict) + assert my_database.name == config_dict.get("name") diff --git a/tests/unit/dbt_postgres_tests/relation_tests/model_tests/test_index.py b/tests/unit/dbt_postgres_tests/relation_tests/model_tests/test_index.py new file mode 100644 index 00000000000..8885f3301a1 --- /dev/null +++ b/tests/unit/dbt_postgres_tests/relation_tests/model_tests/test_index.py @@ -0,0 +1,24 @@ +from typing import Type + +import pytest +from dbt.exceptions import DbtRuntimeError + +from dbt.adapters.postgres.relation.models import PostgresIndexRelation + + +@pytest.mark.parametrize( + "config_dict,exception", + [ + ({"column_names": frozenset({"id", "value"}), "method": "hash", "unique": False}, None), + ({"column_names": frozenset("id"), "method": "btree", "unique": True}, None), + ({}, DbtRuntimeError), + ({"method": "btree", "unique": True}, DbtRuntimeError), + ], +) +def test_create_index(config_dict: dict, exception: Type[Exception]): + if exception: + with pytest.raises(exception): + PostgresIndexRelation.from_dict(config_dict) + else: + my_index = PostgresIndexRelation.from_dict(config_dict) + assert my_index.column_names == config_dict.get("column_names") diff --git a/tests/unit/dbt_postgres_tests/relation_tests/model_tests/test_materialized_view.py b/tests/unit/dbt_postgres_tests/relation_tests/model_tests/test_materialized_view.py new file mode 100644 index 00000000000..4d24889d498 --- /dev/null +++ b/tests/unit/dbt_postgres_tests/relation_tests/model_tests/test_materialized_view.py @@ -0,0 +1,112 @@ +from dataclasses import replace +from typing import Type + +import pytest + +from dbt.adapters.relation.models import RelationChangeAction +from dbt.exceptions import DbtRuntimeError + +from dbt.adapters.postgres.relation.models import ( + PostgresIndexRelation, + PostgresIndexRelationChange, + PostgresMaterializedViewRelation, + PostgresMaterializedViewRelationChangeset, +) + + +@pytest.mark.parametrize( + "config_dict,exception", + [ + ( + { + "name": "my_materialized_view", + "schema": { + "name": "my_schema", + "database": {"name": "my_database"}, + }, + "query": "select 1 from my_favoriate_table", + }, + None, + ), + ( + { + "name": "my_indexed_materialized_view", + "schema": { + "name": "my_schema", + "database": {"name": "my_database"}, + }, + "query": "select 42 from meaning_of_life", + "indexes": [ + { + "column_names": frozenset({"id", "value"}), + "method": "hash", + "unique": False, + }, + {"column_names": frozenset({"id"}), "method": "btree", "unique": True}, + ], + }, + None, + ), + ( + { + "my_name": "my_materialized_view", + "schema": { + "name": "my_schema", + "database": {"name": "my_database"}, + }, + }, + DbtRuntimeError, + ), + ], +) +def test_create_materialized_view(config_dict: dict, exception: Type[Exception]): + if exception: + with pytest.raises(exception): + PostgresMaterializedViewRelation.from_dict(config_dict) + else: + my_materialized_view = PostgresMaterializedViewRelation.from_dict(config_dict) + assert my_materialized_view.name == config_dict.get("name") + assert my_materialized_view.schema_name == config_dict.get("schema").get("name") + assert my_materialized_view.database_name == config_dict.get("schema").get("database").get( + "name" + ) + assert my_materialized_view.query == config_dict.get("query") + if indexes := config_dict.get("indexes"): + parsed = {(index.method, index.unique) for index in my_materialized_view.indexes} + raw = {(index.get("method"), index.get("unique")) for index in indexes} + assert parsed == raw + + +def test_create_materialized_view_changeset(materialized_view_relation): + existing_materialized_view = replace(materialized_view_relation) + + # pulled from `./dbt_postgres_tests/conftest.py` + original_index_1 = PostgresIndexRelation.from_dict( + { + "name": "index_1", + "column_names": frozenset({"id", "value"}), + "method": "hash", + "unique": False, + } + ) + original_index_2 = PostgresIndexRelation.from_dict( + {"name": "index_2", "column_names": frozenset({"id"}), "method": "btree", "unique": True}, + ) + + new_index = PostgresIndexRelation.from_dict( + {"column_names": frozenset({"id", "value"}), "method": "btree", "unique": False} + ) + + target_materialized_view = replace( + existing_materialized_view, indexes=frozenset({new_index, original_index_2}) + ) + + changeset = PostgresMaterializedViewRelationChangeset.from_relations( + existing_materialized_view, target_materialized_view + ) + assert changeset.is_empty is False + assert changeset.requires_full_refresh is False + assert changeset.indexes == { + PostgresIndexRelationChange(action=RelationChangeAction.drop, context=original_index_1), + PostgresIndexRelationChange(action=RelationChangeAction.create, context=new_index), + } diff --git a/tests/unit/dbt_postgres_tests/relation_tests/model_tests/test_schema.py b/tests/unit/dbt_postgres_tests/relation_tests/model_tests/test_schema.py new file mode 100644 index 00000000000..25864f9e3d5 --- /dev/null +++ b/tests/unit/dbt_postgres_tests/relation_tests/model_tests/test_schema.py @@ -0,0 +1,31 @@ +from typing import Type + +import pytest +from dbt.exceptions import DbtRuntimeError + +from dbt.adapters.postgres.relation.models import PostgresSchemaRelation + + +@pytest.mark.parametrize( + "config_dict,exception", + [ + ({"name": "my_schema", "database": {"name": "my_database"}}, None), + ({"name": "my_schema", "database": None}, DbtRuntimeError), + ({"name": "my_schema"}, DbtRuntimeError), + ({"name": "", "database": {"name": "my_database"}}, DbtRuntimeError), + ({"wrong_name": "my_database", "database": {"name": "my_database"}}, DbtRuntimeError), + ( + {"name": "my_schema", "database": {"name": "my_database"}, "meaning_of_life": 42}, + DbtRuntimeError, + ), + ({}, DbtRuntimeError), + ], +) +def test_make_schema(config_dict: dict, exception: Type[Exception]): + if exception: + with pytest.raises(exception): + PostgresSchemaRelation.from_dict(config_dict) + else: + my_schema = PostgresSchemaRelation.from_dict(config_dict) + assert my_schema.name == config_dict.get("name") + assert my_schema.database_name == config_dict.get("database").get("name") diff --git a/tests/unit/dbt_postgres_tests/relation_tests/test_relation_factory.py b/tests/unit/dbt_postgres_tests/relation_tests/test_relation_factory.py new file mode 100644 index 00000000000..d6d18c37159 --- /dev/null +++ b/tests/unit/dbt_postgres_tests/relation_tests/test_relation_factory.py @@ -0,0 +1,79 @@ +""" +Uses the following fixtures in `unit/dbt_postgres_tests/conftest.py`: +- `relation_factory` +- `materialized_view_stub` +""" + +from dbt.contracts.relation import RelationType + +from dbt.adapters.postgres.relation import models + + +def test_make_stub(materialized_view_stub): + assert materialized_view_stub.name == "my_materialized_view" + assert materialized_view_stub.schema_name == "my_schema" + assert materialized_view_stub.database_name == "my_database" + assert materialized_view_stub.type == "materialized_view" + assert materialized_view_stub.can_be_renamed is True + + +def test_make_backup_stub(relation_factory, materialized_view_stub): + backup_stub = relation_factory.make_backup_stub(materialized_view_stub) + assert backup_stub.name == '"my_materialized_view__dbt_backup"' + + +def test_make_intermediate(relation_factory, materialized_view_stub): + intermediate_relation = relation_factory.make_intermediate(materialized_view_stub) + assert intermediate_relation.name == '"my_materialized_view__dbt_tmp"' + + +def test_make_from_describe_relation_results( + relation_factory, materialized_view_describe_relation_results +): + materialized_view = relation_factory.make_from_describe_relation_results( + materialized_view_describe_relation_results, RelationType.MaterializedView + ) + + assert materialized_view.name == "my_materialized_view" + assert materialized_view.schema_name == "my_schema" + assert materialized_view.database_name == "my_database" + assert materialized_view.query == "select 42 from meaning_of_life" + + index_1 = models.PostgresIndexRelation( + column_names=frozenset({"id", "value"}), + method=models.PostgresIndexMethod.hash, + unique=False, + render=models.PostgresRenderPolicy, + ) + index_2 = models.PostgresIndexRelation( + column_names=frozenset({"id"}), + method=models.PostgresIndexMethod.btree, + unique=True, + render=models.PostgresRenderPolicy, + ) + assert index_1 in materialized_view.indexes + assert index_2 in materialized_view.indexes + + +def test_make_from_model_node(relation_factory, materialized_view_model_node): + materialized_view = relation_factory.make_from_model_node(materialized_view_model_node) + + assert materialized_view.name == "my_materialized_view" + assert materialized_view.schema_name == "my_schema" + assert materialized_view.database_name == "my_database" + assert materialized_view.query == "select 42 from meaning_of_life" + + index_1 = models.PostgresIndexRelation( + column_names=frozenset({"id", "value"}), + method=models.PostgresIndexMethod.hash, + unique=False, + render=models.PostgresRenderPolicy, + ) + index_2 = models.PostgresIndexRelation( + column_names=frozenset({"id"}), + method=models.PostgresIndexMethod.btree, + unique=True, + render=models.PostgresRenderPolicy, + ) + assert index_1 in materialized_view.indexes + assert index_2 in materialized_view.indexes From 9400cdccc439c72a9b5a7eb0b6ee82ebabd9e888 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 10 Jul 2023 20:42:17 -0400 Subject: [PATCH 11/28] fixed using actual OrderDict for typehint --- core/dbt/adapters/relation/models/_policy.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/dbt/adapters/relation/models/_policy.py b/core/dbt/adapters/relation/models/_policy.py index 617bece3f4d..d9d6bdf6089 100644 --- a/core/dbt/adapters/relation/models/_policy.py +++ b/core/dbt/adapters/relation/models/_policy.py @@ -1,7 +1,6 @@ from abc import ABC -from collections import OrderedDict from dataclasses import dataclass -from typing import Optional +from typing import Optional, OrderedDict from dbt.contracts.relation import Policy, ComponentName From e7df1222f7dedbe3ab1ac9a5600111544cd0603f Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 10 Jul 2023 21:02:00 -0400 Subject: [PATCH 12/28] mypy --- core/dbt/adapters/materialization/factory.py | 6 ++++-- core/dbt/adapters/relation/factory.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/core/dbt/adapters/materialization/factory.py b/core/dbt/adapters/materialization/factory.py index bae46937172..13623b527cb 100644 --- a/core/dbt/adapters/materialization/factory.py +++ b/core/dbt/adapters/materialization/factory.py @@ -8,8 +8,10 @@ class MaterializationFactory: def __init__( self, - relation_factory: RelationFactory = None, - materialization_map: Dict[models.MaterializationType, Type[models.Materialization]] = None, + relation_factory: RelationFactory, + materialization_map: Optional[ + Dict[models.MaterializationType, Type[models.Materialization]] + ] = None, ): self.relation_factory = relation_factory self.materialization_map = materialization_map or { diff --git a/core/dbt/adapters/relation/factory.py b/core/dbt/adapters/relation/factory.py index d20494648d4..9fa40f2e6b5 100644 --- a/core/dbt/adapters/relation/factory.py +++ b/core/dbt/adapters/relation/factory.py @@ -21,8 +21,8 @@ class RelationFactory: def __init__( self, relation_models: Dict[RelationType, Type[models.Relation]], - relation_changesets: Dict[RelationType, Type[models.RelationChangeset]] = None, - relation_can_be_renamed: Set[RelationType] = None, + relation_changesets: Optional[Dict[RelationType, Type[models.RelationChangeset]]] = None, + relation_can_be_renamed: Optional[Set[RelationType]] = None, render_policy: models.RenderPolicy = models.RenderPolicy(), ): self.relation_models = relation_models From 1cd4e6d60601f41380479d9c3217c09923cbd6a3 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 10 Jul 2023 22:03:02 -0400 Subject: [PATCH 13/28] removed centralized testing logic or moved it to `util` --- core/dbt/tests/util.py | 30 +++++++ .../tests/adapter/materialized_view/base.py | 69 --------------- .../on_configuration_change.py | 84 ------------------- .../test_materialized_view.py | 25 ++++-- .../materialized_view_tests/utils.py | 17 +--- 5 files changed, 51 insertions(+), 174 deletions(-) delete mode 100644 tests/adapter/dbt/tests/adapter/materialized_view/base.py delete mode 100644 tests/adapter/dbt/tests/adapter/materialized_view/on_configuration_change.py diff --git a/core/dbt/tests/util.py b/core/dbt/tests/util.py index 5179ceb2f04..90be8b0b2ce 100644 --- a/core/dbt/tests/util.py +++ b/core/dbt/tests/util.py @@ -8,6 +8,7 @@ from typing import Dict, List, Optional from contextlib import contextmanager from dbt.adapters.factory import Adapter +from dbt.adapters.relation.models import Relation from dbt.cli.main import dbtRunner from dbt.logger import log_manager @@ -588,3 +589,32 @@ def __eq__(self, other): def __repr__(self): return "AnyStringWith<{!r}>".format(self.contains) + + +def assert_message_in_logs(message: str, logs: str, expected_fail: bool = False): + # if the logs are json strings, then 'jsonify' the message because of things like escape quotes + if os.environ.get("DBT_LOG_FORMAT", "") == "json": + message = message.replace(r'"', r"\"") + + if expected_fail: + assert message not in logs + else: + assert message in logs + + +def get_project_config(project): + file_yaml = read_file(project.project_root, "dbt_project.yml") + return yaml.safe_load(file_yaml) + + +def set_project_config(project, config): + config_yaml = yaml.safe_dump(config) + write_file(config_yaml, project.project_root, "dbt_project.yml") + + +def get_model_file(project, relation: Relation) -> str: + return read_file(project.project_root, "models", f"{relation.name}.sql") + + +def set_model_file(project, relation: Relation, model_sql: str): + write_file(model_sql, project.project_root, "models", f"{relation.name}.sql") diff --git a/tests/adapter/dbt/tests/adapter/materialized_view/base.py b/tests/adapter/dbt/tests/adapter/materialized_view/base.py deleted file mode 100644 index 25aef3abc6f..00000000000 --- a/tests/adapter/dbt/tests/adapter/materialized_view/base.py +++ /dev/null @@ -1,69 +0,0 @@ -from typing import List, Tuple, Optional -import os - -import pytest - -from dbt.dataclass_schema import StrEnum -from dbt.tests.util import run_dbt, get_manifest, run_dbt_and_capture - - -def run_model( - model: str, - run_args: Optional[List[str]] = None, - full_refresh: bool = False, - expect_pass: bool = True, -) -> Tuple[list, str]: - args = ["--debug", "run", "--models", model] - if full_refresh: - args.append("--full-refresh") - if run_args: - args.extend(run_args) - return run_dbt_and_capture(args, expect_pass=expect_pass) - - -def assert_message_in_logs(logs: str, message: str, expected_fail: bool = False): - # if the logs are json strings, then 'jsonify' the message because of things like escape quotes - if os.environ.get("DBT_LOG_FORMAT", "") == "json": - message = message.replace(r'"', r"\"") - - if expected_fail: - assert message not in logs - else: - assert message in logs - - -def get_records(project, model: str) -> List[tuple]: - sql = f"select * from {project.database}.{project.test_schema}.{model};" - return [tuple(row) for row in project.run_sql(sql, fetch="all")] - - -def get_row_count(project, model: str) -> int: - sql = f"select count(*) from {project.database}.{project.test_schema}.{model};" - return project.run_sql(sql, fetch="one")[0] - - -def insert_record(project, record: tuple, model: str, columns: List[str]): - sql = f""" - insert into {project.database}.{project.test_schema}.{model} ({', '.join(columns)}) - values ({','.join(str(value) for value in record)}) - ;""" - project.run_sql(sql) - - -def assert_model_exists_and_is_correct_type(project, model: str, relation_type: StrEnum): - # In general, `relation_type` will be of type `RelationType`. - # However, in some cases (e.g. `dbt-snowflake`) adapters will have their own `RelationType`. - manifest = get_manifest(project.project_root) - model_metadata = manifest.nodes[f"model.test.{model}"] - assert model_metadata.config.materialized == relation_type - assert get_row_count(project, model) >= 0 - - -class Base: - @pytest.fixture(scope="function", autouse=True) - def setup(self, project): - run_dbt(["run"]) - - @pytest.fixture(scope="class", autouse=True) - def project(self, project): - yield project diff --git a/tests/adapter/dbt/tests/adapter/materialized_view/on_configuration_change.py b/tests/adapter/dbt/tests/adapter/materialized_view/on_configuration_change.py deleted file mode 100644 index 5f91d06fd75..00000000000 --- a/tests/adapter/dbt/tests/adapter/materialized_view/on_configuration_change.py +++ /dev/null @@ -1,84 +0,0 @@ -from typing import List - -import pytest -import yaml - -from dbt.tests.util import read_file, write_file, relation_from_name -from dbt.contracts.results import RunStatus - -from dbt.tests.adapter.materialized_view.base import ( - Base, - assert_message_in_logs, -) - - -def get_project_config(project): - file_yaml = read_file(project.project_root, "dbt_project.yml") - return yaml.safe_load(file_yaml) - - -def set_project_config(project, config): - config_yaml = yaml.safe_dump(config) - write_file(config_yaml, project.project_root, "dbt_project.yml") - - -def get_model_file(project, model: str) -> str: - return read_file(project.project_root, "models", f"{model}.sql") - - -def set_model_file(project, model: str, model_sql: str): - write_file(model_sql, project.project_root, "models", f"{model}.sql") - - -def assert_proper_scenario( - on_configuration_change, - results, - logs, - status: RunStatus, - messages_in_logs: List[str] = None, - messages_not_in_logs: List[str] = None, -): - assert len(results.results) == 1 - result = results.results[0] - - assert result.node.config.on_configuration_change == on_configuration_change - assert result.status == status - for message in messages_in_logs or []: - assert_message_in_logs(logs, message) - for message in messages_not_in_logs or []: - assert_message_in_logs(logs, message, expected_fail=True) - - -class OnConfigurationChangeBase(Base): - - base_materialized_view = "base_materialized_view" - - @pytest.fixture(scope="function") - def alter_message(self, project): - return f"Applying ALTER to: {relation_from_name(project.adapter, self.base_materialized_view)}" - - @pytest.fixture(scope="function") - def create_message(self, project): - return f"Applying CREATE to: {relation_from_name(project.adapter, self.base_materialized_view)}" - - @pytest.fixture(scope="function") - def refresh_message(self, project): - return f"Applying REFRESH to: {relation_from_name(project.adapter, self.base_materialized_view)}" - - @pytest.fixture(scope="function") - def replace_message(self, project): - return f"Applying REPLACE to: {relation_from_name(project.adapter, self.base_materialized_view)}" - - @pytest.fixture(scope="function") - def configuration_change_continue_message(self, project): - return ( - f"Configuration changes were identified and `on_configuration_change` " - f"was set to `continue` for `{relation_from_name(project.adapter, self.base_materialized_view)}`" - ) - - @pytest.fixture(scope="function") - def configuration_change_fail_message(self, project): - return ( - f"Configuration changes were identified and `on_configuration_change` " - f"was set to `fail` for `{relation_from_name(project.adapter, self.base_materialized_view)}`" - ) diff --git a/tests/functional/materializations/materialized_view_tests/test_materialized_view.py b/tests/functional/materializations/materialized_view_tests/test_materialized_view.py index c5040aea92a..e46da477899 100644 --- a/tests/functional/materializations/materialized_view_tests/test_materialized_view.py +++ b/tests/functional/materializations/materialized_view_tests/test_materialized_view.py @@ -1,8 +1,13 @@ import pytest from dbt.contracts.graph.model_config import OnConfigurationChangeOption -from dbt.tests.util import run_dbt, run_dbt_and_capture - +from dbt.tests.util import ( + assert_message_in_logs, + get_model_file, + run_dbt, + run_dbt_and_capture, + set_model_file, +) from tests.functional.materializations.materialized_view_tests.files import ( MY_SEED, MY_TABLE, @@ -10,11 +15,9 @@ MY_VIEW, ) from tests.functional.materializations.materialized_view_tests.utils import ( - get_model_file, query_indexes, query_relation_type, query_row_count, - set_model_file, ) @@ -42,7 +45,9 @@ def setup(project): def create_materialized_view_and_assert(project, my_materialized_view): _, logs = run_dbt_and_capture(["--debug", "run", "--models", my_materialized_view.name]) assert query_relation_type(project, my_materialized_view) == "materialized_view" - assert f"Applying CREATE to: {my_materialized_view.fully_qualified_path}" in logs + assert_message_in_logs( + f"Applying CREATE to: {my_materialized_view.fully_qualified_path}", logs + ) def swap_indexes(project, my_materialized_view): @@ -64,8 +69,10 @@ def test_materialized_view_create_idempotent(project, my_materialized_view): _, logs = run_dbt_and_capture(["--debug", "run", "--models", my_materialized_view.name]) assert query_relation_type(project, my_materialized_view) == "materialized_view" - assert f"Applying ALTER to: {my_materialized_view.fully_qualified_path}" in logs - assert f"No changes were identified for: {my_materialized_view.fully_qualified_path}" in logs + assert_message_in_logs(f"Applying ALTER to: {my_materialized_view.fully_qualified_path}", logs) + assert_message_in_logs( + f"No changes were identified for: {my_materialized_view.fully_qualified_path}", logs + ) def test_materialized_view_replaces_table(project, my_materialized_view, my_table): @@ -137,7 +144,9 @@ def test_materialized_view_full_refresh(project, my_materialized_view): ["--debug", "run", "--models", my_materialized_view.name, "--full-refresh"] ) assert query_relation_type(project, my_materialized_view) == "materialized_view" - assert f"Applying REPLACE to: {my_materialized_view.fully_qualified_path}" in logs + assert_message_in_logs( + f"Applying REPLACE to: {my_materialized_view.fully_qualified_path}", logs + ) def test_indexes_are_updated_with_apply(project, my_materialized_view): diff --git a/tests/functional/materializations/materialized_view_tests/utils.py b/tests/functional/materializations/materialized_view_tests/utils.py index bc10505188a..62602bff31d 100644 --- a/tests/functional/materializations/materialized_view_tests/utils.py +++ b/tests/functional/materializations/materialized_view_tests/utils.py @@ -1,10 +1,9 @@ from typing import Dict, List, Optional -from dbt.adapters.relation.models import Relation, RelationStub -from dbt.tests.util import read_file, write_file +from dbt.adapters.relation.models import Relation -def query_relation_type(project, relation: RelationStub) -> Optional[str]: +def query_relation_type(project, relation: Relation) -> Optional[str]: sql = f""" select 'table' as relation_type from pg_tables @@ -30,12 +29,12 @@ def query_relation_type(project, relation: RelationStub) -> Optional[str]: return results[0][0] -def query_row_count(project, relation: RelationStub) -> int: +def query_row_count(project, relation: Relation) -> int: sql = f"select count(*) from {relation.fully_qualified_path};" return project.run_sql(sql, fetch="one")[0] -def query_indexes(project, relation: RelationStub) -> List[Dict[str, str]]: +def query_indexes(project, relation: Relation) -> List[Dict[str, str]]: # pulled directly from `postgres__describe_indexes_template` and manually verified sql = f""" select @@ -70,11 +69,3 @@ def query_indexes(project, relation: RelationStub) -> List[Dict[str, str]]: for index in raw_indexes ] return indexes - - -def get_model_file(project, relation: Relation) -> str: - return read_file(project.project_root, "models", f"{relation.name}.sql") - - -def set_model_file(project, relation: Relation, model_sql: str): - write_file(model_sql, project.project_root, "models", f"{relation.name}.sql") From 0e66a67310a896389515bf7f94fe55a84f20c8ba Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 11 Jul 2023 15:25:16 -0400 Subject: [PATCH 14/28] rename `RelationStub` to `RelationRef` --- core/dbt/adapters/base/impl.py | 22 +++++------ core/dbt/adapters/materialization/factory.py | 6 +-- .../models/_materialization.py | 28 +++++++------- .../models/_materialized_view.py | 14 +++---- core/dbt/adapters/relation/factory.py | 12 +++--- core/dbt/adapters/relation/models/__init__.py | 8 ++-- .../{_relation_stub.py => _relation_ref.py} | 38 +++++++++---------- .../materializations/materialized_view.sql | 4 +- .../macros/relations/composite/backup.sql | 6 +-- .../relations/composite/drop_backup.sql | 4 +- core/dbt/include/index.html | 2 +- .../materialized_view_tests/conftest.py | 18 ++++----- tests/unit/dbt_postgres_tests/conftest.py | 12 +++--- .../test_materialization.py | 4 +- .../test_materialization_factory.py | 2 +- .../relation_tests/test_relation_factory.py | 24 ++++++------ 16 files changed, 102 insertions(+), 102 deletions(-) rename core/dbt/adapters/relation/models/{_relation_stub.py => _relation_ref.py} (72%) diff --git a/core/dbt/adapters/base/impl.py b/core/dbt/adapters/base/impl.py index d8ef5f419b3..e7d05101c98 100644 --- a/core/dbt/adapters/base/impl.py +++ b/core/dbt/adapters/base/impl.py @@ -1440,7 +1440,7 @@ def make_materialization_from_runtime_config( self, runtime_config, materialization_type: materialization_models.MaterializationType ) -> materialization_models.Materialization: """ - Produce a `Materialization` instance along with whatever associated `Relation` and `RelationStub` + Produce a `Materialization` instance along with whatever associated `Relation` and `RelationRef` instances are needed. Args: @@ -1450,26 +1450,26 @@ def make_materialization_from_runtime_config( Returns: a `Materialization` instance that contains all the information required to execute the materialization """ - existing_relation_stub = self._get_existing_relation_stub_from_model_node( + existing_relation_ref = self._get_existing_relation_ref_from_model_node( runtime_config.model ) materialization = self.materialization_factory.make_from_runtime_config( - runtime_config, materialization_type, existing_relation_stub + runtime_config, materialization_type, existing_relation_ref ) return materialization - def _get_existing_relation_stub_from_model_node( + def _get_existing_relation_ref_from_model_node( self, model_node - ) -> Optional[relation_models.RelationStub]: + ) -> Optional[relation_models.RelationRef]: """ - We need to get `existing_relation_stub` from `Adapter` because we need access to a bunch of `cache` + We need to get `existing_relation_ref` from `Adapter` because we need access to a bunch of `cache` things, in particular `get_relations`. - TODO: if we refactor the interaction between `Adapter` and `cache`, the calculation of `existing_relation_stub` + TODO: if we refactor the interaction between `Adapter` and `cache`, the calculation of `existing_relation_ref` could be moved here, which is a more intuitive spot (like `target_relation`) for it - (and removes the concern of creating a `RelationStub` from `Adapter` where it doesn't belong + (and removes the concern of creating a `RelationRef` from `Adapter` where it doesn't belong """ existing_base_relation: BaseRelation = self.get_relation( database=model_node.database, @@ -1478,16 +1478,16 @@ def _get_existing_relation_stub_from_model_node( ) if existing_base_relation: - existing_relation_stub = self.relation_factory.make_stub( + existing_relation_ref = self.relation_factory.make_ref( name=existing_base_relation.identifier, schema_name=existing_base_relation.schema, database_name=existing_base_relation.database, relation_type=existing_base_relation.type, ) else: - existing_relation_stub = None + existing_relation_ref = None - return existing_relation_stub + return existing_relation_ref @available def make_changeset( diff --git a/core/dbt/adapters/materialization/factory.py b/core/dbt/adapters/materialization/factory.py index 13623b527cb..51dd0bc5d5e 100644 --- a/core/dbt/adapters/materialization/factory.py +++ b/core/dbt/adapters/materialization/factory.py @@ -2,7 +2,7 @@ from dbt.adapters.materialization import models from dbt.adapters.relation import RelationFactory -from dbt.adapters.relation.models import RelationStub +from dbt.adapters.relation.models import RelationRef class MaterializationFactory: @@ -22,14 +22,14 @@ def make_from_runtime_config( self, runtime_config, materialization_type: models.MaterializationType, - existing_relation_stub: Optional[RelationStub] = None, + existing_relation_ref: Optional[RelationRef] = None, ) -> Optional[models.Materialization]: if parser := self._get_parser(materialization_type): assert self.relation_factory is not None # mypy return parser.from_runtime_config( runtime_config=runtime_config, relation_factory=self.relation_factory, - existing_relation_stub=existing_relation_stub, + existing_relation_ref=existing_relation_ref, ) return None diff --git a/core/dbt/adapters/materialization/models/_materialization.py b/core/dbt/adapters/materialization/models/_materialization.py index 6a01050bc45..fa52f9fdd19 100644 --- a/core/dbt/adapters/materialization/models/_materialization.py +++ b/core/dbt/adapters/materialization/models/_materialization.py @@ -8,7 +8,7 @@ from dbt.utils import filter_null_values from dbt.adapters.relation.factory import RelationFactory -from dbt.adapters.relation.models import DescribeRelationResults, Relation, RelationStub +from dbt.adapters.relation.models import DescribeRelationResults, Relation, RelationRef class MaterializationType(StrEnum): @@ -37,7 +37,7 @@ class Materialization(ABC): type: MaterializationType relation_factory: RelationFactory target_relation: Relation - existing_relation_stub: Optional[RelationStub] = None + existing_relation_ref: Optional[RelationRef] = None is_full_refresh: bool = False grants: dict = field(default_factory=dict) on_configuration_change: OnConfigurationChangeOption = OnConfigurationChangeOption.default() @@ -55,16 +55,16 @@ def existing_relation( self, describe_relation_results: DescribeRelationResults ) -> Optional[Relation]: """ - Produce a full-blown `Relation` instance for `self.existing_relation_stub` using metadata from the database + Produce a full-blown `Relation` instance for `self.existing_relation_ref` using metadata from the database Args: - describe_relation_results: the results from the macro `describe_sql(self.existing_relation_stub)` + describe_relation_results: the results from the macro `describe_sql(self.existing_relation_ref)` Returns: - a `Relation` instance that represents `self.existing_relation_stub` in the database + a `Relation` instance that represents `self.existing_relation_ref` in the database """ - if self.existing_relation_stub: - relation_type = self.existing_relation_stub.type + if self.existing_relation_ref: + relation_type = self.existing_relation_ref.type return self.relation_factory.make_from_describe_relation_results( describe_relation_results, relation_type ) @@ -77,9 +77,9 @@ def intermediate_relation(self) -> Optional[Relation]: return None @property - def backup_relation_stub(self) -> Optional[RelationStub]: - if self.existing_relation_stub: - return self.relation_factory.make_backup_stub(self.existing_relation_stub) + def backup_relation_ref(self) -> Optional[RelationRef]: + if self.existing_relation_ref: + return self.relation_factory.make_backup_ref(self.existing_relation_ref) return None @property @@ -108,10 +108,10 @@ def from_runtime_config( cls, runtime_config, relation_factory: RelationFactory, - existing_relation_stub: Optional[RelationStub] = None, + existing_relation_ref: Optional[RelationRef] = None, ) -> "Materialization": config_dict = cls.parse_runtime_config( - runtime_config, relation_factory, existing_relation_stub + runtime_config, relation_factory, existing_relation_ref ) materialization = cls.from_dict(config_dict) return materialization @@ -121,7 +121,7 @@ def parse_runtime_config( cls, runtime_config, relation_factory: RelationFactory, - existing_relation_stub: Optional[RelationStub] = None, + existing_relation_ref: Optional[RelationRef] = None, ) -> dict: target_relation = relation_factory.make_from_model_node(runtime_config.model) # FULL_REFRESH defaults to False, hence the default in runtime_config.get() @@ -139,5 +139,5 @@ def parse_runtime_config( "is_full_refresh": is_full_refresh, "grants": grants, "on_configuration_change": on_configuration_change, - "existing_relation_stub": existing_relation_stub, + "existing_relation_ref": existing_relation_ref, } diff --git a/core/dbt/adapters/materialization/models/_materialized_view.py b/core/dbt/adapters/materialization/models/_materialized_view.py index 29ccae0eb4e..2cd94b079d3 100644 --- a/core/dbt/adapters/materialization/models/_materialized_view.py +++ b/core/dbt/adapters/materialization/models/_materialized_view.py @@ -3,7 +3,7 @@ from typing import Optional from dbt.adapters.relation.factory import RelationFactory -from dbt.adapters.relation.models import MaterializedViewRelation, RelationStub +from dbt.adapters.relation.models import MaterializedViewRelation, RelationRef from dbt.adapters.materialization.models._materialization import ( Materialization, @@ -20,20 +20,20 @@ class MaterializedViewMaterialization(Materialization, ABC): """ target_relation: MaterializedViewRelation = None # type: ignore - existing_relation_stub: RelationStub = None # type: ignore + existing_relation_ref: RelationRef = None # type: ignore intermediate_relation: MaterializedViewRelation = None # type: ignore - backup_relation_stub: RelationStub = None # type: ignore + backup_relation_ref: RelationRef = None # type: ignore @property def build_strategy(self) -> MaterializationBuildStrategy: # this is a new relation, so just create it - if self.existing_relation_stub is None: + if self.existing_relation_ref is None: return MaterializationBuildStrategy.Create # there is an existing relation, so check if we are going to replace it before determining changes elif self.is_full_refresh or ( - self.target_relation.type != self.existing_relation_stub.type + self.target_relation.type != self.existing_relation_ref.type ): return MaterializationBuildStrategy.Replace @@ -46,10 +46,10 @@ def parse_runtime_config( cls, runtime_config, relation_factory: RelationFactory, - existing_relation_stub: Optional[RelationStub] = None, + existing_relation_ref: Optional[RelationRef] = None, ) -> dict: config_dict = super().parse_runtime_config( - runtime_config, relation_factory, existing_relation_stub + runtime_config, relation_factory, existing_relation_ref ) config_dict.update({"type": MaterializationType.MaterializedView}) return config_dict diff --git a/core/dbt/adapters/relation/factory.py b/core/dbt/adapters/relation/factory.py index 9fa40f2e6b5..c06a4c05e15 100644 --- a/core/dbt/adapters/relation/factory.py +++ b/core/dbt/adapters/relation/factory.py @@ -49,14 +49,14 @@ def make_from_describe_relation_results( return relation return None - def make_stub( + def make_ref( self, name: str, schema_name: str, database_name: str, relation_type: RelationType, - ) -> models.RelationStub: - relation_stub = models.RelationStub.from_dict( + ) -> models.RelationRef: + relation_ref = models.RelationRef.from_dict( { "name": name, "schema": { @@ -72,14 +72,14 @@ def make_stub( "can_be_renamed": relation_type in self.relation_can_be_renamed, } ) - return relation_stub + return relation_ref - def make_backup_stub(self, existing_relation: models.Relation) -> models.RelationStub: + def make_backup_ref(self, existing_relation: models.Relation) -> models.RelationRef: backup_name = self.render_policy.part( ComponentName.Identifier, f"{existing_relation.name}{self.backup_suffix}" ) assert isinstance(backup_name, str) # mypy - return self.make_stub( + return self.make_ref( name=backup_name, schema_name=existing_relation.schema_name, database_name=existing_relation.database_name, diff --git a/core/dbt/adapters/relation/models/__init__.py b/core/dbt/adapters/relation/models/__init__.py index dd1613bc6a5..0f2fa02c5c1 100644 --- a/core/dbt/adapters/relation/models/__init__.py +++ b/core/dbt/adapters/relation/models/__init__.py @@ -14,9 +14,9 @@ DescribeRelationResults, RelationComponent, ) -from dbt.adapters.relation.models._relation_stub import ( - DatabaseRelationStub, - RelationStub, - SchemaRelationStub, +from dbt.adapters.relation.models._relation_ref import ( + DatabaseRelationRef, + RelationRef, + SchemaRelationRef, ) from dbt.adapters.relation.models._schema import SchemaRelation diff --git a/core/dbt/adapters/relation/models/_relation_stub.py b/core/dbt/adapters/relation/models/_relation_ref.py similarity index 72% rename from core/dbt/adapters/relation/models/_relation_stub.py rename to core/dbt/adapters/relation/models/_relation_ref.py index e2c7531a2dd..0d9627ded35 100644 --- a/core/dbt/adapters/relation/models/_relation_stub.py +++ b/core/dbt/adapters/relation/models/_relation_ref.py @@ -3,7 +3,7 @@ relation_type-specific subclasses. It's primarily used to represent a relation that exists in the database without needing to query the database. This is useful with low attribution macros (e.g. `drop_sql`, `rename_sql`) where the details are not needed to perform the action. It should be the case that if a macro supports execution -with a `RelationStub` instance, then it should also support execution with a `Relation` instance. The converse +with a `RelationRef` instance, then it should also support execution with a `Relation` instance. The converse is not true (e.g. `create_sql`). """ from dataclasses import dataclass @@ -18,17 +18,17 @@ @dataclass(frozen=True) -class DatabaseRelationStub(DatabaseRelation): +class DatabaseRelationRef(DatabaseRelation): @classmethod - def from_dict(cls, config_dict) -> "DatabaseRelationStub": - database_stub = cls( + def from_dict(cls, config_dict) -> "DatabaseRelationRef": + database_ref = cls( **{ "name": config_dict["name"], "render": config_dict["render"], } ) - assert isinstance(database_stub, DatabaseRelationStub) - return database_stub + assert isinstance(database_ref, DatabaseRelationRef) + return database_ref @classmethod def parse_model_node(cls, model_node: ModelNode) -> dict: @@ -42,21 +42,21 @@ def parse_describe_relation_results( @dataclass(frozen=True) -class SchemaRelationStub(SchemaRelation): +class SchemaRelationRef(SchemaRelation): render: RenderPolicy @classmethod - def from_dict(cls, config_dict) -> "SchemaRelationStub": - schema_stub = cls( + def from_dict(cls, config_dict) -> "SchemaRelationRef": + schema_ref = cls( **{ "name": config_dict["name"], "database": DatabaseRelation.from_dict(config_dict["database"]), "render": config_dict["render"], - "DatabaseParser": DatabaseRelationStub, + "DatabaseParser": DatabaseRelationRef, } ) - assert isinstance(schema_stub, SchemaRelationStub) - return schema_stub + assert isinstance(schema_ref, SchemaRelationRef) + return schema_ref @classmethod def parse_model_node(cls, model_node: ModelNode) -> dict: @@ -70,23 +70,23 @@ def parse_describe_relation_results( @dataclass(frozen=True) -class RelationStub(Relation): +class RelationRef(Relation): can_be_renamed: bool @classmethod - def from_dict(cls, config_dict) -> "RelationStub": - relation_stub = cls( + def from_dict(cls, config_dict) -> "RelationRef": + relation_ref = cls( **{ "name": config_dict["name"], - "schema": SchemaRelationStub.from_dict(config_dict["schema"]), + "schema": SchemaRelationRef.from_dict(config_dict["schema"]), "render": config_dict["render"], "type": config_dict["type"], "can_be_renamed": config_dict["can_be_renamed"], - "SchemaParser": SchemaRelationStub, + "SchemaParser": SchemaRelationRef, } ) - assert isinstance(relation_stub, RelationStub) - return relation_stub + assert isinstance(relation_ref, RelationRef) + return relation_ref @classmethod def parse_model_node(cls, model_node: ModelNode) -> dict: diff --git a/core/dbt/include/global_project/macros/materializations/materialized_view.sql b/core/dbt/include/global_project/macros/materializations/materialized_view.sql index fc66768a7b8..c82e1336c9e 100644 --- a/core/dbt/include/global_project/macros/materializations/materialized_view.sql +++ b/core/dbt/include/global_project/macros/materializations/materialized_view.sql @@ -30,12 +30,12 @@ {%- elif materialization.build_strategy == 'replace' -%} {%- set build_sql = replace_template( - materialization.existing_relation_stub, materialization.target_relation + materialization.existing_relation_ref, materialization.target_relation ) -%} {%- elif materialization.build_strategy == 'alter' -%} - {% set describe_relation_results = describe_template(materialization.existing_relation_stub ) %} + {% set describe_relation_results = describe_template(materialization.existing_relation_ref ) %} {% set existing_relation = materialization.existing_relation(describe_relation_results) %} {%- if materialization.on_configuration_change == 'apply' -%} diff --git a/core/dbt/include/global_project/macros/relations/composite/backup.sql b/core/dbt/include/global_project/macros/relations/composite/backup.sql index a4c24d7de2a..8cd395bc541 100644 --- a/core/dbt/include/global_project/macros/relations/composite/backup.sql +++ b/core/dbt/include/global_project/macros/relations/composite/backup.sql @@ -9,11 +9,11 @@ {%- macro default__backup_template(relation) -%} -- get the standard backup name - {% set backup_relation_stub = adapter.relation_factory.make_backup_stub(relation) -%} + {% set backup_relation_ref = adapter.relation_factory.make_backup_ref(relation) -%} -- drop any pre-existing backup - {{ drop_template(backup_relation_stub, called_directly=False) }}; + {{ drop_template(backup_relation_ref, called_directly=False) }}; - {{ rename_template(relation, backup_relation_stub.name, called_directly=False) }} + {{ rename_template(relation, backup_relation_ref.name, called_directly=False) }} {%- endmacro -%} diff --git a/core/dbt/include/global_project/macros/relations/composite/drop_backup.sql b/core/dbt/include/global_project/macros/relations/composite/drop_backup.sql index e3d6943afe6..a4d0a3088d7 100644 --- a/core/dbt/include/global_project/macros/relations/composite/drop_backup.sql +++ b/core/dbt/include/global_project/macros/relations/composite/drop_backup.sql @@ -9,8 +9,8 @@ {%- macro default__drop_backup_template(relation) -%} -- get the standard backup name - {% set backup_relation_stub = adapter.relation_factory.make_backup_stub(relation) -%} + {% set backup_relation_ref = adapter.relation_factory.make_backup_ref(relation) -%} - {{ drop_template(backup_relation_stub, called_directly=False) }} + {{ drop_template(backup_relation_ref, called_directly=False) }} {%- endmacro -%} diff --git a/core/dbt/include/index.html b/core/dbt/include/index.html index 2546d651b87..75980a35950 100644 --- a/core/dbt/include/index.html +++ b/core/dbt/include/index.html @@ -24,7 +24,7 @@
icons
-