Skip to content

Commit

Permalink
remove refs to runtime_config and delete test_context.py
Browse files Browse the repository at this point in the history
  • Loading branch information
colin-rogers-dbt committed Jan 2, 2024
1 parent 4487ddf commit 6c60d82
Show file tree
Hide file tree
Showing 10 changed files with 63 additions and 297 deletions.
2 changes: 1 addition & 1 deletion dbt/adapters/redshift/impl.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
import os
from dataclasses import dataclass
from dbt.common.contracts.constraints import ConstraintType
from typing import Optional, Set, Any, Dict, Type
from collections import namedtuple
from dbt.adapters.base import PythonJobHelper
from dbt.adapters.base.impl import AdapterConfig, ConstraintSupport
from dbt.adapters.base.meta import available
from dbt.adapters.sql import SQLAdapter
from dbt.adapters.contracts.connection import AdapterResponse
from dbt.contracts.graph.nodes import ConstraintType
from dbt.adapters.events.logging import AdapterLogger


Expand Down
20 changes: 8 additions & 12 deletions dbt/adapters/redshift/relation.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
from dataclasses import dataclass
from dbt.adapters.contracts.relation import RelationConfig
from typing import Optional

from dbt.adapters.base.relation import BaseRelation
Expand All @@ -7,8 +8,6 @@
RelationConfigChangeAction,
RelationResults,
)
from dbt.context.providers import RuntimeConfigObject
from dbt.contracts.graph.nodes import ModelNode
from dbt.adapters.base import RelationType
from dbt.common.exceptions import DbtRuntimeError

Expand Down Expand Up @@ -60,31 +59,28 @@ def relation_max_name_length(self):
return MAX_CHARACTERS_IN_IDENTIFIER

@classmethod
def from_runtime_config(cls, runtime_config: RuntimeConfigObject) -> RelationConfigBase:
model_node: ModelNode = runtime_config.model
relation_type: str = model_node.config.materialized
def from_config(cls, config: RelationConfig) -> RelationConfigBase:
relation_type: str = config.config.materialized # type: ignore

if relation_config := cls.relation_configs.get(relation_type):
return relation_config.from_model_node(model_node)
return relation_config.from_config(relation_config) # type: ignore

raise DbtRuntimeError(
f"from_runtime_config() is not supported for the provided relation type: {relation_type}"
f"from_config() is not supported for the provided relation type: {relation_type}"
)

@classmethod
def materialized_view_config_changeset(
cls, relation_results: RelationResults, runtime_config: RuntimeConfigObject
cls, relation_results: RelationResults, relation_config: RelationConfig
) -> Optional[RedshiftMaterializedViewConfigChangeset]:
config_change_collection = RedshiftMaterializedViewConfigChangeset()

existing_materialized_view = RedshiftMaterializedViewConfig.from_relation_results(
relation_results
)
new_materialized_view = RedshiftMaterializedViewConfig.from_model_node(
runtime_config.model
new_materialized_view = RedshiftMaterializedViewConfig.from_relation_config(
relation_config
)
assert isinstance(existing_materialized_view, RedshiftMaterializedViewConfig)
assert isinstance(new_materialized_view, RedshiftMaterializedViewConfig)

if new_materialized_view.autorefresh != existing_materialized_view.autorefresh:
config_change_collection.autorefresh = RedshiftAutoRefreshConfigChange(
Expand Down
24 changes: 12 additions & 12 deletions dbt/adapters/redshift/relation_configs/base.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
from dataclasses import dataclass
from typing import Optional
from typing import Optional, Dict

import agate
from dbt.adapters.base.relation import Policy
from dbt.adapters.contracts.relation import ComponentName
from dbt.adapters.contracts.relation import ComponentName, RelationConfig
from dbt.adapters.relation_configs import (
RelationConfigBase,
RelationResults,
)
from dbt.contracts.graph.nodes import ModelNode
from typing_extensions import Self

from dbt.adapters.redshift.relation_configs.policies import (
RedshiftIncludePolicy,
Expand All @@ -31,25 +31,25 @@ def quote_policy(cls) -> Policy:
return RedshiftQuotePolicy()

@classmethod
def from_model_node(cls, model_node: ModelNode) -> "RelationConfigBase":
relation_config = cls.parse_model_node(model_node)
relation = cls.from_dict(relation_config)
return relation
def from_relation_config(cls, relation_config: RelationConfig) -> Self:
relation_config_dict = cls.parse_relation_config(relation_config)
relation = cls.from_dict(relation_config_dict)
return relation # type: ignore

@classmethod
def parse_model_node(cls, model_node: ModelNode) -> dict:
def parse_relation_config(cls, relation_config: RelationConfig) -> Dict:
raise NotImplementedError(
"`parse_model_node()` needs to be implemented on this RelationConfigBase instance"
"`parse_relation_config()` needs to be implemented on this RelationConfigBase instance"
)

@classmethod
def from_relation_results(cls, relation_results: RelationResults) -> "RelationConfigBase":
def from_relation_results(cls, relation_results: RelationResults) -> Self:
relation_config = cls.parse_relation_results(relation_results)
relation = cls.from_dict(relation_config)
return relation
return relation # type: ignore

@classmethod
def parse_relation_results(cls, relation_results: RelationResults) -> dict:
def parse_relation_results(cls, relation_results: RelationResults) -> Dict:
raise NotImplementedError(
"`parse_relation_results()` needs to be implemented on this RelationConfigBase instance"
)
Expand Down
17 changes: 9 additions & 8 deletions dbt/adapters/redshift/relation_configs/dist.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from dataclasses import dataclass
from typing import Optional, Set
from dbt.adapters.contracts.relation import RelationConfig
from typing import Optional, Set, Dict

import agate
from dbt.adapters.relation_configs import (
Expand All @@ -8,9 +9,9 @@
RelationConfigValidationMixin,
RelationConfigValidationRule,
)
from dbt.contracts.graph.nodes import ModelNode
from dbt.common.dataclass_schema import StrEnum
from dbt.common.exceptions import DbtRuntimeError
from typing_extensions import Self

from dbt.adapters.redshift.relation_configs.base import RedshiftRelationConfigBase

Expand Down Expand Up @@ -65,29 +66,29 @@ def validation_rules(self) -> Set[RelationConfigValidationRule]:
}

@classmethod
def from_dict(cls, config_dict) -> "RedshiftDistConfig":
def from_dict(cls, config_dict) -> Self:
kwargs_dict = {
"diststyle": config_dict.get("diststyle"),
"distkey": config_dict.get("distkey"),
}
dist: "RedshiftDistConfig" = super().from_dict(kwargs_dict) # type: ignore
dist: Self = super().from_dict(kwargs_dict) # type: ignore
return dist

@classmethod
def parse_model_node(cls, model_node: ModelNode) -> dict:
def parse_relation_config(cls, relation_config: RelationConfig) -> dict:
"""
Translate ModelNode objects from the user-provided config into a standard dictionary.
Args:
model_node: the description of the distkey and diststyle from the user in this format:
relation_config: the description of the distkey and diststyle from the user in this format:
{
"dist": any("auto", "even", "all") or "<column_name>"
}
Returns: a standard dictionary describing this `RedshiftDistConfig` instance
"""
dist = model_node.config.extra.get("dist", "")
dist = relation_config.config.extra.get("dist", "") # type: ignore

diststyle = dist.lower()

Expand All @@ -107,7 +108,7 @@ def parse_model_node(cls, model_node: ModelNode) -> dict:
return config

@classmethod
def parse_relation_results(cls, relation_results_entry: agate.Row) -> dict:
def parse_relation_results(cls, relation_results_entry: agate.Row) -> Dict:
"""
Translate agate objects from the database into a standard dictionary.
Expand Down
36 changes: 18 additions & 18 deletions dbt/adapters/redshift/relation_configs/materialized_view.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
from dataclasses import dataclass, field
from typing import Optional, Set
from typing import Optional, Set, Dict, Any

import agate
from dbt.adapters.relation_configs import (
Expand All @@ -8,9 +8,9 @@
RelationConfigValidationMixin,
RelationConfigValidationRule,
)
from dbt.contracts.graph.nodes import ModelNode
from dbt.adapters.contracts.relation import ComponentName
from dbt.adapters.contracts.relation import ComponentName, RelationConfig
from dbt.common.exceptions import DbtRuntimeError
from typing_extensions import Self

from dbt.adapters.redshift.relation_configs.base import RedshiftRelationConfigBase
from dbt.adapters.redshift.relation_configs.dist import (
Expand Down Expand Up @@ -95,7 +95,7 @@ def validation_rules(self) -> Set[RelationConfigValidationRule]:
}

@classmethod
def from_dict(cls, config_dict) -> "RedshiftMaterializedViewConfig":
def from_dict(cls, config_dict) -> Self:
kwargs_dict = {
"mv_name": cls._render_part(ComponentName.Identifier, config_dict.get("mv_name")),
"schema_name": cls._render_part(ComponentName.Schema, config_dict.get("schema_name")),
Expand All @@ -114,39 +114,39 @@ def from_dict(cls, config_dict) -> "RedshiftMaterializedViewConfig":
if sort := config_dict.get("sort"):
kwargs_dict.update({"sort": RedshiftSortConfig.from_dict(sort)})

materialized_view: "RedshiftMaterializedViewConfig" = super().from_dict(kwargs_dict) # type: ignore
materialized_view: Self = super().from_dict(kwargs_dict) # type: ignore
return materialized_view

@classmethod
def parse_model_node(cls, model_node: ModelNode) -> dict:
config_dict = {
"mv_name": model_node.identifier,
"schema_name": model_node.schema,
"database_name": model_node.database,
def parse_relation_config(cls, config: RelationConfig) -> Dict[str, Any]:
config_dict: Dict[str, Any] = {
"mv_name": config.identifier,
"schema_name": config.schema,
"database_name": config.database,
}

# backup/autorefresh can be bools or strings
backup_value = model_node.config.extra.get("backup")
backup_value = config.config.extra.get("backup") # type: ignore
if backup_value is not None:
config_dict["backup"] = evaluate_bool(backup_value)

autorefresh_value = model_node.config.extra.get("auto_refresh")
autorefresh_value = config.config.extra.get("auto_refresh") # type: ignore
if autorefresh_value is not None:
config_dict["autorefresh"] = evaluate_bool(autorefresh_value)

if query := model_node.compiled_code:
if query := config.compiled_code: # type: ignore
config_dict.update({"query": query.strip()})

if model_node.config.get("dist"):
config_dict.update({"dist": RedshiftDistConfig.parse_model_node(model_node)})
if config.config.get("dist"):
config_dict.update({"dist": RedshiftDistConfig.parse_relation_config(config)})

if model_node.config.get("sort"):
config_dict.update({"sort": RedshiftSortConfig.parse_model_node(model_node)})
if config.config.get("sort"):
config_dict.update({"sort": RedshiftSortConfig.parse_relation_config(config)})

return config_dict

@classmethod
def parse_relation_results(cls, relation_results: RelationResults) -> dict:
def parse_relation_results(cls, relation_results: RelationResults) -> Dict:
"""
Translate agate objects from the database into a standard dictionary.
Expand Down
19 changes: 10 additions & 9 deletions dbt/adapters/redshift/relation_configs/sort.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from dataclasses import dataclass
from typing import Optional, FrozenSet, Set
from dbt.adapters.contracts.relation import RelationConfig
from typing import Optional, FrozenSet, Set, Dict, Any

import agate
from dbt.adapters.relation_configs import (
Expand All @@ -8,9 +9,9 @@
RelationConfigValidationMixin,
RelationConfigValidationRule,
)
from dbt.contracts.graph.nodes import ModelNode
from dbt.common.dataclass_schema import StrEnum
from dbt.common.exceptions import DbtRuntimeError
from typing_extensions import Self

from dbt.adapters.redshift.relation_configs.base import RedshiftRelationConfigBase

Expand Down Expand Up @@ -97,21 +98,21 @@ def validation_rules(self) -> Set[RelationConfigValidationRule]:
}

@classmethod
def from_dict(cls, config_dict) -> "RedshiftSortConfig":
def from_dict(cls, config_dict) -> Self:
kwargs_dict = {
"sortstyle": config_dict.get("sortstyle"),
"sortkey": frozenset(column for column in config_dict.get("sortkey", {})),
}
sort: "RedshiftSortConfig" = super().from_dict(kwargs_dict) # type: ignore
return sort
sort: Self = super().from_dict(kwargs_dict) # type: ignore
return sort # type: ignore

@classmethod
def parse_model_node(cls, model_node: ModelNode) -> dict:
def parse_relation_config(cls, relation_config: RelationConfig) -> Dict[str, Any]:
"""
Translate ModelNode objects from the user-provided config into a standard dictionary.
Args:
model_node: the description of the sortkey and sortstyle from the user in this format:
relation_config: the description of the sortkey and sortstyle from the user in this format:
{
"sort_key": "<column_name>" or ["<column_name>"] or ["<column1_name>",...]
Expand All @@ -122,10 +123,10 @@ def parse_model_node(cls, model_node: ModelNode) -> dict:
"""
config_dict = {}

if sortstyle := model_node.config.extra.get("sort_type"):
if sortstyle := relation_config.config.extra.get("sort_type"): # type: ignore
config_dict.update({"sortstyle": sortstyle.lower()})

if sortkey := model_node.config.extra.get("sort"):
if sortkey := relation_config.config.extra.get("sort"): # type: ignore
# we allow users to specify the `sort_key` as a string if it's a single column
if isinstance(sortkey, str):
sortkey = [sortkey]
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{% macro redshift__get_materialized_view_configuration_changes(existing_relation, new_config) %}
{% set _existing_materialized_view = redshift__describe_materialized_view(existing_relation) %}
{% set _configuration_changes = existing_relation.materialized_view_config_changeset(_existing_materialized_view, new_config) %}
{% set _configuration_changes = existing_relation.materialized_view_config_changeset(_existing_materialized_view, new_config.model) %}
{% do return(_configuration_changes) %}
{% endmacro %}
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{% macro redshift__get_create_materialized_view_as_sql(relation, sql) %}

{%- set materialized_view = relation.from_runtime_config(config) -%}
{%- set materialized_view = relation.from_config(config.model) -%}

create materialized view {{ materialized_view.path }}
backup {% if materialized_view.backup %}yes{% else %}no{% endif %}
Expand Down
6 changes: 3 additions & 3 deletions tests/unit/relation_configs/test_materialized_view.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ def test_redshift_materialized_view_config_handles_all_valid_bools(bool_value):
model_node.config.extra.get = (
lambda x, y=None: bool_value if x in ["auto_refresh", "backup"] else "someDistValue"
)
config_dict = config.parse_model_node(model_node)
config_dict = config.parse_relation_config(model_node)
assert isinstance(config_dict["autorefresh"], bool)
assert isinstance(config_dict["backup"], bool)

Expand All @@ -37,7 +37,7 @@ def test_redshift_materialized_view_config_throws_expected_exception_with_invali
lambda x, y=None: bool_value if x in ["auto_refresh", "backup"] else "someDistValue"
)
with pytest.raises(TypeError):
config.parse_model_node(model_node)
config.parse_relation_config(model_node)


def test_redshift_materialized_view_config_throws_expected_exception_with_invalid_str():
Expand All @@ -52,4 +52,4 @@ def test_redshift_materialized_view_config_throws_expected_exception_with_invali
lambda x, y=None: "notABool" if x in ["auto_refresh", "backup"] else "someDistValue"
)
with pytest.raises(ValueError):
config.parse_model_node(model_node)
config.parse_relation_config(model_node)
Loading

0 comments on commit 6c60d82

Please sign in to comment.