diff --git a/.bumpversion.cfg b/.bumpversion.cfg index b3daceb77..56940f15e 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.7.0b1 +current_version = 1.7.0b2 parse = (?P[\d]+) # major version number \.(?P[\d]+) # minor version number \.(?P[\d]+) # patch version number diff --git a/.changes/1.7.0-b2.md b/.changes/1.7.0-b2.md new file mode 100644 index 000000000..c7dd145a1 --- /dev/null +++ b/.changes/1.7.0-b2.md @@ -0,0 +1,24 @@ +## dbt-redshift 1.7.0-b2 - September 15, 2023 + +### Features + +- use get_replace_sql in redshift__get_alter_materialized_view_as_sql, avoid renaming materialized views with custom table.sql and view.sql ([#606](https://github.com/dbt-labs/dbt-redshift/issues/606)) + +### Fixes + +- allow auto_refresh to take in boolean and str values for materialized views ([#560](https://github.com/dbt-labs/dbt-redshift/issues/560)) +- Fix to support removal of hologram, remove unused IAMDuration encoder ([#591](https://github.com/dbt-labs/dbt-redshift/issues/591)) +- changes expected value types to AnyInteger to take into account changes in core ([#597](https://github.com/dbt-labs/dbt-redshift/issues/597)) + +### Under the Hood + +- Restructure macro files to be more granular; organize macros by relation and type (create, drop, etc.) ([#590](https://github.com/dbt-labs/dbt-redshift/issues/590)) +- allow for bool/str input to backup/autorefresh when configuring materialized views ([#606](https://github.com/dbt-labs/dbt-redshift/issues/606)) + +### Dependencies + +- Bump mypy from 1.5.0 to 1.5.1 ([#585](https://github.com/dbt-labs/dbt-redshift/pull/585)) +- Update tox requirement from ~=4.9 to ~=4.10 ([#588](https://github.com/dbt-labs/dbt-redshift/pull/588)) +- Update tox requirement from ~=4.10 to ~=4.11 ([#594](https://github.com/dbt-labs/dbt-redshift/pull/594)) +- Update pre-commit requirement from ~=3.3 to ~=3.4 ([#596](https://github.com/dbt-labs/dbt-redshift/pull/596)) +- Update black requirement from ~=23.7 to ~=23.9 ([#600](https://github.com/dbt-labs/dbt-redshift/pull/600)) diff --git a/.changes/unreleased/Dependencies-20230816-195425.yaml b/.changes/1.7.0/Dependencies-20230816-195425.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20230816-195425.yaml rename to .changes/1.7.0/Dependencies-20230816-195425.yaml diff --git a/.changes/unreleased/Dependencies-20230821-192842.yaml b/.changes/1.7.0/Dependencies-20230821-192842.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20230821-192842.yaml rename to .changes/1.7.0/Dependencies-20230821-192842.yaml diff --git a/.changes/unreleased/Dependencies-20230901-230919.yaml b/.changes/1.7.0/Dependencies-20230901-230919.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20230901-230919.yaml rename to .changes/1.7.0/Dependencies-20230901-230919.yaml diff --git a/.changes/unreleased/Dependencies-20230904-193307.yaml b/.changes/1.7.0/Dependencies-20230904-193307.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20230904-193307.yaml rename to .changes/1.7.0/Dependencies-20230904-193307.yaml diff --git a/.changes/unreleased/Dependencies-20230912-002353.yaml b/.changes/1.7.0/Dependencies-20230912-002353.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20230912-002353.yaml rename to .changes/1.7.0/Dependencies-20230912-002353.yaml diff --git a/.changes/unreleased/Fixes-20230815-151655.yaml b/.changes/1.7.0/Fixes-20230815-151655.yaml similarity index 100% rename from .changes/unreleased/Fixes-20230815-151655.yaml rename to .changes/1.7.0/Fixes-20230815-151655.yaml diff --git a/.changes/unreleased/Fixes-20230830-164611.yaml b/.changes/1.7.0/Fixes-20230830-164611.yaml similarity index 100% rename from .changes/unreleased/Fixes-20230830-164611.yaml rename to .changes/1.7.0/Fixes-20230830-164611.yaml diff --git a/.changes/unreleased/Fixes-20230908-113019.yaml b/.changes/1.7.0/Fixes-20230908-113019.yaml similarity index 100% rename from .changes/unreleased/Fixes-20230908-113019.yaml rename to .changes/1.7.0/Fixes-20230908-113019.yaml diff --git a/.changes/unreleased/Fixes-20230912-133327.yaml b/.changes/1.7.0/Fixes-20230912-133327.yaml similarity index 93% rename from .changes/unreleased/Fixes-20230912-133327.yaml rename to .changes/1.7.0/Fixes-20230912-133327.yaml index 14528cad6..e5aa98d47 100644 --- a/.changes/unreleased/Fixes-20230912-133327.yaml +++ b/.changes/1.7.0/Fixes-20230912-133327.yaml @@ -3,3 +3,4 @@ body: use get_replace_sql in redshift__get_alter_materialized_view_as_sql, avoid time: 2023-09-12T13:33:27.451042-07:00 custom: Author: colin-rogers-dbt + Issue: "606" diff --git a/.changes/unreleased/Under the Hood-20230829-113206.yaml b/.changes/1.7.0/Under the Hood-20230829-113206.yaml similarity index 100% rename from .changes/unreleased/Under the Hood-20230829-113206.yaml rename to .changes/1.7.0/Under the Hood-20230829-113206.yaml diff --git a/.changes/1.7.0/Under the Hood-20230914-135547.yaml b/.changes/1.7.0/Under the Hood-20230914-135547.yaml new file mode 100644 index 000000000..3bb8e3c8a --- /dev/null +++ b/.changes/1.7.0/Under the Hood-20230914-135547.yaml @@ -0,0 +1,7 @@ +kind: Under the Hood +body: allow for bool/str input to backup/autorefresh when configuring materialized + views +time: 2023-09-14T13:55:47.951848-07:00 +custom: + Author: colin-rogers-dbt + Issue: "606" diff --git a/CHANGELOG.md b/CHANGELOG.md index 44a71a8bd..2bb546acc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,33 @@ - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version. - Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-redshift/blob/main/CONTRIBUTING.md#adding-changelog-entry) +## dbt-redshift 1.7.0-b2 - September 15, 2023 + +### Features + +- use get_replace_sql in redshift__get_alter_materialized_view_as_sql, avoid renaming materialized views with custom table.sql and view.sql ([#606](https://github.com/dbt-labs/dbt-redshift/issues/606)) + +### Fixes + +- allow auto_refresh to take in boolean and str values for materialized views ([#560](https://github.com/dbt-labs/dbt-redshift/issues/560)) +- Fix to support removal of hologram, remove unused IAMDuration encoder ([#591](https://github.com/dbt-labs/dbt-redshift/issues/591)) +- changes expected value types to AnyInteger to take into account changes in core ([#597](https://github.com/dbt-labs/dbt-redshift/issues/597)) + +### Under the Hood + +- Restructure macro files to be more granular; organize macros by relation and type (create, drop, etc.) ([#590](https://github.com/dbt-labs/dbt-redshift/issues/590)) +- allow for bool/str input to backup/autorefresh when configuring materialized views ([#606](https://github.com/dbt-labs/dbt-redshift/issues/606)) + +### Dependencies + +- Bump mypy from 1.5.0 to 1.5.1 ([#585](https://github.com/dbt-labs/dbt-redshift/pull/585)) +- Update tox requirement from ~=4.9 to ~=4.10 ([#588](https://github.com/dbt-labs/dbt-redshift/pull/588)) +- Update tox requirement from ~=4.10 to ~=4.11 ([#594](https://github.com/dbt-labs/dbt-redshift/pull/594)) +- Update pre-commit requirement from ~=3.3 to ~=3.4 ([#596](https://github.com/dbt-labs/dbt-redshift/pull/596)) +- Update black requirement from ~=23.7 to ~=23.9 ([#600](https://github.com/dbt-labs/dbt-redshift/pull/600)) + + + ## dbt-redshift 1.7.0-b1 - August 17, 2023 ### Features @@ -38,7 +65,6 @@ ### Contributors - [@WillAyd](https://github.com/WillAyd) ([#255](https://github.com/dbt-labs/dbt-redshift/issues/255)) - ## Previous Releases For information on prior major and minor releases, see their changelogs: - [1.6](https://github.com/dbt-labs/dbt-redshift/blob/1.6.latest/CHANGELOG.md) diff --git a/dbt/adapters/redshift/__version__.py b/dbt/adapters/redshift/__version__.py index 48607b01f..3f5d3c0b7 100644 --- a/dbt/adapters/redshift/__version__.py +++ b/dbt/adapters/redshift/__version__.py @@ -1 +1 @@ -version = "1.7.0b1" +version = "1.7.0b2" diff --git a/dbt/adapters/redshift/relation_configs/materialized_view.py b/dbt/adapters/redshift/relation_configs/materialized_view.py index e69469476..1c6fb229a 100644 --- a/dbt/adapters/redshift/relation_configs/materialized_view.py +++ b/dbt/adapters/redshift/relation_configs/materialized_view.py @@ -23,6 +23,7 @@ RedshiftSortConfig, RedshiftSortConfigChange, ) +from dbt.adapters.redshift.utility import evaluate_bool @dataclass(frozen=True, eq=True, unsafe_hash=True) @@ -122,25 +123,16 @@ def parse_model_node(cls, model_node: ModelNode) -> dict: "mv_name": model_node.identifier, "schema_name": model_node.schema, "database_name": model_node.database, - "backup": model_node.config.extra.get("backup"), } + # backup/autorefresh can be bools or strings + backup_value = model_node.config.extra.get("backup") + if backup_value is not None: + config_dict["backup"] = evaluate_bool(backup_value) + autorefresh_value = model_node.config.extra.get("auto_refresh") if autorefresh_value is not None: - if isinstance(autorefresh_value, bool): - config_dict["autorefresh"] = autorefresh_value - elif isinstance(autorefresh_value, str): - lower_autorefresh = autorefresh_value.lower() - if lower_autorefresh == "true": - config_dict["autorefresh"] = True - elif lower_autorefresh == "false": - config_dict["autorefresh"] = False - else: - raise ValueError( - "Invalid autorefresh representation. Please use accepted value ex.( True, 'true', 'True')" - ) - else: - raise TypeError("Invalid autorefresh value: expecting boolean or str.") + config_dict["autorefresh"] = evaluate_bool(autorefresh_value) if query := model_node.compiled_code: config_dict.update({"query": query.strip()}) diff --git a/dbt/adapters/redshift/utility.py b/dbt/adapters/redshift/utility.py new file mode 100644 index 000000000..64f5e9cd8 --- /dev/null +++ b/dbt/adapters/redshift/utility.py @@ -0,0 +1,25 @@ +from typing import Union + + +def evaluate_bool_str(value: str) -> bool: + value = value.strip().lower() + if value == "true": + return True + elif value == "false": + return False + else: + raise ValueError(f"Invalid boolean string value: {value}") + + +def evaluate_bool(value: Union[str, bool]) -> bool: + if not value: + return False + if isinstance(value, bool): + return value + elif isinstance(value, str): + return evaluate_bool_str(value) + else: + raise TypeError( + f"Invalid type for boolean evaluation, " + f"expecting boolean or str, recieved: {type(value)}" + ) diff --git a/tests/functional/adapter/materialized_view_tests/test_materialized_views.py b/tests/functional/adapter/materialized_view_tests/test_materialized_views.py index e61737036..cf7d433c7 100644 --- a/tests/functional/adapter/materialized_view_tests/test_materialized_views.py +++ b/tests/functional/adapter/materialized_view_tests/test_materialized_views.py @@ -11,8 +11,13 @@ MaterializedViewChangesContinueMixin, MaterializedViewChangesFailMixin, ) -from dbt.tests.adapter.materialized_view.files import MY_TABLE, MY_VIEW -from dbt.tests.util import assert_message_in_logs, get_model_file, set_model_file +from dbt.tests.adapter.materialized_view.files import MY_TABLE, MY_VIEW, MY_SEED +from dbt.tests.util import ( + assert_message_in_logs, + get_model_file, + set_model_file, + run_dbt, +) from tests.functional.adapter.materialized_view_tests.utils import ( query_autorefresh, @@ -22,7 +27,6 @@ run_dbt_and_capture_with_retries_redshift_mv, ) - MY_MATERIALIZED_VIEW = """ {{ config( materialized='materialized_view', @@ -233,3 +237,29 @@ class TestRedshiftMaterializedViewChangesFail( ): # Note: using retries doesn't work when we expect `dbt_run` to fail pass + + +NO_BACKUP_MATERIALIZED_VIEW = """ +{{ config( + materialized='materialized_view', + backup=False +) }} +select * from {{ ref('my_seed') }} +""" + + +class TestRedshiftMaterializedViewWithBackupConfig: + @pytest.fixture(scope="class", autouse=True) + def models(self): + yield { + "my_materialized_view.sql": NO_BACKUP_MATERIALIZED_VIEW, + } + + @pytest.fixture(scope="class", autouse=True) + def seeds(self): + return {"my_seed.csv": MY_SEED} + + def test_running_mv_with_backup_false_succeeds(self, project): + run_dbt(["seed"]) + result = run_dbt(["run"]) + assert result[0].node.config_call_dict["backup"] is False diff --git a/tests/unit/relation_configs/test_materialized_view.py b/tests/unit/relation_configs/test_materialized_view.py new file mode 100644 index 000000000..42a3223d0 --- /dev/null +++ b/tests/unit/relation_configs/test_materialized_view.py @@ -0,0 +1,55 @@ +from unittest.mock import Mock + +import pytest + +from dbt.adapters.redshift.relation_configs import RedshiftMaterializedViewConfig + + +@pytest.mark.parametrize("bool_value", [True, False, "True", "False", "true", "false"]) +def test_redshift_materialized_view_config_handles_all_valid_bools(bool_value): + config = RedshiftMaterializedViewConfig( + database_name="somedb", + schema_name="public", + mv_name="someview", + query="select * from sometable", + ) + model_node = Mock() + model_node.config.extra.get = ( + lambda x, y=None: bool_value if x in ["auto_refresh", "backup"] else "someDistValue" + ) + config_dict = config.parse_model_node(model_node) + assert isinstance(config_dict["autorefresh"], bool) + assert isinstance(config_dict["backup"], bool) + + +@pytest.mark.parametrize("bool_value", [1]) +def test_redshift_materialized_view_config_throws_expected_exception_with_invalid_types( + bool_value, +): + config = RedshiftMaterializedViewConfig( + database_name="somedb", + schema_name="public", + mv_name="someview", + query="select * from sometable", + ) + model_node = Mock() + model_node.config.extra.get = ( + lambda x, y=None: bool_value if x in ["auto_refresh", "backup"] else "someDistValue" + ) + with pytest.raises(TypeError): + config.parse_model_node(model_node) + + +def test_redshift_materialized_view_config_throws_expected_exception_with_invalid_str(): + config = RedshiftMaterializedViewConfig( + database_name="somedb", + schema_name="public", + mv_name="someview", + query="select * from sometable", + ) + model_node = Mock() + model_node.config.extra.get = ( + lambda x, y=None: "notABool" if x in ["auto_refresh", "backup"] else "someDistValue" + ) + with pytest.raises(ValueError): + config.parse_model_node(model_node)