From 4c5e49bb5826dd48001156c14445eabde8dee4b5 Mon Sep 17 00:00:00 2001 From: Matthew McKnight <91097623+McKnight-42@users.noreply.github.com> Date: Mon, 11 Sep 2023 11:41:29 -0500 Subject: [PATCH 01/15] change expected_stat types to match agate_helper change in dbt-core (#598) --- .changes/unreleased/Fixes-20230908-113019.yaml | 6 ++++++ tests/functional/adapter/expected_stats.py | 6 +++--- 2 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 .changes/unreleased/Fixes-20230908-113019.yaml diff --git a/.changes/unreleased/Fixes-20230908-113019.yaml b/.changes/unreleased/Fixes-20230908-113019.yaml new file mode 100644 index 000000000..ed5957317 --- /dev/null +++ b/.changes/unreleased/Fixes-20230908-113019.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: changes expected value types to AnyInteger to take into account changes in core +time: 2023-09-08T11:30:19.77143-05:00 +custom: + Author: McKnight-42 + Issue: "597" diff --git a/tests/functional/adapter/expected_stats.py b/tests/functional/adapter/expected_stats.py index 9e635bdbb..265ae225a 100644 --- a/tests/functional/adapter/expected_stats.py +++ b/tests/functional/adapter/expected_stats.py @@ -1,4 +1,4 @@ -from dbt.tests.util import AnyStringWith, AnyFloat, AnyString +from dbt.tests.util import AnyStringWith, AnyInteger, AnyString, AnyFloat def redshift_stats(): @@ -27,14 +27,14 @@ def redshift_stats(): "max_varchar": { "id": "max_varchar", "label": "Max Varchar", - "value": AnyFloat(), + "value": AnyInteger(), "description": "Size of the largest column that uses a VARCHAR data type.", "include": True, }, "size": { "id": "size", "label": "Approximate Size", - "value": AnyFloat(), + "value": AnyInteger(), "description": "Approximate size of the table, calculated from a count of 1MB blocks", "include": True, }, From 9effccedff00885827bffe8b25de01906e425341 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Sep 2023 00:20:29 +0000 Subject: [PATCH 02/15] Update tox requirement from ~=4.10 to ~=4.11 (#594) * Update tox requirement from ~=4.10 to ~=4.11 Updates the requirements on [tox](https://github.com/tox-dev/tox) to permit the latest version. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.10.0...4.11.1) --- updated-dependencies: - dependency-name: tox dependency-type: direct:development ... Signed-off-by: dependabot[bot] * Add automated changelog yaml from template for bot PR --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Github Build Bot --- .changes/unreleased/Dependencies-20230901-230919.yaml | 6 ++++++ dev-requirements.txt | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Dependencies-20230901-230919.yaml diff --git a/.changes/unreleased/Dependencies-20230901-230919.yaml b/.changes/unreleased/Dependencies-20230901-230919.yaml new file mode 100644 index 000000000..178fc9ff5 --- /dev/null +++ b/.changes/unreleased/Dependencies-20230901-230919.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Update tox requirement from ~=4.10 to ~=4.11" +time: 2023-09-01T23:09:19.00000Z +custom: + Author: dependabot[bot] + PR: 594 diff --git a/dev-requirements.txt b/dev-requirements.txt index f887fe214..c4b2df3aa 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -24,7 +24,7 @@ pytest-dotenv~=0.5.2 pytest-logbook~=1.2 pytest-xdist~=3.3 pytz~=2023.3 -tox~=4.10 +tox~=4.11 types-pytz~=2023.3 types-requests~=2.31 twine~=4.0 From 5f242f646d710a37787e22c08bdaa820998e45e1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Sep 2023 00:25:56 +0000 Subject: [PATCH 03/15] Update pre-commit requirement from ~=3.3 to ~=3.4 (#596) * Update pre-commit requirement from ~=3.3 to ~=3.4 Updates the requirements on [pre-commit](https://github.com/pre-commit/pre-commit) to permit the latest version. - [Release notes](https://github.com/pre-commit/pre-commit/releases) - [Changelog](https://github.com/pre-commit/pre-commit/blob/main/CHANGELOG.md) - [Commits](https://github.com/pre-commit/pre-commit/compare/v3.3.0...v3.4.0) --- updated-dependencies: - dependency-name: pre-commit dependency-type: direct:development ... Signed-off-by: dependabot[bot] * Add automated changelog yaml from template for bot PR --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Github Build Bot Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> --- .changes/unreleased/Dependencies-20230904-193307.yaml | 6 ++++++ dev-requirements.txt | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Dependencies-20230904-193307.yaml diff --git a/.changes/unreleased/Dependencies-20230904-193307.yaml b/.changes/unreleased/Dependencies-20230904-193307.yaml new file mode 100644 index 000000000..11707de31 --- /dev/null +++ b/.changes/unreleased/Dependencies-20230904-193307.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Update pre-commit requirement from ~=3.3 to ~=3.4" +time: 2023-09-04T19:33:07.00000Z +custom: + Author: dependabot[bot] + PR: 596 diff --git a/dev-requirements.txt b/dev-requirements.txt index c4b2df3aa..d558aa109 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -16,7 +16,7 @@ freezegun~=1.2 ipdb~=0.13.13 mypy==1.5.1 # patch updates have historically introduced breaking changes pip-tools~=7.3 -pre-commit~=3.3 +pre-commit~=3.4 pre-commit-hooks~=4.4 pytest~=7.4 pytest-csv~=3.0 From 538ad793769269454ff390135566a85d729fcdcc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Sep 2023 02:04:35 +0000 Subject: [PATCH 04/15] Update black requirement from ~=23.7 to ~=23.9 (#600) * Update black requirement from ~=23.7 to ~=23.9 Updates the requirements on [black](https://github.com/psf/black) to permit the latest version. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/23.7.0...23.9.1) --- updated-dependencies: - dependency-name: black dependency-type: direct:development ... Signed-off-by: dependabot[bot] * Add automated changelog yaml from template for bot PR --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Co-authored-by: Github Build Bot --- .changes/unreleased/Dependencies-20230912-002353.yaml | 6 ++++++ dev-requirements.txt | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Dependencies-20230912-002353.yaml diff --git a/.changes/unreleased/Dependencies-20230912-002353.yaml b/.changes/unreleased/Dependencies-20230912-002353.yaml new file mode 100644 index 000000000..be7230348 --- /dev/null +++ b/.changes/unreleased/Dependencies-20230912-002353.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Update black requirement from ~=23.7 to ~=23.9" +time: 2023-09-12T00:23:53.00000Z +custom: + Author: dependabot[bot] + PR: 600 diff --git a/dev-requirements.txt b/dev-requirements.txt index d558aa109..96f252ebe 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -6,7 +6,7 @@ git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-postgres&subdirectory=plugi # if version 1.x or greater -> pin to major version # if version 0.x -> pin to minor -black~=23.7 +black~=23.9 bumpversion~=0.6.0 click~=8.1 ddtrace~=1.18 From 9bb38835b651a1fa85925f1db4b550d15eb74dc0 Mon Sep 17 00:00:00 2001 From: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Tue, 12 Sep 2023 14:19:33 -0700 Subject: [PATCH 05/15] Fix replace sql macro (#602) * use get_replace_sql in redshift__get_alter_materialized_view_as_sql * add changie --- .changes/unreleased/Fixes-20230912-133327.yaml | 5 +++++ .../redshift/macros/relations/materialized_view/alter.sql | 2 +- tests/functional/adapter/materialized_view_tests/utils.py | 5 +++-- 3 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 .changes/unreleased/Fixes-20230912-133327.yaml diff --git a/.changes/unreleased/Fixes-20230912-133327.yaml b/.changes/unreleased/Fixes-20230912-133327.yaml new file mode 100644 index 000000000..afa2f3657 --- /dev/null +++ b/.changes/unreleased/Fixes-20230912-133327.yaml @@ -0,0 +1,5 @@ +kind: Fixes +body: use get_replace_sql in redshift__get_alter_materialized_view_as_sql +time: 2023-09-12T13:33:27.451042-07:00 +custom: + Author: colin-rogers-dbt diff --git a/dbt/include/redshift/macros/relations/materialized_view/alter.sql b/dbt/include/redshift/macros/relations/materialized_view/alter.sql index 6165e441d..7f0379847 100644 --- a/dbt/include/redshift/macros/relations/materialized_view/alter.sql +++ b/dbt/include/redshift/macros/relations/materialized_view/alter.sql @@ -10,7 +10,7 @@ -- apply a full refresh immediately if needed {% if configuration_changes.requires_full_refresh %} - {{ get_replace_materialized_view_as_sql(relation, sql, existing_relation, backup_relation, intermediate_relation) }} + {{ get_replace_sql(existing_relation, relation, sql) }} -- otherwise apply individual changes as needed {% else %} diff --git a/tests/functional/adapter/materialized_view_tests/utils.py b/tests/functional/adapter/materialized_view_tests/utils.py index bc172be69..112ae3057 100644 --- a/tests/functional/adapter/materialized_view_tests/utils.py +++ b/tests/functional/adapter/materialized_view_tests/utils.py @@ -82,6 +82,7 @@ def run_dbt_and_capture_with_retries_redshift_mv(args: List[str], max_retries: i try: # there's no point to using this with expect_pass=False return run_dbt_and_capture(args, expect_pass=True) - except AssertionError: + except AssertionError as e: retries += 1 - return None + if retries == max_retries: + raise e From 56af9123dcde326c21d3f360589cc60cbe674aa0 Mon Sep 17 00:00:00 2001 From: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Wed, 13 Sep 2023 11:50:58 -0700 Subject: [PATCH 06/15] add relation macros and add relation renameable/replaceable config (#603) * add relation macros and add relation renameable/replaceable config * formatting cleanup * PR fixes * make tables irreplaceable --- .../unreleased/Fixes-20230912-133327.yaml | 4 +- dbt/adapters/redshift/relation.py | 11 +++ .../macros/materializations/table.sql | 69 +++++++++++++++++ .../redshift/macros/materializations/view.sql | 77 +++++++++++++++++++ .../redshift/macros/relations/drop.sql | 7 -- .../relations/materialized_view/_replace.sql | 4 - .../redshift/macros/relations/table/drop.sql | 3 + .../macros/relations/table/rename.sql | 3 + .../redshift/macros/relations/view/drop.sql | 3 + .../redshift/macros/relations/view/rename.sql | 3 + .../macros/relations/view/replace.sql | 18 +++++ .../test_materialized_views.py | 12 --- 12 files changed, 189 insertions(+), 25 deletions(-) create mode 100644 dbt/include/redshift/macros/materializations/table.sql create mode 100644 dbt/include/redshift/macros/materializations/view.sql delete mode 100644 dbt/include/redshift/macros/relations/drop.sql delete mode 100644 dbt/include/redshift/macros/relations/materialized_view/_replace.sql create mode 100644 dbt/include/redshift/macros/relations/table/drop.sql create mode 100644 dbt/include/redshift/macros/relations/table/rename.sql create mode 100644 dbt/include/redshift/macros/relations/view/drop.sql create mode 100644 dbt/include/redshift/macros/relations/view/rename.sql create mode 100644 dbt/include/redshift/macros/relations/view/replace.sql diff --git a/.changes/unreleased/Fixes-20230912-133327.yaml b/.changes/unreleased/Fixes-20230912-133327.yaml index afa2f3657..14528cad6 100644 --- a/.changes/unreleased/Fixes-20230912-133327.yaml +++ b/.changes/unreleased/Fixes-20230912-133327.yaml @@ -1,5 +1,5 @@ -kind: Fixes -body: use get_replace_sql in redshift__get_alter_materialized_view_as_sql +kind: Features +body: use get_replace_sql in redshift__get_alter_materialized_view_as_sql, avoid renaming materialized views with custom table.sql and view.sql time: 2023-09-12T13:33:27.451042-07:00 custom: Author: colin-rogers-dbt diff --git a/dbt/adapters/redshift/relation.py b/dbt/adapters/redshift/relation.py index 0ef4fe276..182b9b810 100644 --- a/dbt/adapters/redshift/relation.py +++ b/dbt/adapters/redshift/relation.py @@ -32,6 +32,17 @@ class RedshiftRelation(BaseRelation): relation_configs = { RelationType.MaterializedView.value: RedshiftMaterializedViewConfig, } + renameable_relations = frozenset( + { + RelationType.View, + RelationType.Table, + } + ) + replaceable_relations = frozenset( + { + RelationType.View, + } + ) def __post_init__(self): # Check for length of Redshift table/view names. diff --git a/dbt/include/redshift/macros/materializations/table.sql b/dbt/include/redshift/macros/materializations/table.sql new file mode 100644 index 000000000..907c83874 --- /dev/null +++ b/dbt/include/redshift/macros/materializations/table.sql @@ -0,0 +1,69 @@ +{% materialization table, adapter='redshift' %} + + {%- set existing_relation = load_cached_relation(this) -%} + {%- set target_relation = this.incorporate(type='table') %} + {%- set intermediate_relation = make_intermediate_relation(target_relation) -%} + -- the intermediate_relation should not already exist in the database; get_relation + -- will return None in that case. Otherwise, we get a relation that we can drop + -- later, before we try to use this name for the current operation + {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%} + /* + See ../view/view.sql for more information about this relation. + */ + {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%} + {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%} + -- as above, the backup_relation should not already exist + {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%} + -- grab current tables grants config for comparision later on + {% set grant_config = config.get('grants') %} + + -- drop the temp relations if they exist already in the database + {{ drop_relation_if_exists(preexisting_intermediate_relation) }} + {{ drop_relation_if_exists(preexisting_backup_relation) }} + + {{ run_hooks(pre_hooks, inside_transaction=False) }} + + -- `BEGIN` happens here: + {{ run_hooks(pre_hooks, inside_transaction=True) }} + + -- build model + {% call statement('main') -%} + {{ get_create_table_as_sql(False, intermediate_relation, sql) }} + {%- endcall %} + + -- cleanup + {% if existing_relation is not none %} + /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped + since the variable was first set. */ + {% set existing_relation = load_cached_relation(existing_relation) %} + {% if existing_relation is not none %} + {% if existing_relation.can_be_renamed %} + {{ adapter.rename_relation(existing_relation, backup_relation) }} + {% else %} + {{ drop_relation_if_exists(existing_relation) }} + {% endif %} + {% endif %} + {% endif %} + + + {{ adapter.rename_relation(intermediate_relation, target_relation) }} + + {% do create_indexes(target_relation) %} + + {{ run_hooks(post_hooks, inside_transaction=True) }} + + {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %} + {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} + + {% do persist_docs(target_relation, model) %} + + -- `COMMIT` happens here + {{ adapter.commit() }} + + -- finally, drop the existing/backup relation after the commit + {{ drop_relation_if_exists(backup_relation) }} + + {{ run_hooks(post_hooks, inside_transaction=False) }} + + {{ return({'relations': [target_relation]}) }} +{% endmaterialization %} diff --git a/dbt/include/redshift/macros/materializations/view.sql b/dbt/include/redshift/macros/materializations/view.sql new file mode 100644 index 000000000..f353f913f --- /dev/null +++ b/dbt/include/redshift/macros/materializations/view.sql @@ -0,0 +1,77 @@ +{%- materialization view, adapter='redshift' -%} + + {%- set existing_relation = load_cached_relation(this) -%} + {%- set target_relation = this.incorporate(type='view') -%} + {%- set intermediate_relation = make_intermediate_relation(target_relation) -%} + + -- the intermediate_relation should not already exist in the database; get_relation + -- will return None in that case. Otherwise, we get a relation that we can drop + -- later, before we try to use this name for the current operation + {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%} + /* + This relation (probably) doesn't exist yet. If it does exist, it's a leftover from + a previous run, and we're going to try to drop it immediately. At the end of this + materialization, we're going to rename the "existing_relation" to this identifier, + and then we're going to drop it. In order to make sure we run the correct one of: + - drop view ... + - drop table ... + + We need to set the type of this relation to be the type of the existing_relation, if it exists, + or else "view" as a sane default if it does not. Note that if the existing_relation does not + exist, then there is nothing to move out of the way and subsequentally drop. In that case, + this relation will be effectively unused. + */ + {%- set backup_relation_type = 'view' if existing_relation is none else existing_relation.type -%} + {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%} + -- as above, the backup_relation should not already exist + {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%} + -- grab current tables grants config for comparision later on + {% set grant_config = config.get('grants') %} + + {{ run_hooks(pre_hooks, inside_transaction=False) }} + + -- drop the temp relations if they exist already in the database + {{ drop_relation_if_exists(preexisting_intermediate_relation) }} + {{ drop_relation_if_exists(preexisting_backup_relation) }} + + -- `BEGIN` happens here: + {{ run_hooks(pre_hooks, inside_transaction=True) }} + + -- build model + {% call statement('main') -%} + {{ get_create_view_as_sql(intermediate_relation, sql) }} + {%- endcall %} + + -- cleanup + -- move the existing view out of the way + {% if existing_relation is not none %} + /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped + since the variable was first set. */ + {% set existing_relation = load_cached_relation(existing_relation) %} + {% if existing_relation is not none %} + {% if existing_relation.can_be_renamed %} + {{ adapter.rename_relation(existing_relation, backup_relation) }} + {% else %} + {{ drop_relation_if_exists(existing_relation) }} + {% endif %} + {% endif %} + {% endif %} + + {{ adapter.rename_relation(intermediate_relation, target_relation) }} + + {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %} + {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} + + {% do persist_docs(target_relation, model) %} + + {{ run_hooks(post_hooks, inside_transaction=True) }} + + {{ adapter.commit() }} + + {{ drop_relation_if_exists(backup_relation) }} + + {{ run_hooks(post_hooks, inside_transaction=False) }} + + {{ return({'relations': [target_relation]}) }} + +{%- endmaterialization -%} diff --git a/dbt/include/redshift/macros/relations/drop.sql b/dbt/include/redshift/macros/relations/drop.sql deleted file mode 100644 index 2709f7763..000000000 --- a/dbt/include/redshift/macros/relations/drop.sql +++ /dev/null @@ -1,7 +0,0 @@ -{% macro redshift__get_drop_relation_sql(relation) %} - {%- if relation.is_materialized_view -%} - {{ redshift__drop_materialized_view(relation) }} - {%- else -%} - drop {{ relation.type }} if exists {{ relation }} cascade - {%- endif -%} -{% endmacro %} diff --git a/dbt/include/redshift/macros/relations/materialized_view/_replace.sql b/dbt/include/redshift/macros/relations/materialized_view/_replace.sql deleted file mode 100644 index 2146e0967..000000000 --- a/dbt/include/redshift/macros/relations/materialized_view/_replace.sql +++ /dev/null @@ -1,4 +0,0 @@ -{% macro redshift__get_replace_materialized_view_as_sql(relation, sql, existing_relation, backup_relation, intermediate_relation) %} - {{ redshift__get_drop_relation_sql(existing_relation) }}; - {{ get_create_materialized_view_as_sql(relation, sql) }} -{% endmacro %} diff --git a/dbt/include/redshift/macros/relations/table/drop.sql b/dbt/include/redshift/macros/relations/table/drop.sql new file mode 100644 index 000000000..64ffc1f22 --- /dev/null +++ b/dbt/include/redshift/macros/relations/table/drop.sql @@ -0,0 +1,3 @@ +{%- macro redshift__drop_table(relation) -%} + drop table if exists {{ relation }} cascade +{%- endmacro -%} diff --git a/dbt/include/redshift/macros/relations/table/rename.sql b/dbt/include/redshift/macros/relations/table/rename.sql new file mode 100644 index 000000000..08fd5a172 --- /dev/null +++ b/dbt/include/redshift/macros/relations/table/rename.sql @@ -0,0 +1,3 @@ +{% macro redshift__get_rename_table_sql(relation, new_name) %} + alter table {{ relation }} rename to {{ new_name }} +{% endmacro %} diff --git a/dbt/include/redshift/macros/relations/view/drop.sql b/dbt/include/redshift/macros/relations/view/drop.sql new file mode 100644 index 000000000..cba066a53 --- /dev/null +++ b/dbt/include/redshift/macros/relations/view/drop.sql @@ -0,0 +1,3 @@ +{%- macro redshift__drop_view(relation) -%} + drop view if exists {{ relation }} cascade +{%- endmacro -%} diff --git a/dbt/include/redshift/macros/relations/view/rename.sql b/dbt/include/redshift/macros/relations/view/rename.sql new file mode 100644 index 000000000..0c6cdcdfa --- /dev/null +++ b/dbt/include/redshift/macros/relations/view/rename.sql @@ -0,0 +1,3 @@ +{% macro redshift__get_rename_view_sql(relation, new_name) %} + alter view {{ relation }} rename to {{ new_name }} +{% endmacro %} diff --git a/dbt/include/redshift/macros/relations/view/replace.sql b/dbt/include/redshift/macros/relations/view/replace.sql new file mode 100644 index 000000000..25a9d8b38 --- /dev/null +++ b/dbt/include/redshift/macros/relations/view/replace.sql @@ -0,0 +1,18 @@ +{% macro redshift__get_replace_view_sql(relation, sql) -%} + + {%- set binding = config.get('bind', default=True) -%} + + {% set bind_qualifier = '' if binding else 'with no schema binding' %} + {%- set sql_header = config.get('sql_header', none) -%} + + {{ sql_header if sql_header is not none }} + + create or replace view {{ relation }} + {%- set contract_config = config.get('contract') -%} + {%- if contract_config.enforced -%} + {{ get_assert_columns_equivalent(sql) }} + {%- endif %} as ( + {{ sql }} + ) {{ bind_qualifier }}; + +{%- endmacro %} diff --git a/tests/functional/adapter/materialized_view_tests/test_materialized_views.py b/tests/functional/adapter/materialized_view_tests/test_materialized_views.py index be5414122..e61737036 100644 --- a/tests/functional/adapter/materialized_view_tests/test_materialized_views.py +++ b/tests/functional/adapter/materialized_view_tests/test_materialized_views.py @@ -70,18 +70,6 @@ def test_materialized_view_create_idempotent(self, project, my_materialized_view ) assert self.query_relation_type(project, my_materialized_view) == "materialized_view" - @pytest.mark.skip( - "The current implementation does not support overwriting materialized views with tables." - ) - def test_table_replaces_materialized_view(self, project, my_materialized_view): - super().test_table_replaces_materialized_view(project, my_materialized_view) - - @pytest.mark.skip( - "The current implementation does not support overwriting materialized views with views." - ) - def test_view_replaces_materialized_view(self, project, my_materialized_view): - super().test_view_replaces_materialized_view(project, my_materialized_view) - class RedshiftMaterializedViewChanges(MaterializedViewChanges): @pytest.fixture(scope="class", autouse=True) From b84691d3fdd247e5be31d28cbd01554aabbc55fe Mon Sep 17 00:00:00 2001 From: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Thu, 14 Sep 2023 17:22:02 -0700 Subject: [PATCH 07/15] allow for bool/str input to backup/autorefresh when configuring materialized views (#604) * allow for bool/str input to backup/autorefresh when configuring materialized views * add assert to backup test * add changie --- .../Under the Hood-20230914-135547.yaml | 6 ++ .../relation_configs/materialized_view.py | 22 +++----- dbt/adapters/redshift/utility.py | 25 +++++++++ .../test_materialized_views.py | 36 +++++++++++- .../test_materialized_view.py | 55 +++++++++++++++++++ 5 files changed, 126 insertions(+), 18 deletions(-) create mode 100644 .changes/unreleased/Under the Hood-20230914-135547.yaml create mode 100644 dbt/adapters/redshift/utility.py create mode 100644 tests/unit/relation_configs/test_materialized_view.py diff --git a/.changes/unreleased/Under the Hood-20230914-135547.yaml b/.changes/unreleased/Under the Hood-20230914-135547.yaml new file mode 100644 index 000000000..a910ae660 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20230914-135547.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: allow for bool/str input to backup/autorefresh when configuring materialized + views +time: 2023-09-14T13:55:47.951848-07:00 +custom: + Author: colin-rogers-dbt diff --git a/dbt/adapters/redshift/relation_configs/materialized_view.py b/dbt/adapters/redshift/relation_configs/materialized_view.py index e69469476..1c6fb229a 100644 --- a/dbt/adapters/redshift/relation_configs/materialized_view.py +++ b/dbt/adapters/redshift/relation_configs/materialized_view.py @@ -23,6 +23,7 @@ RedshiftSortConfig, RedshiftSortConfigChange, ) +from dbt.adapters.redshift.utility import evaluate_bool @dataclass(frozen=True, eq=True, unsafe_hash=True) @@ -122,25 +123,16 @@ def parse_model_node(cls, model_node: ModelNode) -> dict: "mv_name": model_node.identifier, "schema_name": model_node.schema, "database_name": model_node.database, - "backup": model_node.config.extra.get("backup"), } + # backup/autorefresh can be bools or strings + backup_value = model_node.config.extra.get("backup") + if backup_value is not None: + config_dict["backup"] = evaluate_bool(backup_value) + autorefresh_value = model_node.config.extra.get("auto_refresh") if autorefresh_value is not None: - if isinstance(autorefresh_value, bool): - config_dict["autorefresh"] = autorefresh_value - elif isinstance(autorefresh_value, str): - lower_autorefresh = autorefresh_value.lower() - if lower_autorefresh == "true": - config_dict["autorefresh"] = True - elif lower_autorefresh == "false": - config_dict["autorefresh"] = False - else: - raise ValueError( - "Invalid autorefresh representation. Please use accepted value ex.( True, 'true', 'True')" - ) - else: - raise TypeError("Invalid autorefresh value: expecting boolean or str.") + config_dict["autorefresh"] = evaluate_bool(autorefresh_value) if query := model_node.compiled_code: config_dict.update({"query": query.strip()}) diff --git a/dbt/adapters/redshift/utility.py b/dbt/adapters/redshift/utility.py new file mode 100644 index 000000000..64f5e9cd8 --- /dev/null +++ b/dbt/adapters/redshift/utility.py @@ -0,0 +1,25 @@ +from typing import Union + + +def evaluate_bool_str(value: str) -> bool: + value = value.strip().lower() + if value == "true": + return True + elif value == "false": + return False + else: + raise ValueError(f"Invalid boolean string value: {value}") + + +def evaluate_bool(value: Union[str, bool]) -> bool: + if not value: + return False + if isinstance(value, bool): + return value + elif isinstance(value, str): + return evaluate_bool_str(value) + else: + raise TypeError( + f"Invalid type for boolean evaluation, " + f"expecting boolean or str, recieved: {type(value)}" + ) diff --git a/tests/functional/adapter/materialized_view_tests/test_materialized_views.py b/tests/functional/adapter/materialized_view_tests/test_materialized_views.py index e61737036..cf7d433c7 100644 --- a/tests/functional/adapter/materialized_view_tests/test_materialized_views.py +++ b/tests/functional/adapter/materialized_view_tests/test_materialized_views.py @@ -11,8 +11,13 @@ MaterializedViewChangesContinueMixin, MaterializedViewChangesFailMixin, ) -from dbt.tests.adapter.materialized_view.files import MY_TABLE, MY_VIEW -from dbt.tests.util import assert_message_in_logs, get_model_file, set_model_file +from dbt.tests.adapter.materialized_view.files import MY_TABLE, MY_VIEW, MY_SEED +from dbt.tests.util import ( + assert_message_in_logs, + get_model_file, + set_model_file, + run_dbt, +) from tests.functional.adapter.materialized_view_tests.utils import ( query_autorefresh, @@ -22,7 +27,6 @@ run_dbt_and_capture_with_retries_redshift_mv, ) - MY_MATERIALIZED_VIEW = """ {{ config( materialized='materialized_view', @@ -233,3 +237,29 @@ class TestRedshiftMaterializedViewChangesFail( ): # Note: using retries doesn't work when we expect `dbt_run` to fail pass + + +NO_BACKUP_MATERIALIZED_VIEW = """ +{{ config( + materialized='materialized_view', + backup=False +) }} +select * from {{ ref('my_seed') }} +""" + + +class TestRedshiftMaterializedViewWithBackupConfig: + @pytest.fixture(scope="class", autouse=True) + def models(self): + yield { + "my_materialized_view.sql": NO_BACKUP_MATERIALIZED_VIEW, + } + + @pytest.fixture(scope="class", autouse=True) + def seeds(self): + return {"my_seed.csv": MY_SEED} + + def test_running_mv_with_backup_false_succeeds(self, project): + run_dbt(["seed"]) + result = run_dbt(["run"]) + assert result[0].node.config_call_dict["backup"] is False diff --git a/tests/unit/relation_configs/test_materialized_view.py b/tests/unit/relation_configs/test_materialized_view.py new file mode 100644 index 000000000..42a3223d0 --- /dev/null +++ b/tests/unit/relation_configs/test_materialized_view.py @@ -0,0 +1,55 @@ +from unittest.mock import Mock + +import pytest + +from dbt.adapters.redshift.relation_configs import RedshiftMaterializedViewConfig + + +@pytest.mark.parametrize("bool_value", [True, False, "True", "False", "true", "false"]) +def test_redshift_materialized_view_config_handles_all_valid_bools(bool_value): + config = RedshiftMaterializedViewConfig( + database_name="somedb", + schema_name="public", + mv_name="someview", + query="select * from sometable", + ) + model_node = Mock() + model_node.config.extra.get = ( + lambda x, y=None: bool_value if x in ["auto_refresh", "backup"] else "someDistValue" + ) + config_dict = config.parse_model_node(model_node) + assert isinstance(config_dict["autorefresh"], bool) + assert isinstance(config_dict["backup"], bool) + + +@pytest.mark.parametrize("bool_value", [1]) +def test_redshift_materialized_view_config_throws_expected_exception_with_invalid_types( + bool_value, +): + config = RedshiftMaterializedViewConfig( + database_name="somedb", + schema_name="public", + mv_name="someview", + query="select * from sometable", + ) + model_node = Mock() + model_node.config.extra.get = ( + lambda x, y=None: bool_value if x in ["auto_refresh", "backup"] else "someDistValue" + ) + with pytest.raises(TypeError): + config.parse_model_node(model_node) + + +def test_redshift_materialized_view_config_throws_expected_exception_with_invalid_str(): + config = RedshiftMaterializedViewConfig( + database_name="somedb", + schema_name="public", + mv_name="someview", + query="select * from sometable", + ) + model_node = Mock() + model_node.config.extra.get = ( + lambda x, y=None: "notABool" if x in ["auto_refresh", "backup"] else "someDistValue" + ) + with pytest.raises(ValueError): + config.parse_model_node(model_node) From 2b5c446dd0672341a6e97cbe9fc9e9dd4a591648 Mon Sep 17 00:00:00 2001 From: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Fri, 15 Sep 2023 11:03:11 -0700 Subject: [PATCH 08/15] Update Fixes-20230912-133327.yaml (#607) --- .changes/unreleased/Fixes-20230912-133327.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.changes/unreleased/Fixes-20230912-133327.yaml b/.changes/unreleased/Fixes-20230912-133327.yaml index 14528cad6..e5aa98d47 100644 --- a/.changes/unreleased/Fixes-20230912-133327.yaml +++ b/.changes/unreleased/Fixes-20230912-133327.yaml @@ -3,3 +3,4 @@ body: use get_replace_sql in redshift__get_alter_materialized_view_as_sql, avoid time: 2023-09-12T13:33:27.451042-07:00 custom: Author: colin-rogers-dbt + Issue: "606" From 893e58eac7f8a21f581d064b3fc962e6054152f0 Mon Sep 17 00:00:00 2001 From: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Fri, 15 Sep 2023 11:28:33 -0700 Subject: [PATCH 09/15] Update Under the Hood-20230914-135547.yaml (#608) --- .changes/unreleased/Under the Hood-20230914-135547.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.changes/unreleased/Under the Hood-20230914-135547.yaml b/.changes/unreleased/Under the Hood-20230914-135547.yaml index a910ae660..3bb8e3c8a 100644 --- a/.changes/unreleased/Under the Hood-20230914-135547.yaml +++ b/.changes/unreleased/Under the Hood-20230914-135547.yaml @@ -4,3 +4,4 @@ body: allow for bool/str input to backup/autorefresh when configuring materializ time: 2023-09-14T13:55:47.951848-07:00 custom: Author: colin-rogers-dbt + Issue: "606" From be5b39e384eab6ca0dc866423ba639f9512a10be Mon Sep 17 00:00:00 2001 From: Github Build Bot Date: Fri, 15 Sep 2023 18:33:16 +0000 Subject: [PATCH 10/15] Bumping version to 1.7.0b2 and generate changelog --- .bumpversion.cfg | 2 +- .changes/1.7.0-b2.md | 24 ++++++++++++++++ .../Dependencies-20230816-195425.yaml | 0 .../Dependencies-20230821-192842.yaml | 0 .../Dependencies-20230901-230919.yaml | 0 .../Dependencies-20230904-193307.yaml | 0 .../Dependencies-20230912-002353.yaml | 0 .../Fixes-20230815-151655.yaml | 0 .../Fixes-20230830-164611.yaml | 0 .../Fixes-20230908-113019.yaml | 0 .../Fixes-20230912-133327.yaml | 0 .../Under the Hood-20230829-113206.yaml | 0 .../Under the Hood-20230914-135547.yaml | 0 CHANGELOG.md | 28 ++++++++++++++++++- dbt/adapters/redshift/__version__.py | 2 +- 15 files changed, 53 insertions(+), 3 deletions(-) create mode 100644 .changes/1.7.0-b2.md rename .changes/{unreleased => 1.7.0}/Dependencies-20230816-195425.yaml (100%) rename .changes/{unreleased => 1.7.0}/Dependencies-20230821-192842.yaml (100%) rename .changes/{unreleased => 1.7.0}/Dependencies-20230901-230919.yaml (100%) rename .changes/{unreleased => 1.7.0}/Dependencies-20230904-193307.yaml (100%) rename .changes/{unreleased => 1.7.0}/Dependencies-20230912-002353.yaml (100%) rename .changes/{unreleased => 1.7.0}/Fixes-20230815-151655.yaml (100%) rename .changes/{unreleased => 1.7.0}/Fixes-20230830-164611.yaml (100%) rename .changes/{unreleased => 1.7.0}/Fixes-20230908-113019.yaml (100%) rename .changes/{unreleased => 1.7.0}/Fixes-20230912-133327.yaml (100%) rename .changes/{unreleased => 1.7.0}/Under the Hood-20230829-113206.yaml (100%) rename .changes/{unreleased => 1.7.0}/Under the Hood-20230914-135547.yaml (100%) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index b3daceb77..56940f15e 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.7.0b1 +current_version = 1.7.0b2 parse = (?P[\d]+) # major version number \.(?P[\d]+) # minor version number \.(?P[\d]+) # patch version number diff --git a/.changes/1.7.0-b2.md b/.changes/1.7.0-b2.md new file mode 100644 index 000000000..c7dd145a1 --- /dev/null +++ b/.changes/1.7.0-b2.md @@ -0,0 +1,24 @@ +## dbt-redshift 1.7.0-b2 - September 15, 2023 + +### Features + +- use get_replace_sql in redshift__get_alter_materialized_view_as_sql, avoid renaming materialized views with custom table.sql and view.sql ([#606](https://github.com/dbt-labs/dbt-redshift/issues/606)) + +### Fixes + +- allow auto_refresh to take in boolean and str values for materialized views ([#560](https://github.com/dbt-labs/dbt-redshift/issues/560)) +- Fix to support removal of hologram, remove unused IAMDuration encoder ([#591](https://github.com/dbt-labs/dbt-redshift/issues/591)) +- changes expected value types to AnyInteger to take into account changes in core ([#597](https://github.com/dbt-labs/dbt-redshift/issues/597)) + +### Under the Hood + +- Restructure macro files to be more granular; organize macros by relation and type (create, drop, etc.) ([#590](https://github.com/dbt-labs/dbt-redshift/issues/590)) +- allow for bool/str input to backup/autorefresh when configuring materialized views ([#606](https://github.com/dbt-labs/dbt-redshift/issues/606)) + +### Dependencies + +- Bump mypy from 1.5.0 to 1.5.1 ([#585](https://github.com/dbt-labs/dbt-redshift/pull/585)) +- Update tox requirement from ~=4.9 to ~=4.10 ([#588](https://github.com/dbt-labs/dbt-redshift/pull/588)) +- Update tox requirement from ~=4.10 to ~=4.11 ([#594](https://github.com/dbt-labs/dbt-redshift/pull/594)) +- Update pre-commit requirement from ~=3.3 to ~=3.4 ([#596](https://github.com/dbt-labs/dbt-redshift/pull/596)) +- Update black requirement from ~=23.7 to ~=23.9 ([#600](https://github.com/dbt-labs/dbt-redshift/pull/600)) diff --git a/.changes/unreleased/Dependencies-20230816-195425.yaml b/.changes/1.7.0/Dependencies-20230816-195425.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20230816-195425.yaml rename to .changes/1.7.0/Dependencies-20230816-195425.yaml diff --git a/.changes/unreleased/Dependencies-20230821-192842.yaml b/.changes/1.7.0/Dependencies-20230821-192842.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20230821-192842.yaml rename to .changes/1.7.0/Dependencies-20230821-192842.yaml diff --git a/.changes/unreleased/Dependencies-20230901-230919.yaml b/.changes/1.7.0/Dependencies-20230901-230919.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20230901-230919.yaml rename to .changes/1.7.0/Dependencies-20230901-230919.yaml diff --git a/.changes/unreleased/Dependencies-20230904-193307.yaml b/.changes/1.7.0/Dependencies-20230904-193307.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20230904-193307.yaml rename to .changes/1.7.0/Dependencies-20230904-193307.yaml diff --git a/.changes/unreleased/Dependencies-20230912-002353.yaml b/.changes/1.7.0/Dependencies-20230912-002353.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20230912-002353.yaml rename to .changes/1.7.0/Dependencies-20230912-002353.yaml diff --git a/.changes/unreleased/Fixes-20230815-151655.yaml b/.changes/1.7.0/Fixes-20230815-151655.yaml similarity index 100% rename from .changes/unreleased/Fixes-20230815-151655.yaml rename to .changes/1.7.0/Fixes-20230815-151655.yaml diff --git a/.changes/unreleased/Fixes-20230830-164611.yaml b/.changes/1.7.0/Fixes-20230830-164611.yaml similarity index 100% rename from .changes/unreleased/Fixes-20230830-164611.yaml rename to .changes/1.7.0/Fixes-20230830-164611.yaml diff --git a/.changes/unreleased/Fixes-20230908-113019.yaml b/.changes/1.7.0/Fixes-20230908-113019.yaml similarity index 100% rename from .changes/unreleased/Fixes-20230908-113019.yaml rename to .changes/1.7.0/Fixes-20230908-113019.yaml diff --git a/.changes/unreleased/Fixes-20230912-133327.yaml b/.changes/1.7.0/Fixes-20230912-133327.yaml similarity index 100% rename from .changes/unreleased/Fixes-20230912-133327.yaml rename to .changes/1.7.0/Fixes-20230912-133327.yaml diff --git a/.changes/unreleased/Under the Hood-20230829-113206.yaml b/.changes/1.7.0/Under the Hood-20230829-113206.yaml similarity index 100% rename from .changes/unreleased/Under the Hood-20230829-113206.yaml rename to .changes/1.7.0/Under the Hood-20230829-113206.yaml diff --git a/.changes/unreleased/Under the Hood-20230914-135547.yaml b/.changes/1.7.0/Under the Hood-20230914-135547.yaml similarity index 100% rename from .changes/unreleased/Under the Hood-20230914-135547.yaml rename to .changes/1.7.0/Under the Hood-20230914-135547.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 44a71a8bd..2bb546acc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,33 @@ - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version. - Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-redshift/blob/main/CONTRIBUTING.md#adding-changelog-entry) +## dbt-redshift 1.7.0-b2 - September 15, 2023 + +### Features + +- use get_replace_sql in redshift__get_alter_materialized_view_as_sql, avoid renaming materialized views with custom table.sql and view.sql ([#606](https://github.com/dbt-labs/dbt-redshift/issues/606)) + +### Fixes + +- allow auto_refresh to take in boolean and str values for materialized views ([#560](https://github.com/dbt-labs/dbt-redshift/issues/560)) +- Fix to support removal of hologram, remove unused IAMDuration encoder ([#591](https://github.com/dbt-labs/dbt-redshift/issues/591)) +- changes expected value types to AnyInteger to take into account changes in core ([#597](https://github.com/dbt-labs/dbt-redshift/issues/597)) + +### Under the Hood + +- Restructure macro files to be more granular; organize macros by relation and type (create, drop, etc.) ([#590](https://github.com/dbt-labs/dbt-redshift/issues/590)) +- allow for bool/str input to backup/autorefresh when configuring materialized views ([#606](https://github.com/dbt-labs/dbt-redshift/issues/606)) + +### Dependencies + +- Bump mypy from 1.5.0 to 1.5.1 ([#585](https://github.com/dbt-labs/dbt-redshift/pull/585)) +- Update tox requirement from ~=4.9 to ~=4.10 ([#588](https://github.com/dbt-labs/dbt-redshift/pull/588)) +- Update tox requirement from ~=4.10 to ~=4.11 ([#594](https://github.com/dbt-labs/dbt-redshift/pull/594)) +- Update pre-commit requirement from ~=3.3 to ~=3.4 ([#596](https://github.com/dbt-labs/dbt-redshift/pull/596)) +- Update black requirement from ~=23.7 to ~=23.9 ([#600](https://github.com/dbt-labs/dbt-redshift/pull/600)) + + + ## dbt-redshift 1.7.0-b1 - August 17, 2023 ### Features @@ -38,7 +65,6 @@ ### Contributors - [@WillAyd](https://github.com/WillAyd) ([#255](https://github.com/dbt-labs/dbt-redshift/issues/255)) - ## Previous Releases For information on prior major and minor releases, see their changelogs: - [1.6](https://github.com/dbt-labs/dbt-redshift/blob/1.6.latest/CHANGELOG.md) diff --git a/dbt/adapters/redshift/__version__.py b/dbt/adapters/redshift/__version__.py index 48607b01f..3f5d3c0b7 100644 --- a/dbt/adapters/redshift/__version__.py +++ b/dbt/adapters/redshift/__version__.py @@ -1 +1 @@ -version = "1.7.0b1" +version = "1.7.0b2" From 9385b49c15373b4843943780c1f20b7d587ca360 Mon Sep 17 00:00:00 2001 From: Wen Date: Mon, 25 Sep 2023 14:10:09 -0400 Subject: [PATCH 11/15] Avoid nested loop in query planner execution (#612) * Adjust not equals application order - Move relation name not equals from outer SELECT `where` clause to dependency CTE - Seems to more reliably push query planner to avoid a nested loop - Should preserve original intent, i.e. to exclude self-references from final result - Execution plan suggests this reduces the row estimate for the outer SELECT - Real-world testing on internal company cluster under conditions where nested loop is used for original, shows that the new approach continues to perform and behave as expected (no nested loop) * Add changie entry --- .changes/unreleased/Fixes-20230923-091155.yaml | 6 ++++++ dbt/include/redshift/macros/relations.sql | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Fixes-20230923-091155.yaml diff --git a/.changes/unreleased/Fixes-20230923-091155.yaml b/.changes/unreleased/Fixes-20230923-091155.yaml new file mode 100644 index 000000000..055d4dc7e --- /dev/null +++ b/.changes/unreleased/Fixes-20230923-091155.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: avoid nested loop in query planner execution of redshift__get_relations +time: 2023-09-23T09:11:55.2152725-04:00 +custom: + Author: slin30 + Issue: "609" diff --git a/dbt/include/redshift/macros/relations.sql b/dbt/include/redshift/macros/relations.sql index 28c6bc377..6d83c36b9 100644 --- a/dbt/include/redshift/macros/relations.sql +++ b/dbt/include/redshift/macros/relations.sql @@ -24,6 +24,7 @@ with from pg_depend left join pg_rewrite on pg_depend.objid = pg_rewrite.oid + where coalesce(pg_rewrite.ev_class, pg_depend.objid) != pg_depend.refobjid ) select distinct @@ -36,7 +37,6 @@ join relation ref on dependency.ref_relation_id = ref.relation_id join relation dep on dependency.dep_relation_id = dep.relation_id -where ref.relation_name != dep.relation_name {%- endcall -%} From 7f7330969e2cd99d4128a55091f4b6bce6cffeaf Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 26 Sep 2023 11:49:22 -0500 Subject: [PATCH 12/15] automate repo cleanup (#618) --- .github/workflows/repository-cleanup.yml | 30 ++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/repository-cleanup.yml diff --git a/.github/workflows/repository-cleanup.yml b/.github/workflows/repository-cleanup.yml new file mode 100644 index 000000000..c1d780281 --- /dev/null +++ b/.github/workflows/repository-cleanup.yml @@ -0,0 +1,30 @@ +# **what?** +# Cleanup branches left over from automation and testing. Also cleanup +# draft releases from release testing. + +# **why?** +# The automations are leaving behind branches and releases that clutter +# the repository. Sometimes we need them to debug processes so we don't +# want them immediately deleted. Running on Saturday to avoid running +# at the same time as an actual release to prevent breaking a release +# mid-release. + +# **when?** +# Mainly on a schedule of 12:00 Saturday. +# Manual trigger can also run on demand + +name: Repository Cleanup + +on: + schedule: + - cron: '0 12 * * SAT' # At 12:00 on Saturday - details in `why` above + + workflow_dispatch: # for manual triggering + +permissions: + contents: write + +jobs: + cleanup-repo: + uses: dbt-labs/actions/.github/workflows/repository-cleanup.yml@main + secrets: inherit From 16d86aa9c570479f4a5a59ee10a6d64a38fd530d Mon Sep 17 00:00:00 2001 From: Quigley Malcolm Date: Tue, 26 Sep 2023 16:44:09 -0700 Subject: [PATCH 13/15] Add tests for `date_spine` macro, and sub macros (#617) Co-authored-by: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> --- tests/functional/adapter/utils/test_utils.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/functional/adapter/utils/test_utils.py b/tests/functional/adapter/utils/test_utils.py index 266103fbc..61a706f4d 100644 --- a/tests/functional/adapter/utils/test_utils.py +++ b/tests/functional/adapter/utils/test_utils.py @@ -8,9 +8,13 @@ from dbt.tests.adapter.utils.test_current_timestamp import BaseCurrentTimestampNaive from dbt.tests.adapter.utils.test_dateadd import BaseDateAdd from dbt.tests.adapter.utils.test_datediff import BaseDateDiff +from dbt.tests.adapter.utils.test_date_spine import BaseDateSpine from dbt.tests.adapter.utils.test_date_trunc import BaseDateTrunc from dbt.tests.adapter.utils.test_escape_single_quotes import BaseEscapeSingleQuotesQuote from dbt.tests.adapter.utils.test_except import BaseExcept +from dbt.tests.adapter.utils.test_generate_series import BaseGenerateSeries +from dbt.tests.adapter.utils.test_get_intervals_between import BaseGetIntervalsBetween +from dbt.tests.adapter.utils.test_get_powers_of_two import BaseGetPowersOfTwo from dbt.tests.adapter.utils.test_hash import BaseHash from dbt.tests.adapter.utils.test_intersect import BaseIntersect from dbt.tests.adapter.utils.test_last_day import BaseLastDay @@ -65,6 +69,10 @@ class TestDateDiff(BaseDateDiff): pass +class TestDateSpine(BaseDateSpine): + pass + + class TestDateTrunc(BaseDateTrunc): pass @@ -77,6 +85,18 @@ class TestExcept(BaseExcept): pass +class TestGenerateSeries(BaseGenerateSeries): + pass + + +class TestGetIntervalsBeteween(BaseGetIntervalsBetween): + pass + + +class TestGetPowersOfTwo(BaseGetPowersOfTwo): + pass + + class TestHash(BaseHash): pass From ca3c7b4f9e6b2df0ae6d2334edd58b6bb08a578e Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Thu, 28 Sep 2023 19:39:11 +0100 Subject: [PATCH 14/15] add dbt show tests (#611) * add dbt show tests * changelog entry * repoint to core main * change to a WITH statement in model definition for dbt_show test model * reuse core fixture for dbt show sql header test * undo dev-requirements changes --------- Co-authored-by: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> Co-authored-by: Matthew McKnight <91097623+McKnight-42@users.noreply.github.com> Co-authored-by: Matthew McKnight --- .changes/unreleased/Under the Hood-20230925-150132.yaml | 6 ++++++ tests/functional/adapter/dbt_show/test_dbt_show.py | 9 +++++++++ 2 files changed, 15 insertions(+) create mode 100644 .changes/unreleased/Under the Hood-20230925-150132.yaml create mode 100644 tests/functional/adapter/dbt_show/test_dbt_show.py diff --git a/.changes/unreleased/Under the Hood-20230925-150132.yaml b/.changes/unreleased/Under the Hood-20230925-150132.yaml new file mode 100644 index 000000000..c61a28d56 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20230925-150132.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Add tests for inlined limit + sql_header in dbt show query +time: 2023-09-25T15:01:32.025325+01:00 +custom: + Author: michelleark + Issue: "616" diff --git a/tests/functional/adapter/dbt_show/test_dbt_show.py b/tests/functional/adapter/dbt_show/test_dbt_show.py new file mode 100644 index 000000000..808a7733c --- /dev/null +++ b/tests/functional/adapter/dbt_show/test_dbt_show.py @@ -0,0 +1,9 @@ +from dbt.tests.adapter.dbt_show.test_dbt_show import BaseShowSqlHeader, BaseShowLimit + + +class TestRedshiftShowLimit(BaseShowLimit): + pass + + +class TestRedshiftShowSqlHeader(BaseShowSqlHeader): + pass From 85d37209921042e6af0e18bc11dd6017ba331622 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Oct 2023 11:38:39 -0500 Subject: [PATCH 15/15] Update ddtrace requirement from ~=1.18 to ~=1.19 (#610) * Update ddtrace requirement from ~=1.18 to ~=1.19 Updates the requirements on [ddtrace](https://github.com/DataDog/dd-trace-py) to permit the latest version. - [Release notes](https://github.com/DataDog/dd-trace-py/releases) - [Changelog](https://github.com/DataDog/dd-trace-py/blob/2.x/CHANGELOG.md) - [Commits](https://github.com/DataDog/dd-trace-py/compare/v1.18.0...v1.19.0) --- updated-dependencies: - dependency-name: ddtrace dependency-type: direct:development ... Signed-off-by: dependabot[bot] * Add automated changelog yaml from template for bot PR --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Github Build Bot Co-authored-by: Matthew McKnight <91097623+McKnight-42@users.noreply.github.com> --- .changes/unreleased/Dependencies-20230918-190833.yaml | 6 ++++++ dev-requirements.txt | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Dependencies-20230918-190833.yaml diff --git a/.changes/unreleased/Dependencies-20230918-190833.yaml b/.changes/unreleased/Dependencies-20230918-190833.yaml new file mode 100644 index 000000000..f784102ca --- /dev/null +++ b/.changes/unreleased/Dependencies-20230918-190833.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Update ddtrace requirement from ~=1.18 to ~=1.19" +time: 2023-09-18T19:08:33.00000Z +custom: + Author: dependabot[bot] + PR: 610 diff --git a/dev-requirements.txt b/dev-requirements.txt index 96f252ebe..4b3b09cda 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -9,7 +9,7 @@ git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-postgres&subdirectory=plugi black~=23.9 bumpversion~=0.6.0 click~=8.1 -ddtrace~=1.18 +ddtrace~=1.19 flake8~=6.1 flaky~=3.7 freezegun~=1.2