From c7d7495a2023a66457a61d69b24ad2e921f842d7 Mon Sep 17 00:00:00 2001 From: Dan Bryan Date: Wed, 24 Nov 2021 04:56:00 -0600 Subject: [PATCH] Add Redshift parameter to create tables with backup option specified (#42) * Update impl and adapters to support backup parameter * Add test files * Add test files * Add PR link to Changelog * Add EOF newlines * Debug and split test into two separate cases * Add contributor info Co-authored-by: Jeremy Cohen --- CHANGELOG.md | 6 ++ dbt/adapters/redshift/impl.py | 1 + dbt/include/redshift/macros/adapters.sql | 2 + .../models/model_backup_false.sql | 7 ++ .../models/model_backup_true.sql | 7 ++ .../models/model_backup_true_view.sql | 7 ++ .../models/model_backup_undefined.sql | 7 ++ .../test_backup_table_option.py | 90 +++++++++++++++++++ 8 files changed, 127 insertions(+) create mode 100644 tests/integration/backup_table_tests/models/model_backup_false.sql create mode 100644 tests/integration/backup_table_tests/models/model_backup_true.sql create mode 100644 tests/integration/backup_table_tests/models/model_backup_true_view.sql create mode 100644 tests/integration/backup_table_tests/models/model_backup_undefined.sql create mode 100644 tests/integration/backup_table_tests/test_backup_table_option.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 39547c3e1..9fc9e2e40 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ ## dbt-redshift 1.0.0 (Release TBD) +### Under the hood +- Add optional Redshift parameter to create tables with BACKUP NO set, to exclude them from snapshots. ([#18](https://github.com/dbt-labs/dbt-redshift/issues/18), [#42](https://github.com/dbt-labs/dbt-redshift/pull/42)) + +### Contributors +- [@dlb8685](https://github.com/dlb8685) ([#42](https://github.com/dbt-labs/dbt-redshift/pull/42)) + ## dbt-redshift 1.0.0rc1 (November 10, 2021) ### Under the hood diff --git a/dbt/adapters/redshift/impl.py b/dbt/adapters/redshift/impl.py index 46d36d8b6..0ee975973 100644 --- a/dbt/adapters/redshift/impl.py +++ b/dbt/adapters/redshift/impl.py @@ -19,6 +19,7 @@ class RedshiftConfig(AdapterConfig): dist: Optional[str] = None sort: Optional[str] = None bind: Optional[bool] = None + backup: Optional[bool] = True class RedshiftAdapter(PostgresAdapter, SQLAdapter): diff --git a/dbt/include/redshift/macros/adapters.sql b/dbt/include/redshift/macros/adapters.sql index 95f92811c..e81e823df 100644 --- a/dbt/include/redshift/macros/adapters.sql +++ b/dbt/include/redshift/macros/adapters.sql @@ -39,6 +39,7 @@ 'sort', validator=validation.any[list, basestring]) -%} {%- set sql_header = config.get('sql_header', none) -%} + {%- set backup = config.get('backup') -%} {{ sql_header if sql_header is not none }} @@ -46,6 +47,7 @@ {{ relation.include(database=(not temporary), schema=(not temporary)) }} {{ dist(_dist) }} {{ sort(_sort_type, _sort) }} + {% if backup == false -%}backup no{%- endif %} as ( {{ sql }} ); diff --git a/tests/integration/backup_table_tests/models/model_backup_false.sql b/tests/integration/backup_table_tests/models/model_backup_false.sql new file mode 100644 index 000000000..67900ac06 --- /dev/null +++ b/tests/integration/backup_table_tests/models/model_backup_false.sql @@ -0,0 +1,7 @@ +{{ + config( + materialized='table', backup=False + ) +}} + +select 1 diff --git a/tests/integration/backup_table_tests/models/model_backup_true.sql b/tests/integration/backup_table_tests/models/model_backup_true.sql new file mode 100644 index 000000000..882522da9 --- /dev/null +++ b/tests/integration/backup_table_tests/models/model_backup_true.sql @@ -0,0 +1,7 @@ +{{ + config( + materialized='table', backup=True + ) +}} + +select 2 diff --git a/tests/integration/backup_table_tests/models/model_backup_true_view.sql b/tests/integration/backup_table_tests/models/model_backup_true_view.sql new file mode 100644 index 000000000..841070a0a --- /dev/null +++ b/tests/integration/backup_table_tests/models/model_backup_true_view.sql @@ -0,0 +1,7 @@ +{{ + config( + materialized='view', backup=True + ) +}} + +select 3 diff --git a/tests/integration/backup_table_tests/models/model_backup_undefined.sql b/tests/integration/backup_table_tests/models/model_backup_undefined.sql new file mode 100644 index 000000000..54468d510 --- /dev/null +++ b/tests/integration/backup_table_tests/models/model_backup_undefined.sql @@ -0,0 +1,7 @@ +{{ + config( + materialized='table' + ) +}} + +select 4 diff --git a/tests/integration/backup_table_tests/test_backup_table_option.py b/tests/integration/backup_table_tests/test_backup_table_option.py new file mode 100644 index 000000000..b31da3adf --- /dev/null +++ b/tests/integration/backup_table_tests/test_backup_table_option.py @@ -0,0 +1,90 @@ +import os + +from tests.integration.base import DBTIntegrationTest, use_profile + + +class TestBackupTableOption(DBTIntegrationTest): + @property + def schema(self): + return 'backup_table_tests' + + @staticmethod + def dir(path): + return os.path.normpath(path) + + @property + def models(self): + return self.dir("models") + + @property + def project_config(self): + return { + 'config-version': 2 + } + + def check_backup_param_template(self, test_table_name, backup_is_expected): + # Use raw DDL statement to confirm backup is set correctly on new table + with open('target/run/test/models/{}.sql'.format(test_table_name), 'r') as ddl_file: + ddl_statement = ddl_file.readlines() + self.assertEqual('backup no' not in ' '.join(ddl_statement).lower(), backup_is_expected) + + @use_profile('redshift') + def test__redshift_backup_table_option(self): + self.assertEqual(len(self.run_dbt()), 4) + + # model_backup_undefined should not contain a BACKUP NO parameter in the table DDL + self.check_backup_param_template('model_backup_undefined', True) + + # model_backup_true should not contain a BACKUP NO parameter in the table DDL + self.check_backup_param_template('model_backup_true', True) + + # model_backup_false should contain a BACKUP NO parameter in the table DDL + self.check_backup_param_template('model_backup_false', False) + + # Any view should not contain a BACKUP NO parameter, regardless of the specified config (create will fail) + self.check_backup_param_template('model_backup_true_view', True) + + +class TestBackupTableOptionProjectFalse(DBTIntegrationTest): + @property + def schema(self): + return 'backup_table_tests' + + @staticmethod + def dir(path): + return os.path.normpath(path) + + @property + def models(self): + return self.dir("models") + + @property + def project_config(self): + # Update project config to set backup to False. + # This should make the 'model_backup_undefined' switch to BACKUP NO + return { + 'config-version': 2, + 'models': {'backup': False} + } + + def check_backup_param_template(self, test_table_name, backup_is_expected): + # Use raw DDL statement to confirm backup is set correctly on new table + with open('target/run/test/models/{}.sql'.format(test_table_name), 'r') as ddl_file: + ddl_statement = ddl_file.readlines() + self.assertEqual('backup no' not in ' '.join(ddl_statement).lower(), backup_is_expected) + + @use_profile('redshift') + def test__redshift_backup_table_option_project_config_false(self): + self.assertEqual(len(self.run_dbt()), 4) + + # model_backup_undefined should contain a BACKUP NO parameter in the table DDL + self.check_backup_param_template('model_backup_undefined', False) + + # model_backup_true should not contain a BACKUP NO parameter in the table DDL + self.check_backup_param_template('model_backup_true', True) + + # model_backup_false should contain a BACKUP NO parameter in the table DDL + self.check_backup_param_template('model_backup_false', False) + + # Any view should not contain a BACKUP NO parameter, regardless of the specified config (create will fail) + self.check_backup_param_template('model_backup_true_view', True)