From 7c63a40c6948e0bd27d756e7adb9604b44347352 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Mon, 18 Nov 2024 15:51:51 -0500 Subject: [PATCH 1/6] Make `list_relations_without_caching` pagination configurable (#1235) * make list relations configurable * update iteration to page in the config settings * update the warning to recommend how to account for breaching the limit on list_relations --- .../unreleased/Features-20241107-170307.yaml | 7 + dbt/include/snowflake/macros/adapters.sql | 10 +- .../list_relations_tests/test_pagination.py | 247 +++++------------- 3 files changed, 85 insertions(+), 179 deletions(-) create mode 100644 .changes/unreleased/Features-20241107-170307.yaml diff --git a/.changes/unreleased/Features-20241107-170307.yaml b/.changes/unreleased/Features-20241107-170307.yaml new file mode 100644 index 000000000..1479c5805 --- /dev/null +++ b/.changes/unreleased/Features-20241107-170307.yaml @@ -0,0 +1,7 @@ +kind: Features +body: 'Allow configurable pagination on list_relations_without_caching to support + users with a large number of objects per schema' +time: 2024-11-07T17:03:07.826352-05:00 +custom: + Author: mikealfare + Issue: "1234" diff --git a/dbt/include/snowflake/macros/adapters.sql b/dbt/include/snowflake/macros/adapters.sql index 0ca756c6c..3c93d41ad 100644 --- a/dbt/include/snowflake/macros/adapters.sql +++ b/dbt/include/snowflake/macros/adapters.sql @@ -111,9 +111,10 @@ {%- if loop.index == max_iter -%} {%- set msg -%} - dbt will list a maximum of {{ max_total_results }} objects in schema {{ schema_relation }}. - Your schema exceeds this limit. Please contact support@getdbt.com for troubleshooting tips, - or review and reduce the number of objects contained. + dbt is currently configured to list a maximum of {{ max_total_results }} objects per schema. + {{ schema_relation }} exceeds this limit. If this is expected, you may configure this limit + by setting list_relations_per_page and list_relations_page_limit in your project flags. + It is recommended to start by increasing list_relations_page_limit to something more than the default of 10. {%- endset -%} {% do exceptions.raise_compiler_error(msg) %} @@ -135,6 +136,9 @@ {% endmacro %} {% macro snowflake__list_relations_without_caching(schema_relation, max_iter=10, max_results_per_iter=10000) %} + + {%- set max_results_per_iter = adapter.config.flags.get('list_relations_per_page', max_results_per_iter) -%} + {%- set max_iter = adapter.config.flags.get('list_relations_page_limit', max_iter) -%} {%- set max_total_results = max_results_per_iter * max_iter -%} {%- set sql -%} {% if schema_relation is string %} diff --git a/tests/functional/adapter/list_relations_tests/test_pagination.py b/tests/functional/adapter/list_relations_tests/test_pagination.py index 407f9c501..7dd382af5 100644 --- a/tests/functional/adapter/list_relations_tests/test_pagination.py +++ b/tests/functional/adapter/list_relations_tests/test_pagination.py @@ -1,34 +1,31 @@ import os + import pytest -import json -from dbt.tests.util import run_dbt, run_dbt_and_capture -from dbt.adapters.snowflake import SnowflakeRelation # Ensure this is the correct import path - -# Testing rationale: -# - snowflake SHOW TERSE OBJECTS command returns at max 10K objects in a single call -# - when dbt attempts to write into a schema with more than 10K objects, compilation will fail -# unless we paginate the result -# - however, testing this process is difficult at a full scale of 10K actual objects populated -# into a fresh testing schema -# - accordingly, we create a smaller set of views and test the looping iteration logic in -# smaller chunks - -NUM_VIEWS = 90 -NUM_DYNAMIC_TABLES = 10 -# the total number should be between the numbers referenced in the "passing" and "failing" macros below -# - MACROS__VALIDATE__SNOWFLAKE__LIST_RELATIONS_WITHOUT_CACHING (11 iter * 10 results per iter -> 110 objects) -# - MACROS__VALIDATE__SNOWFLAKE__LIST_RELATIONS_WITHOUT_CACHING_RAISE_ERROR (33 iter * 3 results per iter -> 99 objects) -NUM_EXPECTED_RELATIONS = 1 + NUM_VIEWS + NUM_DYNAMIC_TABLES - -TABLE_BASE_SQL = """ -{{ config(materialized='table') }} +from dbt_common.exceptions import CompilationError +from dbt.tests.util import run_dbt + +""" +Testing rationale: +- snowflake SHOW TERSE OBJECTS command returns at max 10K objects in a single call +- when dbt attempts to write into a schema with more than 10K objects, compilation will fail + unless we paginate the result +- we default pagination to 10 pages, but users want to configure this + - we instead use that here to force failures by making it smaller +""" + + +TABLE = """ +{{ config(materialized='table') }} select 1 as id -""".lstrip() +""" + -VIEW_X_SQL = """ +VIEW = """ +{{ config(materialized='view') }} select id from {{ ref('my_model_base') }} -""".lstrip() +""" + DYNAMIC_TABLE = ( """ @@ -44,173 +41,71 @@ """ ) -MACROS__VALIDATE__SNOWFLAKE__LIST_RELATIONS_WITHOUT_CACHING = """ -{% macro validate_list_relations_without_caching(schema_relation) %} - {% set relation_list_result = snowflake__list_relations_without_caching(schema_relation, max_iter=11, max_results_per_iter=10) %} - {% set n_relations = relation_list_result | length %} - {{ log("n_relations: " ~ n_relations) }} -{% endmacro %} -""" - -MACROS__VALIDATE__SNOWFLAKE__LIST_RELATIONS_WITHOUT_CACHING_RAISE_ERROR = """ -{% macro validate_list_relations_without_caching_raise_error(schema_relation) %} - {{ snowflake__list_relations_without_caching(schema_relation, max_iter=33, max_results_per_iter=3) }} -{% endmacro %} -""" - - -def parse_json_logs(json_log_output): - parsed_logs = [] - for line in json_log_output.split("\n"): - try: - log = json.loads(line) - except ValueError: - continue - - parsed_logs.append(log) - - return parsed_logs +class BaseConfig: + VIEWS = 90 + DYNAMIC_TABLES = 10 -def find_result_in_parsed_logs(parsed_logs, result_name): - return next( - ( - item["data"]["msg"] - for item in parsed_logs - if result_name in item["data"].get("msg", "msg") - ), - False, - ) - - -def find_exc_info_in_parsed_logs(parsed_logs, exc_info_name): - return next( - ( - item["data"]["exc_info"] - for item in parsed_logs - if exc_info_name in item["data"].get("exc_info", "exc_info") - ), - False, - ) - - -class TestListRelationsWithoutCachingSingle: @pytest.fixture(scope="class") def models(self): - my_models = {"my_model_base.sql": TABLE_BASE_SQL} - for view in range(0, NUM_VIEWS): - my_models.update({f"my_model_{view}.sql": VIEW_X_SQL}) - for dynamic_table in range(0, NUM_DYNAMIC_TABLES): - my_models.update({f"my_dynamic_table_{dynamic_table}.sql": DYNAMIC_TABLE}) + my_models = {"my_model_base.sql": TABLE} + for view in range(0, self.VIEWS): + my_models[f"my_model_{view}.sql"] = VIEW + for dynamic_table in range(0, self.DYNAMIC_TABLES): + my_models[f"my_dynamic_table_{dynamic_table}.sql"] = DYNAMIC_TABLE return my_models - @pytest.fixture(scope="class") - def macros(self): - return { - "validate_list_relations_without_caching.sql": MACROS__VALIDATE__SNOWFLAKE__LIST_RELATIONS_WITHOUT_CACHING, - } + @pytest.fixture(scope="class", autouse=True) + def setup(self, project): + run_dbt(["run"]) - def test__snowflake__list_relations_without_caching_termination(self, project): - """ - validates that we do NOT trigger pagination logic snowflake__list_relations_without_caching - macro when there are fewer than max_results_per_iter relations in the target schema - """ - run_dbt(["run", "-s", "my_model_base"]) - - database = project.database - schemas = project.created_schemas - - for schema in schemas: - schema_relation = SnowflakeRelation.create(database=database, schema=schema) - kwargs = {"schema_relation": schema_relation.render()} - _, log_output = run_dbt_and_capture( - [ - "--debug", - "--log-format=json", - "run-operation", - "validate_list_relations_without_caching", - "--args", - str(kwargs), - ] + def test_list_relations(self, project): + kwargs = {"schema_relation": project.test_schema} + with project.adapter.connection_named("__test"): + relations = project.adapter.execute_macro( + "snowflake__list_relations_without_caching", kwargs=kwargs ) + assert len(relations) == self.VIEWS + self.DYNAMIC_TABLES + 1 - parsed_logs = parse_json_logs(log_output) - n_relations = find_result_in_parsed_logs(parsed_logs, "n_relations") - assert n_relations == "n_relations: 1" +class TestListRelationsWithoutCachingSmall(BaseConfig): + pass -class TestListRelationsWithoutCachingFull: - @pytest.fixture(scope="class") - def models(self): - my_models = {"my_model_base.sql": TABLE_BASE_SQL} - for view in range(0, NUM_VIEWS): - my_models.update({f"my_model_{view}.sql": VIEW_X_SQL}) - for dynamic_table in range(0, NUM_DYNAMIC_TABLES): - my_models.update({f"my_dynamic_table_{dynamic_table}.sql": DYNAMIC_TABLE}) - return my_models +class TestListRelationsWithoutCachingLarge(BaseConfig): @pytest.fixture(scope="class") - def macros(self): + def profiles_config_update(self): return { - "validate_list_relations_without_caching.sql": MACROS__VALIDATE__SNOWFLAKE__LIST_RELATIONS_WITHOUT_CACHING, - "validate_list_relations_without_caching_raise_error.sql": MACROS__VALIDATE__SNOWFLAKE__LIST_RELATIONS_WITHOUT_CACHING_RAISE_ERROR, + "flags": { + "list_relations_per_page": 10, + "list_relations_page_limit": 20, + } } - def test__snowflake__list_relations_without_caching(self, project): - """ - validates pagination logic in snowflake__list_relations_without_caching macro counts - the correct number of objects in the target schema when having to make multiple looped - calls of SHOW TERSE OBJECTS. - """ - # purpose of the first run is to create the replicated views in the target schema - run_dbt(["run"]) - database = project.database - schemas = project.created_schemas - - for schema in schemas: - schema_relation = SnowflakeRelation.create(database=database, schema=schema) - kwargs = {"schema_relation": schema_relation.render()} - _, log_output = run_dbt_and_capture( - [ - "--debug", - "--log-format=json", - "run-operation", - "validate_list_relations_without_caching", - "--args", - str(kwargs), - ] - ) - parsed_logs = parse_json_logs(log_output) - n_relations = find_result_in_parsed_logs(parsed_logs, "n_relations") +class TestListRelationsWithoutCachingTooLarge(BaseConfig): - assert n_relations == f"n_relations: {NUM_EXPECTED_RELATIONS}" - - def test__snowflake__list_relations_without_caching_raise_error(self, project): - """ - validates pagination logic terminates and raises a compilation error - when exceeding the limit of how many results to return. - """ - run_dbt(["run"]) + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "flags": { + "list_relations_per_page": 10, + "list_relations_page_limit": 5, + } + } - database = project.database - schemas = project.created_schemas - - for schema in schemas: - schema_relation = SnowflakeRelation.create(database=database, schema=schema) - - kwargs = {"schema_relation": schema_relation.render()} - _, log_output = run_dbt_and_capture( - [ - "--debug", - "--log-format=json", - "run-operation", - "validate_list_relations_without_caching_raise_error", - "--args", - str(kwargs), - ], - expect_pass=False, - ) - parsed_logs = parse_json_logs(log_output) - traceback = find_exc_info_in_parsed_logs(parsed_logs, "Traceback") - assert "dbt will list a maximum of 99 objects in schema " in traceback + def test_list_relations(self, project): + kwargs = {"schema_relation": project.test_schema} + with project.adapter.connection_named("__test"): + with pytest.raises(CompilationError) as error: + project.adapter.execute_macro( + "snowflake__list_relations_without_caching", kwargs=kwargs + ) + assert "list_relations_per_page" in error.value.msg + assert "list_relations_page_limit" in error.value.msg + + def test_on_run(self, project): + with pytest.raises(CompilationError) as error: + run_dbt(["run"]) + assert "list_relations_per_page" in error.value.msg + assert "list_relations_page_limit" in error.value.msg From dca565bb01a2fc8fa31eb37db0a39847af586270 Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Tue, 19 Nov 2024 10:14:20 -0800 Subject: [PATCH 2/6] Adap 209/update iceberg docs note (#1253) * Add url. * Add docs url to flag itself. * Update url in other error for more precision. * Add changelog. --- .changes/unreleased/Under the Hood-20241118-231637.yaml | 6 ++++++ dbt/adapters/snowflake/impl.py | 1 + dbt/include/snowflake/macros/relations/table/create.sql | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Under the Hood-20241118-231637.yaml diff --git a/.changes/unreleased/Under the Hood-20241118-231637.yaml b/.changes/unreleased/Under the Hood-20241118-231637.yaml new file mode 100644 index 000000000..f787a2a79 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20241118-231637.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Add iceberg docs url to behavior flag. +time: 2024-11-18T23:16:37.926576-08:00 +custom: + Author: versusfacit + Issue: "210" diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index dc0926d93..dc256c1cb 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -92,6 +92,7 @@ def _behavior_flags(self) -> List[BehaviorFlag]: "benefits only those actively using it, we've made this behavior opt-in to " "prevent unnecessary latency for other users." ), + "docs_url": "https://docs.getdbt.com/reference/resource-configs/snowflake-configs#iceberg-table-format", } ] diff --git a/dbt/include/snowflake/macros/relations/table/create.sql b/dbt/include/snowflake/macros/relations/table/create.sql index e2141df4d..50bedd78f 100644 --- a/dbt/include/snowflake/macros/relations/table/create.sql +++ b/dbt/include/snowflake/macros/relations/table/create.sql @@ -1,7 +1,7 @@ {% macro snowflake__create_table_as(temporary, relation, compiled_code, language='sql') -%} {%- if relation.is_iceberg_format and not adapter.behavior.enable_iceberg_materializations.no_warn %} - {% do exceptions.raise_compiler_error('Was unable to create model as Iceberg Table Format. Please set the `enable_iceberg_materializations` behavior flag to True in your dbt_project.yml. For more information, go to https://docs.getdbt.com/reference/resource-configs/snowflake-configs.') %} + {% do exceptions.raise_compiler_error('Was unable to create model as Iceberg Table Format. Please set the `enable_iceberg_materializations` behavior flag to True in your dbt_project.yml. For more information, go to https://docs.getdbt.com/reference/resource-configs/snowflake-configs#iceberg-table-format') %} {%- endif %} {%- set materialization_prefix = relation.get_ddl_prefix_for_create(config.model.config, temporary) -%} From 8e027d8fcfbf3958cbc97dfe38fba153d1077a6d Mon Sep 17 00:00:00 2001 From: pei Date: Wed, 20 Nov 2024 10:31:44 +0900 Subject: [PATCH 3/6] Handle non-English Snowflake error message for non-existing schemas (#840) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Handle Japanese Snowflake error message for non-existing schemas * impl.py を更新 Co-authored-by: Anders * Changelog entry * Update dbt/adapters/snowflake/impl.py Co-authored-by: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> --------- Co-authored-by: Anders Co-authored-by: Doug Beatty Co-authored-by: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> Co-authored-by: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> --- .changes/unreleased/Fixes-20231129-124145.yaml | 6 ++++++ dbt/adapters/snowflake/impl.py | 4 +++- 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Fixes-20231129-124145.yaml diff --git a/.changes/unreleased/Fixes-20231129-124145.yaml b/.changes/unreleased/Fixes-20231129-124145.yaml new file mode 100644 index 000000000..72a889d17 --- /dev/null +++ b/.changes/unreleased/Fixes-20231129-124145.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Handle non-English Snowflake error message for non-existing schemas +time: 2023-11-29T12:41:45.1273-07:00 +custom: + Author: pei0804 + Issue: "834" diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index dc256c1cb..ac0d903db 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -257,7 +257,9 @@ def list_relations_without_caching( # if the schema doesn't exist, we just want to return. # Alternatively, we could query the list of schemas before we start # and skip listing the missing ones, which sounds expensive. - if "Object does not exist" in str(exc): + # "002043 (02000)" is error code for "object does not exist or is not found" + # The error message text may vary across languages, but the error code is expected to be more stable + if "002043 (02000)" in str(exc): return [] raise From 54999b260f54b5a1e0e52a0e6d0b4a683b064aae Mon Sep 17 00:00:00 2001 From: FishtownBuildBot <77737458+FishtownBuildBot@users.noreply.github.com> Date: Sun, 1 Dec 2024 21:53:45 -0500 Subject: [PATCH 4/6] Cleanup main after cutting new 1.9.latest branch (#1255) * Clean up changelog on main * Bumping version to 1.10.0a1 * Code quality cleanup --- .bumpversion.cfg | 2 +- .changes/1.9.0-b1.md | 61 ------------------ .../1.9.0/Dependencies-20231219-125152.yaml | 6 -- .../1.9.0/Dependencies-20240412-155921.yaml | 6 -- .../1.9.0/Dependencies-20240429-124038.yaml | 6 -- .../1.9.0/Dependencies-20240429-124044.yaml | 6 -- .../1.9.0/Dependencies-20240624-122538.yaml | 6 -- .../1.9.0/Dependencies-20240718-120848.yaml | 6 -- .../1.9.0/Dependencies-20240718-120849.yaml | 6 -- .../1.9.0/Dependencies-20240718-120852.yaml | 6 -- .../1.9.0/Dependencies-20240718-120857.yaml | 6 -- .../1.9.0/Dependencies-20240719-120828.yaml | 6 -- .changes/1.9.0/Features-20240131-125318.yaml | 6 -- .changes/1.9.0/Features-20240430-185714.yaml | 6 -- .changes/1.9.0/Features-20240501-151901.yaml | 6 -- .changes/1.9.0/Features-20240604-154856.yaml | 6 -- .changes/1.9.0/Features-20240610-171026.yaml | 6 -- .changes/1.9.0/Features-20240709-194316.yaml | 6 -- .changes/1.9.0/Features-20240710-172345.yaml | 6 -- .changes/1.9.0/Features-20240911-001806.yaml | 6 -- .changes/1.9.0/Features-20240913-215416.yaml | 6 -- .changes/1.9.0/Features-20240917-100505.yaml | 6 -- .changes/1.9.0/Features-20240923-203204.yaml | 6 -- .changes/1.9.0/Features-20240930-112041.yaml | 6 -- .changes/1.9.0/Fixes-20240516-174337.yaml | 6 -- .changes/1.9.0/Fixes-20240516-224134.yaml | 6 -- .changes/1.9.0/Fixes-20240522-160538.yaml | 6 -- .changes/1.9.0/Fixes-20240605-125611.yaml | 6 -- .changes/1.9.0/Fixes-20240607-102708.yaml | 6 -- .changes/1.9.0/Fixes-20240628-190140.yaml | 7 --- .changes/1.9.0/Fixes-20240705-165932.yaml | 6 -- .changes/1.9.0/Fixes-20240920-193613.yaml | 6 -- .../1.9.0/Under the Hood-20240327-001304.yaml | 6 -- .../1.9.0/Under the Hood-20240425-144556.yaml | 6 -- .../1.9.0/Under the Hood-20240517-143743.yaml | 6 -- .../1.9.0/Under the Hood-20240614-170858.yaml | 6 -- .../1.9.0/Under the Hood-20240716-174655.yaml | 6 -- .../1.9.0/Under the Hood-20240719-125618.yaml | 6 -- .../1.9.0/Under the Hood-20240722-143114.yaml | 6 -- .../1.9.0/Under the Hood-20240806-215935.yaml | 6 -- .../1.9.0/Under the Hood-20240917-181147.yaml | 6 -- .../Breaking Changes-20241016-183143.yaml | 6 -- .../unreleased/Features-20241107-170307.yaml | 7 --- .../unreleased/Fixes-20231129-124145.yaml | 6 -- .../unreleased/Fixes-20241008-122635.yaml | 6 -- .../unreleased/Fixes-20241104-104610.yaml | 7 --- .../unreleased/Fixes-20241104-172349.yaml | 6 -- .../Under the Hood-20241016-035544.yaml | 6 -- .../Under the Hood-20241106-113249.yaml | 6 -- .../Under the Hood-20241118-231637.yaml | 6 -- CHANGELOG.md | 63 ------------------- dbt/adapters/snowflake/__version__.py | 2 +- 52 files changed, 2 insertions(+), 417 deletions(-) delete mode 100644 .changes/1.9.0-b1.md delete mode 100644 .changes/1.9.0/Dependencies-20231219-125152.yaml delete mode 100644 .changes/1.9.0/Dependencies-20240412-155921.yaml delete mode 100644 .changes/1.9.0/Dependencies-20240429-124038.yaml delete mode 100644 .changes/1.9.0/Dependencies-20240429-124044.yaml delete mode 100644 .changes/1.9.0/Dependencies-20240624-122538.yaml delete mode 100644 .changes/1.9.0/Dependencies-20240718-120848.yaml delete mode 100644 .changes/1.9.0/Dependencies-20240718-120849.yaml delete mode 100644 .changes/1.9.0/Dependencies-20240718-120852.yaml delete mode 100644 .changes/1.9.0/Dependencies-20240718-120857.yaml delete mode 100644 .changes/1.9.0/Dependencies-20240719-120828.yaml delete mode 100644 .changes/1.9.0/Features-20240131-125318.yaml delete mode 100644 .changes/1.9.0/Features-20240430-185714.yaml delete mode 100644 .changes/1.9.0/Features-20240501-151901.yaml delete mode 100644 .changes/1.9.0/Features-20240604-154856.yaml delete mode 100644 .changes/1.9.0/Features-20240610-171026.yaml delete mode 100644 .changes/1.9.0/Features-20240709-194316.yaml delete mode 100644 .changes/1.9.0/Features-20240710-172345.yaml delete mode 100644 .changes/1.9.0/Features-20240911-001806.yaml delete mode 100644 .changes/1.9.0/Features-20240913-215416.yaml delete mode 100644 .changes/1.9.0/Features-20240917-100505.yaml delete mode 100644 .changes/1.9.0/Features-20240923-203204.yaml delete mode 100644 .changes/1.9.0/Features-20240930-112041.yaml delete mode 100644 .changes/1.9.0/Fixes-20240516-174337.yaml delete mode 100644 .changes/1.9.0/Fixes-20240516-224134.yaml delete mode 100644 .changes/1.9.0/Fixes-20240522-160538.yaml delete mode 100644 .changes/1.9.0/Fixes-20240605-125611.yaml delete mode 100644 .changes/1.9.0/Fixes-20240607-102708.yaml delete mode 100644 .changes/1.9.0/Fixes-20240628-190140.yaml delete mode 100644 .changes/1.9.0/Fixes-20240705-165932.yaml delete mode 100644 .changes/1.9.0/Fixes-20240920-193613.yaml delete mode 100644 .changes/1.9.0/Under the Hood-20240327-001304.yaml delete mode 100644 .changes/1.9.0/Under the Hood-20240425-144556.yaml delete mode 100644 .changes/1.9.0/Under the Hood-20240517-143743.yaml delete mode 100644 .changes/1.9.0/Under the Hood-20240614-170858.yaml delete mode 100644 .changes/1.9.0/Under the Hood-20240716-174655.yaml delete mode 100644 .changes/1.9.0/Under the Hood-20240719-125618.yaml delete mode 100644 .changes/1.9.0/Under the Hood-20240722-143114.yaml delete mode 100644 .changes/1.9.0/Under the Hood-20240806-215935.yaml delete mode 100644 .changes/1.9.0/Under the Hood-20240917-181147.yaml delete mode 100644 .changes/unreleased/Breaking Changes-20241016-183143.yaml delete mode 100644 .changes/unreleased/Features-20241107-170307.yaml delete mode 100644 .changes/unreleased/Fixes-20231129-124145.yaml delete mode 100644 .changes/unreleased/Fixes-20241008-122635.yaml delete mode 100644 .changes/unreleased/Fixes-20241104-104610.yaml delete mode 100644 .changes/unreleased/Fixes-20241104-172349.yaml delete mode 100644 .changes/unreleased/Under the Hood-20241016-035544.yaml delete mode 100644 .changes/unreleased/Under the Hood-20241106-113249.yaml delete mode 100644 .changes/unreleased/Under the Hood-20241118-231637.yaml diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 7ff98322d..1d774733d 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.9.0b1 +current_version = 1.10.0a1 parse = (?P[\d]+) # major version number \.(?P[\d]+) # minor version number \.(?P[\d]+) # patch version number diff --git a/.changes/1.9.0-b1.md b/.changes/1.9.0-b1.md deleted file mode 100644 index 15a01afe7..000000000 --- a/.changes/1.9.0-b1.md +++ /dev/null @@ -1,61 +0,0 @@ -## dbt-snowflake 1.9.0-b1 - October 01, 2024 - -### Features - -- Support refresh_mode and initialize parameters for dynamic tables ([#1076](https://github.com/dbt-labs/dbt-snowflake/issues/1076)) -- Add tests for cross-database `cast` macro ([#1009](https://github.com/dbt-labs/dbt-snowflake/issues/1009)) -- Cross-database `date` macro ([#1013](https://github.com/dbt-labs/dbt-snowflake/issues/1013)) -- Replace underscores with hyphens in account IDs to prevent SSL issues ([#1068](https://github.com/dbt-labs/dbt-snowflake/issues/1068)) -- Support JWT Authentication ([#1079](https://github.com/dbt-labs/dbt-snowflake/issues/1079), [#726](https://github.com/dbt-labs/dbt-snowflake/issues/726)) -- Improve run times for large projects by reusing connections by default ([#1082](https://github.com/dbt-labs/dbt-snowflake/issues/1082)) -- Improve run times when using key pair auth by caching the private key ([#1082](https://github.com/dbt-labs/dbt-snowflake/issues/1082)) -- Add support for Iceberg table materializations. ([#321](https://github.com/dbt-labs/dbt-snowflake/issues/321)) -- Microbatch incremental strategy ([#1182](https://github.com/dbt-labs/dbt-snowflake/issues/1182)) -- Add support for Iceberg table format in Dynamic Tables ([#1183](https://github.com/dbt-labs/dbt-snowflake/issues/1183)) -- Add Iceberg format Incremental Models ([#321](https://github.com/dbt-labs/dbt-snowflake/issues/321)) -- Add support for all on_schema_change incremental model strategies. ([#321](https://github.com/dbt-labs/dbt-snowflake/issues/321)) - -### Fixes - -- Get catalog metadata for a single relation in the most optimized way using the get_catalog_for_single_relation macro and capability ([#1048](https://github.com/dbt-labs/dbt-snowflake/issues/1048)) -- Update relation caching to correctly identify dynamic tables, accounting for Snowflake's `2024_03` bundle ([#1016](https://github.com/dbt-labs/dbt-snowflake/issues/1016)) -- Rename targets for tables and views use fully qualified names ([#1031](https://github.com/dbt-labs/dbt-snowflake/issues/1031)) -- Surface SSO token expiration in logs ([#851](https://github.com/dbt-labs/dbt-snowflake/issues/851)) -- return to previous naming convention to return to quoting policy ([#1074](https://github.com/dbt-labs/dbt-snowflake/issues/1074)) -- Fix scenario where using the `--empty` flag causes metadata queries to contain limit clauses ([#1033](https://github.com/dbt-labs/dbt-snowflake/issues/1033)) -- Use show ... starts with instead of show ... like in _show_object_metadata ([#1102](https://github.com/dbt-labs/dbt-snowflake/issues/1102)) -- Fix issue where dbt-snowflake attempts to drop database roles during grants sync ([#1151](https://github.com/dbt-labs/dbt-snowflake/issues/1151)) - -### Under the Hood - -- Lazy load agate ([#953](https://github.com/dbt-labs/dbt-snowflake/issues/953)) -- Speedup catalog string comparison by using ilike before equals ([#1035](https://github.com/dbt-labs/dbt-snowflake/issues/1035)) -- Improve memory efficiency of the process_results() override. ([#1053](https://github.com/dbt-labs/dbt-snowflake/issues/1053)) -- Automate all manual integration tests for Dynamic Tables ([#1084](https://github.com/dbt-labs/dbt-snowflake/issues/1084)) -- Add support for experimental record/replay testing. ([#1106](https://github.com/dbt-labs/dbt-snowflake/issues/1106)) -- Remove `freezegun` as a testing dependency; this package is no longer used ([#1136](https://github.com/dbt-labs/dbt-snowflake/issues/1136)) -- Add support for Python 3.12 ([#903](https://github.com/dbt-labs/dbt-snowflake/issues/903)) -- Isolating distribution testing ([#1130](https://github.com/dbt-labs/dbt-snowflake/issues/1130)) -- Change behavior flag semantics to log iceberg flag warnings.. ([#321](https://github.com/dbt-labs/dbt-snowflake/issues/321)) - -### Dependencies - -- Update freezegun requirement from ~=1.3 to ~=1.4 ([#869](https://github.com/dbt-labs/dbt-snowflake/pull/869)) -- Bump actions/upload-artifact from 3 to 4 ([#971](https://github.com/dbt-labs/dbt-snowflake/pull/971)) -- Bump dbt-labs/actions from 1.1.0 to 1.1.1 ([#1006](https://github.com/dbt-labs/dbt-snowflake/pull/1006)) -- Bump actions/download-artifact from 3 to 4 ([#1007](https://github.com/dbt-labs/dbt-snowflake/pull/1007)) -- Bump aurelien-baudet/workflow-dispatch from 2 to 4 ([#1093](https://github.com/dbt-labs/dbt-snowflake/pull/1093)) -- Update twine requirement from ~=4.0 to ~=5.1 ([#1120](https://github.com/dbt-labs/dbt-snowflake/pull/1120)) -- Bump pre-commit from 3.7.0 to 3.7.1 ([#1119](https://github.com/dbt-labs/dbt-snowflake/pull/1119)) -- Update wheel requirement from ~=0.42 to ~=0.43 ([#1121](https://github.com/dbt-labs/dbt-snowflake/pull/1121)) -- Update pytest-xdist requirement from ~=3.5 to ~=3.6 ([#1122](https://github.com/dbt-labs/dbt-snowflake/pull/1122)) -- Update tox requirement from ~=4.11 to ~=4.16 ([#1135](https://github.com/dbt-labs/dbt-snowflake/pull/1135)) - -### Contributors -- [@HenkvanDyk,mikealfare](https://github.com/HenkvanDyk,mikealfare) ([#1076](https://github.com/dbt-labs/dbt-snowflake/issues/1076)) -- [@McKnight-42](https://github.com/McKnight-42) ([#851](https://github.com/dbt-labs/dbt-snowflake/issues/851), [#1074](https://github.com/dbt-labs/dbt-snowflake/issues/1074)) -- [@amardatar](https://github.com/amardatar) ([#1082](https://github.com/dbt-labs/dbt-snowflake/issues/1082)) -- [@dwreeves](https://github.com/dwreeves) ([#953](https://github.com/dbt-labs/dbt-snowflake/issues/953)) -- [@leahwicz](https://github.com/leahwicz) ([#1130](https://github.com/dbt-labs/dbt-snowflake/issues/1130)) -- [@llam15](https://github.com/llam15) ([#1079](https://github.com/dbt-labs/dbt-snowflake/issues/1079), [#726](https://github.com/dbt-labs/dbt-snowflake/issues/726)) -- [@mikealfare,](https://github.com/mikealfare,) ([#851](https://github.com/dbt-labs/dbt-snowflake/issues/851)) diff --git a/.changes/1.9.0/Dependencies-20231219-125152.yaml b/.changes/1.9.0/Dependencies-20231219-125152.yaml deleted file mode 100644 index 2d730daf1..000000000 --- a/.changes/1.9.0/Dependencies-20231219-125152.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update freezegun requirement from ~=1.3 to ~=1.4" -time: 2023-12-19T12:51:52.00000Z -custom: - Author: dependabot[bot] - PR: 869 diff --git a/.changes/1.9.0/Dependencies-20240412-155921.yaml b/.changes/1.9.0/Dependencies-20240412-155921.yaml deleted file mode 100644 index f83e5b404..000000000 --- a/.changes/1.9.0/Dependencies-20240412-155921.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Bump actions/upload-artifact from 3 to 4" -time: 2024-04-12T15:59:21.00000Z -custom: - Author: dependabot[bot] - PR: 971 diff --git a/.changes/1.9.0/Dependencies-20240429-124038.yaml b/.changes/1.9.0/Dependencies-20240429-124038.yaml deleted file mode 100644 index 5fa954c8a..000000000 --- a/.changes/1.9.0/Dependencies-20240429-124038.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Bump dbt-labs/actions from 1.1.0 to 1.1.1" -time: 2024-04-29T12:40:38.00000Z -custom: - Author: dependabot[bot] - PR: 1006 diff --git a/.changes/1.9.0/Dependencies-20240429-124044.yaml b/.changes/1.9.0/Dependencies-20240429-124044.yaml deleted file mode 100644 index 834fce096..000000000 --- a/.changes/1.9.0/Dependencies-20240429-124044.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Bump actions/download-artifact from 3 to 4" -time: 2024-04-29T12:40:44.00000Z -custom: - Author: dependabot[bot] - PR: 1007 diff --git a/.changes/1.9.0/Dependencies-20240624-122538.yaml b/.changes/1.9.0/Dependencies-20240624-122538.yaml deleted file mode 100644 index e47731aef..000000000 --- a/.changes/1.9.0/Dependencies-20240624-122538.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Bump aurelien-baudet/workflow-dispatch from 2 to 4" -time: 2024-06-24T12:25:38.00000Z -custom: - Author: dependabot[bot] - PR: 1093 diff --git a/.changes/1.9.0/Dependencies-20240718-120848.yaml b/.changes/1.9.0/Dependencies-20240718-120848.yaml deleted file mode 100644 index c46a30eba..000000000 --- a/.changes/1.9.0/Dependencies-20240718-120848.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update twine requirement from ~=4.0 to ~=5.1" -time: 2024-07-18T12:08:48.00000Z -custom: - Author: dependabot[bot] - PR: 1120 diff --git a/.changes/1.9.0/Dependencies-20240718-120849.yaml b/.changes/1.9.0/Dependencies-20240718-120849.yaml deleted file mode 100644 index df248ff7d..000000000 --- a/.changes/1.9.0/Dependencies-20240718-120849.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Bump pre-commit from 3.7.0 to 3.7.1" -time: 2024-07-18T12:08:49.00000Z -custom: - Author: dependabot[bot] - PR: 1119 diff --git a/.changes/1.9.0/Dependencies-20240718-120852.yaml b/.changes/1.9.0/Dependencies-20240718-120852.yaml deleted file mode 100644 index 40c171f93..000000000 --- a/.changes/1.9.0/Dependencies-20240718-120852.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update wheel requirement from ~=0.42 to ~=0.43" -time: 2024-07-18T12:08:52.00000Z -custom: - Author: dependabot[bot] - PR: 1121 diff --git a/.changes/1.9.0/Dependencies-20240718-120857.yaml b/.changes/1.9.0/Dependencies-20240718-120857.yaml deleted file mode 100644 index e4bfe04d0..000000000 --- a/.changes/1.9.0/Dependencies-20240718-120857.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update pytest-xdist requirement from ~=3.5 to ~=3.6" -time: 2024-07-18T12:08:57.00000Z -custom: - Author: dependabot[bot] - PR: 1122 diff --git a/.changes/1.9.0/Dependencies-20240719-120828.yaml b/.changes/1.9.0/Dependencies-20240719-120828.yaml deleted file mode 100644 index ea7af843c..000000000 --- a/.changes/1.9.0/Dependencies-20240719-120828.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update tox requirement from ~=4.11 to ~=4.16" -time: 2024-07-19T12:08:28.00000Z -custom: - Author: dependabot[bot] - PR: 1135 diff --git a/.changes/1.9.0/Features-20240131-125318.yaml b/.changes/1.9.0/Features-20240131-125318.yaml deleted file mode 100644 index 63771d71e..000000000 --- a/.changes/1.9.0/Features-20240131-125318.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Support refresh_mode and initialize parameters for dynamic tables -time: 2024-01-31T12:53:18.111616Z -custom: - Author: HenkvanDyk,mikealfare - Issue: "1076" diff --git a/.changes/1.9.0/Features-20240430-185714.yaml b/.changes/1.9.0/Features-20240430-185714.yaml deleted file mode 100644 index 9fd1e97ea..000000000 --- a/.changes/1.9.0/Features-20240430-185714.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Add tests for cross-database `cast` macro -time: 2024-04-30T18:57:14.753057-06:00 -custom: - Author: dbeatty10 - Issue: "1009" diff --git a/.changes/1.9.0/Features-20240501-151901.yaml b/.changes/1.9.0/Features-20240501-151901.yaml deleted file mode 100644 index 0f792c40e..000000000 --- a/.changes/1.9.0/Features-20240501-151901.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Cross-database `date` macro -time: 2024-05-01T15:19:01.141157-06:00 -custom: - Author: dbeatty10 - Issue: 1013 diff --git a/.changes/1.9.0/Features-20240604-154856.yaml b/.changes/1.9.0/Features-20240604-154856.yaml deleted file mode 100644 index 7d83b1da7..000000000 --- a/.changes/1.9.0/Features-20240604-154856.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Replace underscores with hyphens in account IDs to prevent SSL issues -time: 2024-06-04T15:48:56.845374-07:00 -custom: - Author: colin-rogers-dbt - Issue: "1068" diff --git a/.changes/1.9.0/Features-20240610-171026.yaml b/.changes/1.9.0/Features-20240610-171026.yaml deleted file mode 100644 index 5cc055160..000000000 --- a/.changes/1.9.0/Features-20240610-171026.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Support JWT Authentication -time: 2024-06-10T17:10:26.421463-04:00 -custom: - Author: llam15 - Issue: 1079 726 diff --git a/.changes/1.9.0/Features-20240709-194316.yaml b/.changes/1.9.0/Features-20240709-194316.yaml deleted file mode 100644 index a867387e3..000000000 --- a/.changes/1.9.0/Features-20240709-194316.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Improve run times for large projects by reusing connections by default -time: 2024-07-09T19:43:16.489649-04:00 -custom: - Author: mikealfare amardatar - Issue: "1082" diff --git a/.changes/1.9.0/Features-20240710-172345.yaml b/.changes/1.9.0/Features-20240710-172345.yaml deleted file mode 100644 index e68f63812..000000000 --- a/.changes/1.9.0/Features-20240710-172345.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Improve run times when using key pair auth by caching the private key -time: 2024-07-10T17:23:45.046905-04:00 -custom: - Author: mikealfare aranke - Issue: "1082" diff --git a/.changes/1.9.0/Features-20240911-001806.yaml b/.changes/1.9.0/Features-20240911-001806.yaml deleted file mode 100644 index 024480b96..000000000 --- a/.changes/1.9.0/Features-20240911-001806.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Add support for Iceberg table materializations. -time: 2024-09-11T00:18:06.780586-07:00 -custom: - Author: versusfacit - Issue: "321" diff --git a/.changes/1.9.0/Features-20240913-215416.yaml b/.changes/1.9.0/Features-20240913-215416.yaml deleted file mode 100644 index b2a6e556e..000000000 --- a/.changes/1.9.0/Features-20240913-215416.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Microbatch incremental strategy -time: 2024-09-13T21:54:16.492885-04:00 -custom: - Author: michelleark - Issue: "1182" diff --git a/.changes/1.9.0/Features-20240917-100505.yaml b/.changes/1.9.0/Features-20240917-100505.yaml deleted file mode 100644 index 22cabc904..000000000 --- a/.changes/1.9.0/Features-20240917-100505.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Add support for Iceberg table format in Dynamic Tables -time: 2024-09-17T10:05:05.609859-04:00 -custom: - Author: mikealfare - Issue: "1183" diff --git a/.changes/1.9.0/Features-20240923-203204.yaml b/.changes/1.9.0/Features-20240923-203204.yaml deleted file mode 100644 index eaca4906b..000000000 --- a/.changes/1.9.0/Features-20240923-203204.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Add Iceberg format Incremental Models -time: 2024-09-23T20:32:04.783741-07:00 -custom: - Author: versusfacit - Issue: "321" diff --git a/.changes/1.9.0/Features-20240930-112041.yaml b/.changes/1.9.0/Features-20240930-112041.yaml deleted file mode 100644 index 1395a8bf7..000000000 --- a/.changes/1.9.0/Features-20240930-112041.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Add support for all on_schema_change incremental model strategies. -time: 2024-09-30T11:20:41.99589-07:00 -custom: - Author: versusfacit - Issue: "321" diff --git a/.changes/1.9.0/Fixes-20240516-174337.yaml b/.changes/1.9.0/Fixes-20240516-174337.yaml deleted file mode 100644 index 955d90ed3..000000000 --- a/.changes/1.9.0/Fixes-20240516-174337.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Update relation caching to correctly identify dynamic tables, accounting for Snowflake's `2024_03` bundle -time: 2024-05-16T17:43:37.336858-04:00 -custom: - Author: mikealfare - Issue: "1016" diff --git a/.changes/1.9.0/Fixes-20240516-224134.yaml b/.changes/1.9.0/Fixes-20240516-224134.yaml deleted file mode 100644 index 011ecb449..000000000 --- a/.changes/1.9.0/Fixes-20240516-224134.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Get catalog metadata for a single relation in the most optimized way using the get_catalog_for_single_relation macro and capability -time: 2024-05-16T22:41:34.256095+01:00 -custom: - Author: aranke - Issue: "1048" diff --git a/.changes/1.9.0/Fixes-20240522-160538.yaml b/.changes/1.9.0/Fixes-20240522-160538.yaml deleted file mode 100644 index 4921706a9..000000000 --- a/.changes/1.9.0/Fixes-20240522-160538.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: 'Rename targets for tables and views use fully qualified names' -time: 2024-05-22T16:05:38.602074-04:00 -custom: - Author: mikealfare - Issue: "1031" diff --git a/.changes/1.9.0/Fixes-20240605-125611.yaml b/.changes/1.9.0/Fixes-20240605-125611.yaml deleted file mode 100644 index c4560774c..000000000 --- a/.changes/1.9.0/Fixes-20240605-125611.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Surface SSO token expiration in logs -time: 2024-06-05T12:56:11.802237-04:00 -custom: - Author: mikealfare, McKnight-42 - Issue: "851" diff --git a/.changes/1.9.0/Fixes-20240607-102708.yaml b/.changes/1.9.0/Fixes-20240607-102708.yaml deleted file mode 100644 index 58cd9bbee..000000000 --- a/.changes/1.9.0/Fixes-20240607-102708.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: return to previous naming convention to return to quoting policy -time: 2024-06-07T10:27:08.542159-05:00 -custom: - Author: McKnight-42 - Issue: "1074" diff --git a/.changes/1.9.0/Fixes-20240628-190140.yaml b/.changes/1.9.0/Fixes-20240628-190140.yaml deleted file mode 100644 index c58b465fd..000000000 --- a/.changes/1.9.0/Fixes-20240628-190140.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: Fixes -body: Fix scenario where using the `--empty` flag causes metadata queries to contain - limit clauses -time: 2024-06-28T19:01:40.558234-04:00 -custom: - Author: mikealfare - Issue: "1033" diff --git a/.changes/1.9.0/Fixes-20240705-165932.yaml b/.changes/1.9.0/Fixes-20240705-165932.yaml deleted file mode 100644 index ffe902c92..000000000 --- a/.changes/1.9.0/Fixes-20240705-165932.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Use show ... starts with instead of show ... like in _show_object_metadata -time: 2024-07-05T16:59:32.087555+01:00 -custom: - Author: aranke - Issue: "1102" diff --git a/.changes/1.9.0/Fixes-20240920-193613.yaml b/.changes/1.9.0/Fixes-20240920-193613.yaml deleted file mode 100644 index f85f6fc56..000000000 --- a/.changes/1.9.0/Fixes-20240920-193613.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Fix issue where dbt-snowflake attempts to drop database roles during grants sync -time: 2024-09-20T19:36:13.671173-04:00 -custom: - Author: mikealfare - Issue: "1151" diff --git a/.changes/1.9.0/Under the Hood-20240327-001304.yaml b/.changes/1.9.0/Under the Hood-20240327-001304.yaml deleted file mode 100644 index 3e823ec86..000000000 --- a/.changes/1.9.0/Under the Hood-20240327-001304.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Lazy load agate -time: 2024-03-27T00:13:04.246062-04:00 -custom: - Author: dwreeves - Issue: "953" diff --git a/.changes/1.9.0/Under the Hood-20240425-144556.yaml b/.changes/1.9.0/Under the Hood-20240425-144556.yaml deleted file mode 100644 index 002da3c1f..000000000 --- a/.changes/1.9.0/Under the Hood-20240425-144556.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Speedup catalog string comparison by using ilike before equals -time: 2024-04-25T14:45:56.549787+02:00 -custom: - Author: aranke - Issue: '1035' diff --git a/.changes/1.9.0/Under the Hood-20240517-143743.yaml b/.changes/1.9.0/Under the Hood-20240517-143743.yaml deleted file mode 100644 index 598c60ad4..000000000 --- a/.changes/1.9.0/Under the Hood-20240517-143743.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Improve memory efficiency of the process_results() override. -time: 2024-05-17T14:37:43.7414-04:00 -custom: - Author: peterallenwebb - Issue: "1053" diff --git a/.changes/1.9.0/Under the Hood-20240614-170858.yaml b/.changes/1.9.0/Under the Hood-20240614-170858.yaml deleted file mode 100644 index cc806726b..000000000 --- a/.changes/1.9.0/Under the Hood-20240614-170858.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Automate all manual integration tests for Dynamic Tables -time: 2024-06-14T17:08:58.231472-04:00 -custom: - Author: mikealfare - Issue: "1084" diff --git a/.changes/1.9.0/Under the Hood-20240716-174655.yaml b/.changes/1.9.0/Under the Hood-20240716-174655.yaml deleted file mode 100644 index 14c3c8d76..000000000 --- a/.changes/1.9.0/Under the Hood-20240716-174655.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Add support for experimental record/replay testing. -time: 2024-07-16T17:46:55.11204-04:00 -custom: - Author: peterallenwebb - Issue: "1106" diff --git a/.changes/1.9.0/Under the Hood-20240719-125618.yaml b/.changes/1.9.0/Under the Hood-20240719-125618.yaml deleted file mode 100644 index 3d90b732c..000000000 --- a/.changes/1.9.0/Under the Hood-20240719-125618.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Remove `freezegun` as a testing dependency; this package is no longer used -time: 2024-07-19T12:56:18.957049-04:00 -custom: - Author: mikealfare - Issue: "1136" diff --git a/.changes/1.9.0/Under the Hood-20240722-143114.yaml b/.changes/1.9.0/Under the Hood-20240722-143114.yaml deleted file mode 100644 index dc5c2dbb1..000000000 --- a/.changes/1.9.0/Under the Hood-20240722-143114.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Add support for Python 3.12 -time: 2024-07-22T14:31:14.024865-07:00 -custom: - Author: versusfacit - Issue: "903" diff --git a/.changes/1.9.0/Under the Hood-20240806-215935.yaml b/.changes/1.9.0/Under the Hood-20240806-215935.yaml deleted file mode 100644 index 660918350..000000000 --- a/.changes/1.9.0/Under the Hood-20240806-215935.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Isolating distribution testing -time: 2024-08-06T21:59:35.284641-04:00 -custom: - Author: leahwicz - Issue: "1130" diff --git a/.changes/1.9.0/Under the Hood-20240917-181147.yaml b/.changes/1.9.0/Under the Hood-20240917-181147.yaml deleted file mode 100644 index 2f52174dd..000000000 --- a/.changes/1.9.0/Under the Hood-20240917-181147.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Change behavior flag semantics to log iceberg flag warnings.. -time: 2024-09-17T18:11:47.525026-07:00 -custom: - Author: versusfacit - Issue: "321" diff --git a/.changes/unreleased/Breaking Changes-20241016-183143.yaml b/.changes/unreleased/Breaking Changes-20241016-183143.yaml deleted file mode 100644 index 26cc4b6de..000000000 --- a/.changes/unreleased/Breaking Changes-20241016-183143.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Breaking Changes -body: Drop support for Python 3.8 -time: 2024-10-16T18:31:43.4167-04:00 -custom: - Author: mikealfare - Issue: "1211" diff --git a/.changes/unreleased/Features-20241107-170307.yaml b/.changes/unreleased/Features-20241107-170307.yaml deleted file mode 100644 index 1479c5805..000000000 --- a/.changes/unreleased/Features-20241107-170307.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: Features -body: 'Allow configurable pagination on list_relations_without_caching to support - users with a large number of objects per schema' -time: 2024-11-07T17:03:07.826352-05:00 -custom: - Author: mikealfare - Issue: "1234" diff --git a/.changes/unreleased/Fixes-20231129-124145.yaml b/.changes/unreleased/Fixes-20231129-124145.yaml deleted file mode 100644 index 72a889d17..000000000 --- a/.changes/unreleased/Fixes-20231129-124145.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Handle non-English Snowflake error message for non-existing schemas -time: 2023-11-29T12:41:45.1273-07:00 -custom: - Author: pei0804 - Issue: "834" diff --git a/.changes/unreleased/Fixes-20241008-122635.yaml b/.changes/unreleased/Fixes-20241008-122635.yaml deleted file mode 100644 index c069283d6..000000000 --- a/.changes/unreleased/Fixes-20241008-122635.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Dynamic Iceberg table base_location_subpath generation fix. -time: 2024-10-08T12:26:35.521308-07:00 -custom: - Author: versusfacit - Issue: "1200" diff --git a/.changes/unreleased/Fixes-20241104-104610.yaml b/.changes/unreleased/Fixes-20241104-104610.yaml deleted file mode 100644 index c512d0bdd..000000000 --- a/.changes/unreleased/Fixes-20241104-104610.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: Fixes -body: 'Performance fixes for snowflake microbatch strategy: use temp view instead - of table, remove unnecessary ''using'' clause' -time: 2024-11-04T10:46:10.005317-05:00 -custom: - Author: michelleark - Issue: "1228" diff --git a/.changes/unreleased/Fixes-20241104-172349.yaml b/.changes/unreleased/Fixes-20241104-172349.yaml deleted file mode 100644 index 07c90d93c..000000000 --- a/.changes/unreleased/Fixes-20241104-172349.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Iceberg quoting ignore fix. -time: 2024-11-04T17:23:49.706297-08:00 -custom: - Author: versusfacit - Issue: "1227" diff --git a/.changes/unreleased/Under the Hood-20241016-035544.yaml b/.changes/unreleased/Under the Hood-20241016-035544.yaml deleted file mode 100644 index 59e4f70de..000000000 --- a/.changes/unreleased/Under the Hood-20241016-035544.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Add telemetry function -time: 2024-10-16T03:55:44.144174-07:00 -custom: - Author: versusfacit - Issue: "301" diff --git a/.changes/unreleased/Under the Hood-20241106-113249.yaml b/.changes/unreleased/Under the Hood-20241106-113249.yaml deleted file mode 100644 index 0437a8c88..000000000 --- a/.changes/unreleased/Under the Hood-20241106-113249.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: remove SnowflakeAdapterResponse in favor of updated AdapterResponse in base -time: 2024-11-06T11:32:49.503467-08:00 -custom: - Author: colin-rogers-dbt - Issue: "1233" diff --git a/.changes/unreleased/Under the Hood-20241118-231637.yaml b/.changes/unreleased/Under the Hood-20241118-231637.yaml deleted file mode 100644 index f787a2a79..000000000 --- a/.changes/unreleased/Under the Hood-20241118-231637.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Add iceberg docs url to behavior flag. -time: 2024-11-18T23:16:37.926576-08:00 -custom: - Author: versusfacit - Issue: "210" diff --git a/CHANGELOG.md b/CHANGELOG.md index 599c20195..8b6702b8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,69 +5,6 @@ - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version. - Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-snowflake/blob/main/CONTRIBUTING.md#adding-changelog-entry) -## dbt-snowflake 1.9.0-b1 - October 01, 2024 - -### Features - -- Support refresh_mode and initialize parameters for dynamic tables ([#1076](https://github.com/dbt-labs/dbt-snowflake/issues/1076)) -- Add tests for cross-database `cast` macro ([#1009](https://github.com/dbt-labs/dbt-snowflake/issues/1009)) -- Cross-database `date` macro ([#1013](https://github.com/dbt-labs/dbt-snowflake/issues/1013)) -- Replace underscores with hyphens in account IDs to prevent SSL issues ([#1068](https://github.com/dbt-labs/dbt-snowflake/issues/1068)) -- Support JWT Authentication ([#1079](https://github.com/dbt-labs/dbt-snowflake/issues/1079), [#726](https://github.com/dbt-labs/dbt-snowflake/issues/726)) -- Improve run times for large projects by reusing connections by default ([#1082](https://github.com/dbt-labs/dbt-snowflake/issues/1082)) -- Improve run times when using key pair auth by caching the private key ([#1082](https://github.com/dbt-labs/dbt-snowflake/issues/1082)) -- Add support for Iceberg table materializations. ([#321](https://github.com/dbt-labs/dbt-snowflake/issues/321)) -- Microbatch incremental strategy ([#1182](https://github.com/dbt-labs/dbt-snowflake/issues/1182)) -- Add support for Iceberg table format in Dynamic Tables ([#1183](https://github.com/dbt-labs/dbt-snowflake/issues/1183)) -- Add Iceberg format Incremental Models ([#321](https://github.com/dbt-labs/dbt-snowflake/issues/321)) -- Add support for all on_schema_change incremental model strategies. ([#321](https://github.com/dbt-labs/dbt-snowflake/issues/321)) - -### Fixes - -- Get catalog metadata for a single relation in the most optimized way using the get_catalog_for_single_relation macro and capability ([#1048](https://github.com/dbt-labs/dbt-snowflake/issues/1048)) -- Update relation caching to correctly identify dynamic tables, accounting for Snowflake's `2024_03` bundle ([#1016](https://github.com/dbt-labs/dbt-snowflake/issues/1016)) -- Rename targets for tables and views use fully qualified names ([#1031](https://github.com/dbt-labs/dbt-snowflake/issues/1031)) -- Surface SSO token expiration in logs ([#851](https://github.com/dbt-labs/dbt-snowflake/issues/851)) -- return to previous naming convention to return to quoting policy ([#1074](https://github.com/dbt-labs/dbt-snowflake/issues/1074)) -- Fix scenario where using the `--empty` flag causes metadata queries to contain limit clauses ([#1033](https://github.com/dbt-labs/dbt-snowflake/issues/1033)) -- Use show ... starts with instead of show ... like in _show_object_metadata ([#1102](https://github.com/dbt-labs/dbt-snowflake/issues/1102)) -- Fix issue where dbt-snowflake attempts to drop database roles during grants sync ([#1151](https://github.com/dbt-labs/dbt-snowflake/issues/1151)) - -### Under the Hood - -- Lazy load agate ([#953](https://github.com/dbt-labs/dbt-snowflake/issues/953)) -- Speedup catalog string comparison by using ilike before equals ([#1035](https://github.com/dbt-labs/dbt-snowflake/issues/1035)) -- Improve memory efficiency of the process_results() override. ([#1053](https://github.com/dbt-labs/dbt-snowflake/issues/1053)) -- Automate all manual integration tests for Dynamic Tables ([#1084](https://github.com/dbt-labs/dbt-snowflake/issues/1084)) -- Add support for experimental record/replay testing. ([#1106](https://github.com/dbt-labs/dbt-snowflake/issues/1106)) -- Remove `freezegun` as a testing dependency; this package is no longer used ([#1136](https://github.com/dbt-labs/dbt-snowflake/issues/1136)) -- Add support for Python 3.12 ([#903](https://github.com/dbt-labs/dbt-snowflake/issues/903)) -- Isolating distribution testing ([#1130](https://github.com/dbt-labs/dbt-snowflake/issues/1130)) -- Change behavior flag semantics to log iceberg flag warnings.. ([#321](https://github.com/dbt-labs/dbt-snowflake/issues/321)) - -### Dependencies - -- Update freezegun requirement from ~=1.3 to ~=1.4 ([#869](https://github.com/dbt-labs/dbt-snowflake/pull/869)) -- Bump actions/upload-artifact from 3 to 4 ([#971](https://github.com/dbt-labs/dbt-snowflake/pull/971)) -- Bump dbt-labs/actions from 1.1.0 to 1.1.1 ([#1006](https://github.com/dbt-labs/dbt-snowflake/pull/1006)) -- Bump actions/download-artifact from 3 to 4 ([#1007](https://github.com/dbt-labs/dbt-snowflake/pull/1007)) -- Bump aurelien-baudet/workflow-dispatch from 2 to 4 ([#1093](https://github.com/dbt-labs/dbt-snowflake/pull/1093)) -- Update twine requirement from ~=4.0 to ~=5.1 ([#1120](https://github.com/dbt-labs/dbt-snowflake/pull/1120)) -- Bump pre-commit from 3.7.0 to 3.7.1 ([#1119](https://github.com/dbt-labs/dbt-snowflake/pull/1119)) -- Update wheel requirement from ~=0.42 to ~=0.43 ([#1121](https://github.com/dbt-labs/dbt-snowflake/pull/1121)) -- Update pytest-xdist requirement from ~=3.5 to ~=3.6 ([#1122](https://github.com/dbt-labs/dbt-snowflake/pull/1122)) -- Update tox requirement from ~=4.11 to ~=4.16 ([#1135](https://github.com/dbt-labs/dbt-snowflake/pull/1135)) - -### Contributors -- [@HenkvanDyk,mikealfare](https://github.com/HenkvanDyk,mikealfare) ([#1076](https://github.com/dbt-labs/dbt-snowflake/issues/1076)) -- [@McKnight-42](https://github.com/McKnight-42) ([#851](https://github.com/dbt-labs/dbt-snowflake/issues/851), [#1074](https://github.com/dbt-labs/dbt-snowflake/issues/1074)) -- [@amardatar](https://github.com/amardatar) ([#1082](https://github.com/dbt-labs/dbt-snowflake/issues/1082)) -- [@dwreeves](https://github.com/dwreeves) ([#953](https://github.com/dbt-labs/dbt-snowflake/issues/953)) -- [@leahwicz](https://github.com/leahwicz) ([#1130](https://github.com/dbt-labs/dbt-snowflake/issues/1130)) -- [@llam15](https://github.com/llam15) ([#1079](https://github.com/dbt-labs/dbt-snowflake/issues/1079), [#726](https://github.com/dbt-labs/dbt-snowflake/issues/726)) -- [@mikealfare,](https://github.com/mikealfare,) ([#851](https://github.com/dbt-labs/dbt-snowflake/issues/851)) - - ## Previous Releases For information on prior major and minor releases, see their changelogs: - [1.6](https://github.com/dbt-labs/dbt-snowflake/blob/1.6.latest/CHANGELOG.md) diff --git a/dbt/adapters/snowflake/__version__.py b/dbt/adapters/snowflake/__version__.py index a4077fff2..1af777a62 100644 --- a/dbt/adapters/snowflake/__version__.py +++ b/dbt/adapters/snowflake/__version__.py @@ -1 +1 @@ -version = "1.9.0b1" +version = "1.10.0a1" From 5cc5d224f9cc45802b8141472a594c2cdf0facb1 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Mon, 2 Dec 2024 11:05:29 -0500 Subject: [PATCH 5/6] Use timestamp_tz type in microbatch `delete` DDL (#1257) --- .changes/unreleased/Fixes-20241127-162204.yaml | 6 ++++++ .../snowflake/macros/materializations/merge.sql | 4 ++-- .../adapter/test_incremental_microbatch.py | 14 ++++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 .changes/unreleased/Fixes-20241127-162204.yaml diff --git a/.changes/unreleased/Fixes-20241127-162204.yaml b/.changes/unreleased/Fixes-20241127-162204.yaml new file mode 100644 index 000000000..2b990b1f9 --- /dev/null +++ b/.changes/unreleased/Fixes-20241127-162204.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Use timestamp_tz type in microbatch `delete` DDL +time: 2024-11-27T16:22:04.103212-05:00 +custom: + Author: michelleark + Issue: "1256" diff --git a/dbt/include/snowflake/macros/materializations/merge.sql b/dbt/include/snowflake/macros/materializations/merge.sql index c8ac8d6fd..716a325c1 100644 --- a/dbt/include/snowflake/macros/materializations/merge.sql +++ b/dbt/include/snowflake/macros/materializations/merge.sql @@ -58,10 +58,10 @@ {#-- Add additional incremental_predicates to filter for batch --#} {% if model.config.get("__dbt_internal_microbatch_event_time_start") -%} - {% do incremental_predicates.append("DBT_INTERNAL_TARGET." ~ model.config.event_time ~ " >= TIMESTAMP '" ~ model.config.__dbt_internal_microbatch_event_time_start ~ "'") %} + {% do incremental_predicates.append("DBT_INTERNAL_TARGET." ~ model.config.event_time ~ " >= to_timestamp_tz('" ~ model.config.__dbt_internal_microbatch_event_time_start ~ "')") %} {% endif %} {% if model.config.__dbt_internal_microbatch_event_time_end -%} - {% do incremental_predicates.append("DBT_INTERNAL_TARGET." ~ model.config.event_time ~ " < TIMESTAMP '" ~ model.config.__dbt_internal_microbatch_event_time_end ~ "'") %} + {% do incremental_predicates.append("DBT_INTERNAL_TARGET." ~ model.config.event_time ~ " < to_timestamp_tz('" ~ model.config.__dbt_internal_microbatch_event_time_end ~ "')") %} {% endif %} {% do arg_dict.update({'incremental_predicates': incremental_predicates}) %} diff --git a/tests/functional/adapter/test_incremental_microbatch.py b/tests/functional/adapter/test_incremental_microbatch.py index f228c370c..f087596e1 100644 --- a/tests/functional/adapter/test_incremental_microbatch.py +++ b/tests/functional/adapter/test_incremental_microbatch.py @@ -3,6 +3,16 @@ BaseMicrobatch, ) +# Create input with UTC timestamps +_input_model_sql = """ +{{ config(materialized='table', event_time='event_time') }} +select 1 as id, to_timestamp_tz('2020-01-01 00:00:00-0') as event_time +union all +select 2 as id, to_timestamp_tz('2020-01-02 00:00:00-0') as event_time +union all +select 3 as id, to_timestamp_tz('2020-01-03 00:00:00-0') as event_time +""" + # No requirement for a unique_id for snowflake microbatch! _microbatch_model_no_unique_id_sql = """ @@ -16,6 +26,10 @@ class TestSnowflakeMicrobatch(BaseMicrobatch): def microbatch_model_sql(self) -> str: return _microbatch_model_no_unique_id_sql + @pytest.fixture(scope="class") + def input_model_sql(self) -> str: + return _input_model_sql + @pytest.fixture(scope="class") def insert_two_rows_sql(self, project) -> str: test_schema_relation = project.adapter.Relation.create( From 0b24a5a2d311ffec5f996ca28532076a637aa6b3 Mon Sep 17 00:00:00 2001 From: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Wed, 4 Dec 2024 09:05:38 -0800 Subject: [PATCH 6/6] update libpq-dev dependency to 13.18-0+deb11u1 (#1262) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 17315b12d..16060db61 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -9,7 +9,7 @@ RUN apt-get update \ build-essential=12.9 \ ca-certificates=20210119 \ git=1:2.30.2-1+deb11u2 \ - libpq-dev=13.14-0+deb11u1 \ + libpq-dev=13.18-0+deb11u1 \ make=4.3-4.1 \ openssh-client=1:8.4p1-5+deb11u3 \ software-properties-common=0.96.20.2-2.1 \