diff --git a/.changes/unreleased/Dependencies-20231002-164037.yaml b/.changes/unreleased/Dependencies-20231002-164037.yaml new file mode 100644 index 000000000..54f51a402 --- /dev/null +++ b/.changes/unreleased/Dependencies-20231002-164037.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Update ddtrace requirement from ~=1.19 to ~=1.20" +time: 2023-10-02T16:40:37.00000Z +custom: + Author: dependabot[bot] + PR: 622 diff --git a/.changes/unreleased/Dependencies-20231009-192801.yaml b/.changes/unreleased/Dependencies-20231009-192801.yaml new file mode 100644 index 000000000..2a53a074f --- /dev/null +++ b/.changes/unreleased/Dependencies-20231009-192801.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Update pre-commit-hooks requirement from ~=4.4 to ~=4.5" +time: 2023-10-09T19:28:01.00000Z +custom: + Author: dependabot[bot] + PR: 627 diff --git a/.changes/unreleased/Dependencies-20231010-195348.yaml b/.changes/unreleased/Dependencies-20231010-195348.yaml new file mode 100644 index 000000000..4e8af02c1 --- /dev/null +++ b/.changes/unreleased/Dependencies-20231010-195348.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Bump mypy from 1.5.1 to 1.6.0" +time: 2023-10-10T19:53:48.00000Z +custom: + Author: dependabot[bot] + PR: 629 diff --git a/.changes/unreleased/Dependencies-20231013-190517.yaml b/.changes/unreleased/Dependencies-20231013-190517.yaml new file mode 100644 index 000000000..8db17538a --- /dev/null +++ b/.changes/unreleased/Dependencies-20231013-190517.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Update pre-commit requirement from ~=3.4 to ~=3.5" +time: 2023-10-13T19:05:17.00000Z +custom: + Author: dependabot[bot] + PR: 634 diff --git a/.changes/unreleased/Dependencies-20231017-191545.yaml b/.changes/unreleased/Dependencies-20231017-191545.yaml new file mode 100644 index 000000000..407615e32 --- /dev/null +++ b/.changes/unreleased/Dependencies-20231017-191545.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Update black requirement from ~=23.9 to ~=23.10" +time: 2023-10-17T19:15:45.00000Z +custom: + Author: dependabot[bot] + PR: 636 diff --git a/.changes/unreleased/Dependencies-20231027-173152.yaml b/.changes/unreleased/Dependencies-20231027-173152.yaml new file mode 100644 index 000000000..8ed464977 --- /dev/null +++ b/.changes/unreleased/Dependencies-20231027-173152.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Bump mypy from 1.6.0 to 1.6.1" +time: 2023-10-27T17:31:52.00000Z +custom: + Author: dependabot[bot] + PR: 648 diff --git a/.changes/unreleased/Dependencies-20231030-193514.yaml b/.changes/unreleased/Dependencies-20231030-193514.yaml new file mode 100644 index 000000000..e2771051b --- /dev/null +++ b/.changes/unreleased/Dependencies-20231030-193514.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Update ddtrace requirement from ~=1.20 to ~=2.1" +time: 2023-10-30T19:35:14.00000Z +custom: + Author: dependabot[bot] + PR: 651 diff --git a/.changes/unreleased/Features-20231030-101055.yaml b/.changes/unreleased/Features-20231030-101055.yaml new file mode 100644 index 000000000..8648762e2 --- /dev/null +++ b/.changes/unreleased/Features-20231030-101055.yaml @@ -0,0 +1,6 @@ +kind: Features +body: allow user to set debug level for redshift-connector via env var +time: 2023-10-30T10:10:55.976191-07:00 +custom: + Author: colin-rogers-dbt + Issue: "650" diff --git a/.changes/unreleased/Fixes-20231025-203732.yaml b/.changes/unreleased/Fixes-20231025-203732.yaml new file mode 100644 index 000000000..9e6bf1af7 --- /dev/null +++ b/.changes/unreleased/Fixes-20231025-203732.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Fix parsing of database results for materialized view auto refresh +time: 2023-10-25T20:37:32.191259-04:00 +custom: + Author: mikealfare + Issue: "643" diff --git a/.changes/unreleased/Fixes-20231030-234315.yaml b/.changes/unreleased/Fixes-20231030-234315.yaml new file mode 100644 index 000000000..823e252dd --- /dev/null +++ b/.changes/unreleased/Fixes-20231030-234315.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Catalog queries assign the appropriate type for materialized views +time: 2023-10-30T23:43:15.281772-04:00 +custom: + Author: mikealfare + Issue: "652" diff --git a/dbt/adapters/redshift/impl.py b/dbt/adapters/redshift/impl.py index ae9f18392..fbb30c784 100644 --- a/dbt/adapters/redshift/impl.py +++ b/dbt/adapters/redshift/impl.py @@ -1,3 +1,4 @@ +import os from dataclasses import dataclass from typing import Optional, Set, Any, Dict, Type from collections import namedtuple @@ -14,9 +15,15 @@ from dbt.adapters.redshift import RedshiftConnectionManager, RedshiftRelation - logger = AdapterLogger("Redshift") - +packages = ["redshift_connector", "redshift_connector.core"] +if os.getenv("DBT_REDSHIFT_CONNECTOR_DEBUG_LOGGING"): + level = "DEBUG" +else: + level = "ERROR" +for package in packages: + logger.debug(f"Setting {package} to {level}") + logger.set_adapter_dependency_log_level(package, level) GET_RELATIONS_MACRO_NAME = "redshift__get_relations" diff --git a/dbt/adapters/redshift/relation_configs/materialized_view.py b/dbt/adapters/redshift/relation_configs/materialized_view.py index 1c6fb229a..44e18e3ac 100644 --- a/dbt/adapters/redshift/relation_configs/materialized_view.py +++ b/dbt/adapters/redshift/relation_configs/materialized_view.py @@ -182,10 +182,14 @@ def parse_relation_results(cls, relation_results: RelationResults) -> dict: "mv_name": materialized_view.get("table"), "schema_name": materialized_view.get("schema"), "database_name": materialized_view.get("database"), - "autorefresh": materialized_view.get("autorefresh"), "query": cls._parse_query(query.get("definition")), } + autorefresh_value = materialized_view.get("autorefresh") + if autorefresh_value is not None: + bool_filter = {"t": True, "f": False} + config_dict["autorefresh"] = bool_filter.get(autorefresh_value, autorefresh_value) + # the default for materialized views differs from the default for diststyle in general # only set it if we got a value if materialized_view.get("diststyle"): diff --git a/dbt/include/redshift/macros/catalog.sql b/dbt/include/redshift/macros/catalog.sql index b34460035..69dc71713 100644 --- a/dbt/include/redshift/macros/catalog.sql +++ b/dbt/include/redshift/macros/catalog.sql @@ -23,13 +23,28 @@ order by "column_index" ), + materialized_views as ( + select + table_schema as nspname, + table_name as relname + from information_schema.views + where ( + {%- for schema in schemas -%} + upper(table_schema) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} + {%- endfor -%} + ) + and table_catalog = '{{ database }}' + and view_definition ilike '%create materialized view%' + ), + early_binding as ( select '{{ database }}'::varchar as table_database, sch.nspname as table_schema, tbl.relname as table_name, - case tbl.relkind - when 'v' then 'VIEW' + case + when tbl.relkind = 'v' and materialized_views.relname is not null then 'MATERIALIZED VIEW' + when tbl.relkind = 'v' then 'VIEW' else 'BASE TABLE' end as table_type, tbl_desc.description as table_comment, @@ -43,6 +58,7 @@ join pg_catalog.pg_attribute col on col.attrelid = tbl.oid left outer join pg_catalog.pg_description tbl_desc on (tbl_desc.objoid = tbl.oid and tbl_desc.objsubid = 0) left outer join pg_catalog.pg_description col_desc on (col_desc.objoid = tbl.oid and col_desc.objsubid = col.attnum) + left outer join materialized_views on (materialized_views.nspname = sch.nspname and materialized_views.relname = tbl.relname) where ( {%- for schema in schemas -%} upper(sch.nspname) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} diff --git a/dev-requirements.txt b/dev-requirements.txt index 4b3b09cda..b55f4521d 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -6,18 +6,18 @@ git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-postgres&subdirectory=plugi # if version 1.x or greater -> pin to major version # if version 0.x -> pin to minor -black~=23.9 +black~=23.10 bumpversion~=0.6.0 click~=8.1 -ddtrace~=1.19 +ddtrace~=2.1 flake8~=6.1 flaky~=3.7 freezegun~=1.2 ipdb~=0.13.13 -mypy==1.5.1 # patch updates have historically introduced breaking changes +mypy==1.6.1 # patch updates have historically introduced breaking changes pip-tools~=7.3 -pre-commit~=3.4 -pre-commit-hooks~=4.4 +pre-commit~=3.5 +pre-commit-hooks~=4.5 pytest~=7.4 pytest-csv~=3.0 pytest-dotenv~=0.5.2 diff --git a/tests/functional/adapter/catalog_tests/files.py b/tests/functional/adapter/catalog_tests/files.py new file mode 100644 index 000000000..9c19522e7 --- /dev/null +++ b/tests/functional/adapter/catalog_tests/files.py @@ -0,0 +1,33 @@ +MY_SEED = """ +id,value,record_valid_date +1,100,2023-01-01 00:00:00 +2,200,2023-01-02 00:00:00 +3,300,2023-01-02 00:00:00 +""".strip() + + +MY_TABLE = """ +{{ config( + materialized='table', +) }} +select * +from {{ ref('my_seed') }} +""" + + +MY_VIEW = """ +{{ config( + materialized='view', +) }} +select * +from {{ ref('my_seed') }} +""" + + +MY_MATERIALIZED_VIEW = """ +{{ config( + materialized='materialized_view', +) }} +select * +from {{ ref('my_seed') }} +""" diff --git a/tests/functional/adapter/catalog_tests/test_relation_types.py b/tests/functional/adapter/catalog_tests/test_relation_types.py new file mode 100644 index 000000000..9b9156dec --- /dev/null +++ b/tests/functional/adapter/catalog_tests/test_relation_types.py @@ -0,0 +1,44 @@ +from dbt.contracts.results import CatalogArtifact +from dbt.tests.util import run_dbt +import pytest + +from tests.functional.adapter.catalog_tests import files + + +class TestCatalogRelationTypes: + @pytest.fixture(scope="class", autouse=True) + def seeds(self): + return {"my_seed.csv": files.MY_SEED} + + @pytest.fixture(scope="class", autouse=True) + def models(self): + yield { + "my_table.sql": files.MY_TABLE, + "my_view.sql": files.MY_VIEW, + "my_materialized_view.sql": files.MY_MATERIALIZED_VIEW, + } + + @pytest.fixture(scope="class", autouse=True) + def docs(self, project): + run_dbt(["seed"]) + run_dbt(["run"]) + yield run_dbt(["docs", "generate"]) + + @pytest.mark.parametrize( + "node_name,relation_type", + [ + ("seed.test.my_seed", "BASE TABLE"), + ("model.test.my_table", "BASE TABLE"), + ("model.test.my_view", "VIEW"), + ("model.test.my_materialized_view", "MATERIALIZED VIEW"), + ], + ) + def test_relation_types_populate_correctly( + self, docs: CatalogArtifact, node_name: str, relation_type: str + ): + """ + This test addresses: https://github.com/dbt-labs/dbt-redshift/issues/652 + """ + assert node_name in docs.nodes + node = docs.nodes[node_name] + assert node.metadata.type == relation_type diff --git a/tests/functional/adapter/materialized_view_tests/test_materialized_views.py b/tests/functional/adapter/materialized_view_tests/test_materialized_views.py index cf7d433c7..d2bf1cda4 100644 --- a/tests/functional/adapter/materialized_view_tests/test_materialized_views.py +++ b/tests/functional/adapter/materialized_view_tests/test_materialized_views.py @@ -163,19 +163,6 @@ def test_change_is_applied_via_alter_str_true(self, project, my_materialized_vie assert_message_in_logs(f"Applying ALTER to: {my_materialized_view}", logs) assert_message_in_logs(f"Applying REPLACE to: {my_materialized_view}", logs, False) - def test_change_is_applied_via_alter_str_false(self, project, my_materialized_view): - self.check_start_state(project, my_materialized_view) - - self.change_config_via_alter_str_false(project, my_materialized_view) - _, logs = run_dbt_and_capture_with_retries_redshift_mv( - ["--debug", "run", "--models", my_materialized_view.name] - ) - - self.check_state_alter_change_is_applied_str_false(project, my_materialized_view) - - assert_message_in_logs(f"Applying ALTER to: {my_materialized_view}", logs) - assert_message_in_logs(f"Applying REPLACE to: {my_materialized_view}", logs, False) - def test_change_is_applied_via_replace(self, project, my_materialized_view): self.check_start_state(project, my_materialized_view)