diff --git a/.changes/unreleased/Fixes-20241101-150335.yaml b/.changes/unreleased/Fixes-20241101-150335.yaml new file mode 100644 index 000000000..cbf43d01c --- /dev/null +++ b/.changes/unreleased/Fixes-20241101-150335.yaml @@ -0,0 +1,7 @@ +kind: Fixes +body: Remove `redshift_connector`'s `get_columns` method for column metadata generation, + avoiding a warning on every run +time: 2024-11-01T15:03:35.44697-04:00 +custom: + Author: "mikealfare" + Issue: "914" diff --git a/dbt/adapters/redshift/connections.py b/dbt/adapters/redshift/connections.py index 8e7ae36d2..d3fbcafea 100644 --- a/dbt/adapters/redshift/connections.py +++ b/dbt/adapters/redshift/connections.py @@ -3,7 +3,6 @@ from contextlib import contextmanager from typing import Any, Callable, Dict, Tuple, Union, Optional, List, TYPE_CHECKING from dataclasses import dataclass, field -import time import sqlparse import redshift_connector @@ -13,14 +12,10 @@ from dbt.adapters.sql import SQLConnectionManager from dbt.adapters.contracts.connection import AdapterResponse, Connection, Credentials from dbt.adapters.events.logging import AdapterLogger -from dbt.adapters.events.types import SQLQuery, SQLQueryStatus from dbt_common.contracts.util import Replaceable from dbt_common.dataclass_schema import dbtClassMixin, StrEnum, ValidationError -from dbt_common.events.contextvars import get_node_info -from dbt_common.events.functions import fire_event from dbt_common.helper_types import Port from dbt_common.exceptions import DbtRuntimeError, CompilationError, DbtDatabaseError -from dbt_common.utils import cast_to_str if TYPE_CHECKING: # Indirectly imported via agate_helper, which is lazy loaded further downfile. @@ -465,51 +460,3 @@ def _initialize_sqlparse_lexer(): if hasattr(Lexer, "get_default_instance"): Lexer.get_default_instance() - - def columns_in_relation(self, relation) -> List[Dict[str, Any]]: - connection = self.get_thread_connection() - - fire_event( - SQLQuery( - conn_name=cast_to_str(connection.name), - sql=f"call redshift_connector.Connection.get_columns({relation.database}, {relation.schema}, {relation.identifier})", - node_info=get_node_info(), - ) - ) - - pre = time.perf_counter() - - cursor = connection.handle.cursor() - columns = cursor.get_columns( - catalog=relation.database, - schema_pattern=relation.schema, - tablename_pattern=relation.identifier, - ) - - fire_event( - SQLQueryStatus( - status=str(self.get_response(cursor)), - elapsed=time.perf_counter() - pre, - node_info=get_node_info(), - ) - ) - - return [self._parse_column_results(column) for column in columns] - - @staticmethod - def _parse_column_results(record: Tuple[Any, ...]) -> Dict[str, Any]: - _, _, _, column_name, dtype_code, dtype_name, column_size, _, decimals, *_ = record - - char_dtypes = [1, 12] - num_dtypes = [2, 3, 4, 5, 6, 7, 8, -5, 2003] - - if dtype_code in char_dtypes: - return {"column": column_name, "dtype": dtype_name, "char_size": column_size} - elif dtype_code in num_dtypes: - return { - "column": column_name, - "dtype": dtype_name, - "numeric_precision": column_size, - "numeric_scale": decimals, - } - return {"column": column_name, "dtype": dtype_name, "char_size": column_size} diff --git a/dbt/adapters/redshift/impl.py b/dbt/adapters/redshift/impl.py index e0cefb989..e69c8c448 100644 --- a/dbt/adapters/redshift/impl.py +++ b/dbt/adapters/redshift/impl.py @@ -1,12 +1,10 @@ import os from dataclasses import dataclass -from dbt_common.behavior_flags import BehaviorFlag from dbt_common.contracts.constraints import ConstraintType -from typing import Optional, Set, Any, Dict, Type, TYPE_CHECKING, List +from typing import Optional, Set, Any, Dict, Type, TYPE_CHECKING from collections import namedtuple from dbt.adapters.base import PythonJobHelper -from dbt.adapters.base.column import Column from dbt.adapters.base.impl import AdapterConfig, ConstraintSupport from dbt.adapters.base.meta import available from dbt.adapters.capability import Capability, CapabilityDict, CapabilitySupport, Support @@ -68,23 +66,6 @@ class RedshiftAdapter(SQLAdapter): } ) - @property - def _behavior_flags(self) -> List[BehaviorFlag]: - return [ - { - "name": "restrict_direct_pg_catalog_access", - "default": False, - "description": ( - "The dbt-redshift adapter is migrating from using pg_ tables " - "to using Redshift Metadata API and information_schema tables " - "in order to support additional Redshift functionalities.\n" - "We do not expect this to impact your dbt experience. " - "Please report any issues using this GitHub discussion: https://github.com/dbt-labs/dbt-redshift/discussions/921" - ), - "docs_url": "https://docs.getdbt.com/reference/global-configs/behavior-changes#redshift-restrict_direct_pg_catalog_access", - } - ] - @classmethod def date_function(cls): return "getdate()" @@ -107,12 +88,6 @@ def drop_relation(self, relation): with self.connections.fresh_transaction(): return super().drop_relation(relation) - def get_columns_in_relation(self, relation) -> List[Column]: - if self.behavior.restrict_direct_pg_catalog_access: - column_configs = self.connections.columns_in_relation(relation) - return [Column(**column) for column in column_configs] - return super().get_columns_in_relation(relation) - @classmethod def convert_text_type(cls, agate_table: "agate.Table", col_idx): column = agate_table.columns[col_idx] diff --git a/tests/functional/test_columns_in_relation.py b/tests/functional/test_columns_in_relation.py index 608217a41..ca6de67c3 100644 --- a/tests/functional/test_columns_in_relation.py +++ b/tests/functional/test_columns_in_relation.py @@ -1,5 +1,3 @@ -import os - from dbt.adapters.base import Column from dbt.tests.util import run_dbt, run_dbt_and_capture import pytest @@ -47,6 +45,12 @@ def expected_columns(self): ] +@pytest.mark.skip( + """ + There is a discrepancy between our custom query and the get_columns SDK call. + This test should be skipped for now, but re-enabled once get_columns is implemented. +""" +) class TestColumnsInRelationBehaviorFlagOn(ColumnsInRelation): @pytest.fixture(scope="class") def project_config_update(self): @@ -74,6 +78,12 @@ def expected_columns(self): """ +@pytest.mark.skip( + """ + There is a discrepancy between our custom query and the get_columns SDK call. + This test should be skipped for now, but re-enabled once get_columns is implemented. +""" +) class TestBehaviorFlagFiresOnce: @pytest.fixture(scope="class") def project_config_update(self):