From 361d1968b00ffb9ff6db6e26b668c601f4240bc4 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Thu, 16 May 2024 22:43:56 +0100 Subject: [PATCH 01/21] Allow enabling/disabling stats collection on catalog table using env var --- .changes/unreleased/Fixes-20240516-224134.yaml | 6 ++++++ dbt/include/snowflake/macros/catalog.sql | 11 +++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 .changes/unreleased/Fixes-20240516-224134.yaml diff --git a/.changes/unreleased/Fixes-20240516-224134.yaml b/.changes/unreleased/Fixes-20240516-224134.yaml new file mode 100644 index 000000000..7efc4c722 --- /dev/null +++ b/.changes/unreleased/Fixes-20240516-224134.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Allow enabling/disabling stats collection on catalog table using env var +time: 2024-05-16T22:41:34.256095+01:00 +custom: + Author: aranke + Issue: "1048" diff --git a/dbt/include/snowflake/macros/catalog.sql b/dbt/include/snowflake/macros/catalog.sql index bde8b8f8f..c35805d33 100644 --- a/dbt/include/snowflake/macros/catalog.sql +++ b/dbt/include/snowflake/macros/catalog.sql @@ -37,6 +37,8 @@ {% macro snowflake__get_catalog_tables_sql(information_schema) -%} + {%- set enable_stats = env_var('DBT_CATALOG_TABLE_ENABLE_STATS', 'true') == 'true' -%} + select table_catalog as "table_database", table_schema as "table_schema", @@ -46,11 +48,10 @@ else table_type end as "table_type", comment as "table_comment", + table_owner as "table_owner" - -- note: this is the _role_ that owns the table - table_owner as "table_owner", - - 'Clustering Key' as "stats:clustering_key:label", + {%- if enable_stats %} + ,'Clustering Key' as "stats:clustering_key:label", clustering_key as "stats:clustering_key:value", 'The key used to cluster this table' as "stats:clustering_key:description", (clustering_key is not null) as "stats:clustering_key:include", @@ -69,6 +70,8 @@ to_varchar(convert_timezone('UTC', last_altered), 'yyyy-mm-dd HH24:MI'||'UTC') as "stats:last_modified:value", 'The timestamp for last update/change' as "stats:last_modified:description", (last_altered is not null and table_type='BASE TABLE') as "stats:last_modified:include" + {%- endif %} + from {{ information_schema }}.tables {%- endmacro %} From 4970ddfa0858cab4143a4e5daa122121bdc24677 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Tue, 28 May 2024 11:09:22 +0100 Subject: [PATCH 02/21] Implement get_relation_metadata macro --- dbt/adapters/snowflake/impl.py | 64 +++++++++++++++++++++++ dbt/include/snowflake/macros/adapters.sql | 9 ++++ tests/unit/mock_adapter.py | 8 ++- 3 files changed, 79 insertions(+), 2 deletions(-) diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index ebb15f753..617922156 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -10,7 +10,9 @@ from dbt.adapters.sql.impl import ( LIST_SCHEMAS_MACRO_NAME, LIST_RELATIONS_MACRO_NAME, + GET_RELATION_METADATA_NAME, ) +from dbt.artifacts.schemas.catalog import TableMetadata, StatsDict, StatsItem from dbt.adapters.snowflake import SnowflakeConnectionManager from dbt.adapters.snowflake import SnowflakeRelation @@ -54,6 +56,7 @@ class SnowflakeAdapter(SQLAdapter): Capability.SchemaMetadataByRelations: CapabilitySupport(support=Support.Full), Capability.TableLastModifiedMetadata: CapabilitySupport(support=Support.Full), Capability.TableLastModifiedMetadataBatch: CapabilitySupport(support=Support.Full), + Capability.GetRelationMetadata: CapabilitySupport(support=Support.Full), } ) @@ -129,6 +132,67 @@ def get_columns_in_relation(self, relation): else: raise + def get_relation_metadata( + self, relation: SnowflakeRelation + ) -> Tuple[Optional[TableMetadata], StatsDict]: + kwargs = {"relation": relation} + try: + results = self.execute_macro( + GET_RELATION_METADATA_NAME, kwargs=kwargs, needs_conn=True + ) + except DbtDatabaseError as exc: + # If we don't have permissions to the object, return empty + if "Object does not exist" in str(exc): + return None, {} + + # If we don't get any results, return empty + if len(results) == 0: + return None, {} + + row = results[0] + + is_dynamic = row.get("is_dynamic") + kind = row.get("kind") + + if is_dynamic == "YES" and kind == "BASE TABLE": + table_type = "DYNAMIC TABLE" + else: + table_type = kind + + table_metadata = TableMetadata( + type=table_type, + schema=row.get("schema_name"), + name=row.get("name"), + database=row.get("database_name"), + comment=row.get("comment"), + owner=row.get("owner"), + ) + stats_dict: StatsDict = { + "has_stats": StatsItem( + id="has_stats", + label="Has Stats?", + value=True, + include=False, + description="Indicates whether there are statistics for this table", + ), + "row_count": StatsItem( + id="row_count", + label="Row Count", + value=row.get("rows"), + include=True, + description="Number of rows in the table as reported by Snowflake", + ), + "bytes": StatsItem( + id="bytes", + label="Approximate Size", + value=row.get("bytes"), + include=True, + description="Size of the table as reported by Snowflake", + ), + } + + return table_metadata, stats_dict + def list_relations_without_caching( self, schema_relation: SnowflakeRelation ) -> List[SnowflakeRelation]: diff --git a/dbt/include/snowflake/macros/adapters.sql b/dbt/include/snowflake/macros/adapters.sql index 157738187..6edcdbcba 100644 --- a/dbt/include/snowflake/macros/adapters.sql +++ b/dbt/include/snowflake/macros/adapters.sql @@ -47,6 +47,15 @@ {% do return(columns) %} {% endmacro %} +{% macro snowflake__get_relation_metadata(relation) %} + {%- set sql -%} + show objects like '{{ relation.identifier }}' in {{ relation.database }}.{{ relation.schema }} limit 1 + {%- endset -%} + + {%- set result = run_query(sql) -%} + {{ return(result) }} +{% endmacro %} + {% macro snowflake__list_schemas(database) -%} {# 10k limit from here: https://docs.snowflake.net/manuals/sql-reference/sql/show-schemas.html#usage-notes #} {% set maximum = 10000 %} diff --git a/tests/unit/mock_adapter.py b/tests/unit/mock_adapter.py index d3bdf87b2..4a53e8935 100644 --- a/tests/unit/mock_adapter.py +++ b/tests/unit/mock_adapter.py @@ -1,7 +1,8 @@ +from contextlib import contextmanager +from typing import List from unittest import mock -from dbt.adapters.base import BaseAdapter -from contextlib import contextmanager +from dbt.adapters.base import BaseAdapter, BaseRelation def adapter_factory(): @@ -33,6 +34,9 @@ def rename_relation(self, *args, **kwargs): def get_columns_in_relation(self, *args, **kwargs): return self.responder.get_columns_in_relation(*args, **kwargs) + def get_relation_metadata(self, relation: BaseRelation) -> List[BaseRelation]: + return self.responder.get_relation_metadata(relation) + def expand_column_types(self, *args, **kwargs): return self.responder.expand_column_types(*args, **kwargs) From be990e12ccd645392ec069f6349c14a30b210a99 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Tue, 28 May 2024 17:14:03 +0100 Subject: [PATCH 03/21] Discard changes to dbt/include/snowflake/macros/catalog.sql --- dbt/include/snowflake/macros/catalog.sql | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/dbt/include/snowflake/macros/catalog.sql b/dbt/include/snowflake/macros/catalog.sql index c35805d33..bde8b8f8f 100644 --- a/dbt/include/snowflake/macros/catalog.sql +++ b/dbt/include/snowflake/macros/catalog.sql @@ -37,8 +37,6 @@ {% macro snowflake__get_catalog_tables_sql(information_schema) -%} - {%- set enable_stats = env_var('DBT_CATALOG_TABLE_ENABLE_STATS', 'true') == 'true' -%} - select table_catalog as "table_database", table_schema as "table_schema", @@ -48,10 +46,11 @@ else table_type end as "table_type", comment as "table_comment", - table_owner as "table_owner" - {%- if enable_stats %} - ,'Clustering Key' as "stats:clustering_key:label", + -- note: this is the _role_ that owns the table + table_owner as "table_owner", + + 'Clustering Key' as "stats:clustering_key:label", clustering_key as "stats:clustering_key:value", 'The key used to cluster this table' as "stats:clustering_key:description", (clustering_key is not null) as "stats:clustering_key:include", @@ -70,8 +69,6 @@ to_varchar(convert_timezone('UTC', last_altered), 'yyyy-mm-dd HH24:MI'||'UTC') as "stats:last_modified:value", 'The timestamp for last update/change' as "stats:last_modified:description", (last_altered is not null and table_type='BASE TABLE') as "stats:last_modified:include" - {%- endif %} - from {{ information_schema }}.tables {%- endmacro %} From 1783849f68f6d4a2116701d2659c85857c0b6776 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Wed, 29 May 2024 15:19:42 +0100 Subject: [PATCH 04/21] Import artifacts from dbt_common --- dbt/adapters/snowflake/impl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index 463fc06b1..325c316f8 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -12,7 +12,7 @@ LIST_RELATIONS_MACRO_NAME, GET_RELATION_METADATA_NAME, ) -from dbt.artifacts.schemas.catalog import TableMetadata, StatsDict, StatsItem +from dbt_common.artifacts.schemas.catalog import TableMetadata, StatsDict, StatsItem from dbt.adapters.snowflake import SnowflakeConnectionManager from dbt.adapters.snowflake import SnowflakeRelation From d19e3efb6d5a8933f38f72620c108b736cc7ffe4 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Wed, 29 May 2024 17:15:42 +0100 Subject: [PATCH 05/21] fix dbt_common import --- dbt/adapters/snowflake/impl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index 325c316f8..158fb22ee 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -12,7 +12,7 @@ LIST_RELATIONS_MACRO_NAME, GET_RELATION_METADATA_NAME, ) -from dbt_common.artifacts.schemas.catalog import TableMetadata, StatsDict, StatsItem +from dbt_common.artifacts.catalog import TableMetadata, StatsDict, StatsItem from dbt.adapters.snowflake import SnowflakeConnectionManager from dbt.adapters.snowflake import SnowflakeRelation From 2213a163feb02a0df1d94467831daa4d9774adb2 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Wed, 29 May 2024 21:05:20 +0100 Subject: [PATCH 06/21] Use get_relation_metadata from dbt-common --- dbt/adapters/snowflake/impl.py | 2 +- setup.py | 2 +- tests/unit/mock_adapter.py | 7 +++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index 158fb22ee..d4b74e801 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -12,7 +12,7 @@ LIST_RELATIONS_MACRO_NAME, GET_RELATION_METADATA_NAME, ) -from dbt_common.artifacts.catalog import TableMetadata, StatsDict, StatsItem +from dbt_common.contracts.metadata import TableMetadata, StatsDict, StatsItem from dbt.adapters.snowflake import SnowflakeConnectionManager from dbt.adapters.snowflake import SnowflakeRelation diff --git a/setup.py b/setup.py index f8ff363ed..1497609fd 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ def _plugin_version() -> str: packages=find_namespace_packages(include=["dbt", "dbt.*"]), include_package_data=True, install_requires=[ - "dbt-common>=1.0.4,<2.0", + "dbt-common>=1.2.0,<2.0", "dbt-adapters>=1.1.1,<2.0", "snowflake-connector-python[secure-local-storage]~=3.0", # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency diff --git a/tests/unit/mock_adapter.py b/tests/unit/mock_adapter.py index 4a53e8935..04d83ce3c 100644 --- a/tests/unit/mock_adapter.py +++ b/tests/unit/mock_adapter.py @@ -1,8 +1,9 @@ from contextlib import contextmanager -from typing import List +from typing import Tuple, Optional from unittest import mock from dbt.adapters.base import BaseAdapter, BaseRelation +from dbt_common.contracts.metadata import TableMetadata, StatsDict def adapter_factory(): @@ -34,7 +35,9 @@ def rename_relation(self, *args, **kwargs): def get_columns_in_relation(self, *args, **kwargs): return self.responder.get_columns_in_relation(*args, **kwargs) - def get_relation_metadata(self, relation: BaseRelation) -> List[BaseRelation]: + def get_relation_metadata( + self, relation: BaseRelation + ) -> Tuple[Optional[TableMetadata], StatsDict]: return self.responder.get_relation_metadata(relation) def expand_column_types(self, *args, **kwargs): From efb35951ae35945800d7bdad75770c8837ef1235 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Wed, 29 May 2024 21:09:50 +0100 Subject: [PATCH 07/21] Update Fixes-20240516-224134.yaml --- .changes/unreleased/Fixes-20240516-224134.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changes/unreleased/Fixes-20240516-224134.yaml b/.changes/unreleased/Fixes-20240516-224134.yaml index 7efc4c722..794b5d4a6 100644 --- a/.changes/unreleased/Fixes-20240516-224134.yaml +++ b/.changes/unreleased/Fixes-20240516-224134.yaml @@ -1,5 +1,5 @@ kind: Fixes -body: Allow enabling/disabling stats collection on catalog table using env var +body: Implement get_relation_metadata macro time: 2024-05-16T22:41:34.256095+01:00 custom: Author: aranke From 1944fb71010a9c34a262f1c3fb41bb3e96be2434 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Wed, 29 May 2024 21:19:32 +0100 Subject: [PATCH 08/21] use snowflakerelation --- tests/unit/mock_adapter.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/unit/mock_adapter.py b/tests/unit/mock_adapter.py index 04d83ce3c..e0f0f8057 100644 --- a/tests/unit/mock_adapter.py +++ b/tests/unit/mock_adapter.py @@ -2,9 +2,11 @@ from typing import Tuple, Optional from unittest import mock -from dbt.adapters.base import BaseAdapter, BaseRelation +from dbt.adapters.base import BaseAdapter from dbt_common.contracts.metadata import TableMetadata, StatsDict +from dbt.adapters.snowflake import SnowflakeRelation + def adapter_factory(): class MockAdapter(BaseAdapter): @@ -36,7 +38,7 @@ def get_columns_in_relation(self, *args, **kwargs): return self.responder.get_columns_in_relation(*args, **kwargs) def get_relation_metadata( - self, relation: BaseRelation + self, relation: SnowflakeRelation ) -> Tuple[Optional[TableMetadata], StatsDict]: return self.responder.get_relation_metadata(relation) From 8fa4f1d74b28501a599aa57e3bea66bb6edcedfc Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Wed, 5 Jun 2024 16:49:27 +0100 Subject: [PATCH 09/21] add broken test --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 1497609fd..468097e9a 100644 --- a/setup.py +++ b/setup.py @@ -57,8 +57,8 @@ def _plugin_version() -> str: packages=find_namespace_packages(include=["dbt", "dbt.*"]), include_package_data=True, install_requires=[ - "dbt-common>=1.2.0,<2.0", - "dbt-adapters>=1.1.1,<2.0", + # "dbt-common>=1.2.0,<2.0", + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@get_relation_metadata", "snowflake-connector-python[secure-local-storage]~=3.0", # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency "dbt-core>=1.8.0", From f76e29b8de97315e6b1054d2f5b707a23ffe07e1 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Thu, 6 Jun 2024 15:12:41 +0200 Subject: [PATCH 10/21] wip --- .gitignore | 1 + dbt/adapters/snowflake/impl.py | 15 +++++++++++---- dev-requirements.txt | 6 +++--- tests/functional/adapter/test_basic.py | 7 +++++++ 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 780d98f70..de95d4777 100644 --- a/.gitignore +++ b/.gitignore @@ -94,3 +94,4 @@ venv/ # vscode .vscode/ +.venv/ diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index d4b74e801..a7d0ce8c7 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -2,6 +2,7 @@ from typing import Mapping, Any, Optional, List, Union, Dict, FrozenSet, Tuple import agate +from dbt.adapters.base import BaseRelation from dbt.adapters.base.impl import AdapterConfig, ConstraintSupport from dbt.adapters.base.meta import available @@ -10,9 +11,9 @@ from dbt.adapters.sql.impl import ( LIST_SCHEMAS_MACRO_NAME, LIST_RELATIONS_MACRO_NAME, - GET_RELATION_METADATA_NAME, + GET_CATALOG_FOR_SINGLE_RELATION_NAME, ) -from dbt_common.contracts.metadata import TableMetadata, StatsDict, StatsItem +from dbt_common.contracts.metadata import TableMetadata, StatsDict, StatsItem, CatalogTable from dbt.adapters.snowflake import SnowflakeConnectionManager from dbt.adapters.snowflake import SnowflakeRelation @@ -56,7 +57,7 @@ class SnowflakeAdapter(SQLAdapter): Capability.SchemaMetadataByRelations: CapabilitySupport(support=Support.Full), Capability.TableLastModifiedMetadata: CapabilitySupport(support=Support.Full), Capability.TableLastModifiedMetadataBatch: CapabilitySupport(support=Support.Full), - Capability.GetRelationMetadata: CapabilitySupport(support=Support.Full), + Capability.GetCatalogForSingleRelation: CapabilitySupport(support=Support.Full), } ) @@ -132,13 +133,19 @@ def get_columns_in_relation(self, relation): else: raise + def get_catalog_for_single_relation(self, relation: BaseRelation) -> Optional[CatalogTable]: + kwargs = {"relation": relation} + return self.execute_macro( + GET_CATALOG_FOR_SINGLE_RELATION_NAME, kwargs=kwargs, needs_conn=True + ) + def get_relation_metadata( self, relation: SnowflakeRelation ) -> Tuple[Optional[TableMetadata], StatsDict]: kwargs = {"relation": relation} try: results = self.execute_macro( - GET_RELATION_METADATA_NAME, kwargs=kwargs, needs_conn=True + GET_CATALOG_FOR_SINGLE_RELATION_NAME, kwargs=kwargs, needs_conn=True ) except DbtDatabaseError as exc: # If we don't have permissions to the object, return empty diff --git a/dev-requirements.txt b/dev-requirements.txt index 0906cba2e..f0ce2409d 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,8 +1,8 @@ # install latest changes in dbt-core git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-adapters.git -git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter -git+https://github.com/dbt-labs/dbt-common.git +git+https://github.com/dbt-labs/dbt-adapters.git@get_relation_metadata +git+https://github.com/dbt-labs/dbt-adapters.git@get_relation_metadata#subdirectory=dbt-tests-adapter +git+https://github.com/dbt-labs/dbt-common.git@main # dev ipdb~=0.13.13 diff --git a/tests/functional/adapter/test_basic.py b/tests/functional/adapter/test_basic.py index 45bdcf150..8602a6a2f 100644 --- a/tests/functional/adapter/test_basic.py +++ b/tests/functional/adapter/test_basic.py @@ -5,6 +5,9 @@ from dbt.tests.adapter.basic.test_singular_tests_ephemeral import ( BaseSingularTestsEphemeral, ) +from dbt.tests.adapter.basic.test_get_catalog_for_single_relation import ( + BaseGetCatalogForSingleRelation, +) from dbt.tests.adapter.basic.test_empty import BaseEmpty from dbt.tests.adapter.basic.test_ephemeral import BaseEphemeral from dbt.tests.adapter.basic.test_incremental import BaseIncremental @@ -25,6 +28,10 @@ class TestSingularTestsSnowflake(BaseSingularTests): pass +class TestGetCatalogForSingleRelationSnowflake(BaseGetCatalogForSingleRelation): + pass + + class TestSingularTestsEphemeralSnowflake(BaseSingularTestsEphemeral): pass From 6e09133d14b4ef0c7dbddfc537fdaf6e26c4071b Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Tue, 11 Jun 2024 17:34:01 +0100 Subject: [PATCH 11/21] add test for get_catalog_for_single_relation --- dbt/adapters/snowflake/impl.py | 144 ++++++++++++---------- dbt/include/snowflake/macros/adapters.sql | 2 +- tests/unit/mock_adapter.py | 10 +- 3 files changed, 83 insertions(+), 73 deletions(-) diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index a7d0ce8c7..f7715c9bd 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -3,7 +3,6 @@ import agate from dbt.adapters.base import BaseRelation - from dbt.adapters.base.impl import AdapterConfig, ConstraintSupport from dbt.adapters.base.meta import available from dbt.adapters.capability import CapabilityDict, CapabilitySupport, Support, Capability @@ -11,17 +10,24 @@ from dbt.adapters.sql.impl import ( LIST_SCHEMAS_MACRO_NAME, LIST_RELATIONS_MACRO_NAME, - GET_CATALOG_FOR_SINGLE_RELATION_NAME, ) -from dbt_common.contracts.metadata import TableMetadata, StatsDict, StatsItem, CatalogTable - -from dbt.adapters.snowflake import SnowflakeConnectionManager -from dbt.adapters.snowflake import SnowflakeRelation -from dbt.adapters.snowflake import SnowflakeColumn from dbt_common.contracts.constraints import ConstraintType +from dbt_common.contracts.metadata import ( + TableMetadata, + StatsDict, + StatsItem, + CatalogTable, + ColumnMetadata, +) from dbt_common.exceptions import CompilationError, DbtDatabaseError, DbtRuntimeError from dbt_common.utils import filter_null_values +from dbt.adapters.snowflake import SnowflakeColumn +from dbt.adapters.snowflake import SnowflakeConnectionManager +from dbt.adapters.snowflake import SnowflakeRelation + +SHOW_OBJECT_METADATA_MACRO_NAME = "snowflake__show_object_metadata" + @dataclass class SnowflakeConfig(AdapterConfig): @@ -133,72 +139,82 @@ def get_columns_in_relation(self, relation): else: raise - def get_catalog_for_single_relation(self, relation: BaseRelation) -> Optional[CatalogTable]: - kwargs = {"relation": relation} - return self.execute_macro( - GET_CATALOG_FOR_SINGLE_RELATION_NAME, kwargs=kwargs, needs_conn=True - ) - - def get_relation_metadata( - self, relation: SnowflakeRelation - ) -> Tuple[Optional[TableMetadata], StatsDict]: - kwargs = {"relation": relation} + def _show_object_metadata(self, relation): try: - results = self.execute_macro( - GET_CATALOG_FOR_SINGLE_RELATION_NAME, kwargs=kwargs, needs_conn=True - ) + kwargs = {"relation": relation} + results = self.execute_macro(SHOW_OBJECT_METADATA_MACRO_NAME, kwargs=kwargs) except DbtDatabaseError as exc: # If we don't have permissions to the object, return empty if "Object does not exist" in str(exc): - return None, {} + return None + else: + # If we don't get any results, return empty + if len(results) == 0: + return None - # If we don't get any results, return empty - if len(results) == 0: - return None, {} + return results - row = results[0] + def get_catalog_for_single_relation(self, relation: BaseRelation) -> Optional[CatalogTable]: + object_metadata = self._show_object_metadata(relation) - is_dynamic = row.get("is_dynamic") - kind = row.get("kind") + if object_metadata: + row = object_metadata[0] - if is_dynamic == "YES" and kind == "BASE TABLE": - table_type = "DYNAMIC TABLE" - else: - table_type = kind - - table_metadata = TableMetadata( - type=table_type, - schema=row.get("schema_name"), - name=row.get("name"), - database=row.get("database_name"), - comment=row.get("comment"), - owner=row.get("owner"), - ) - stats_dict: StatsDict = { - "has_stats": StatsItem( - id="has_stats", - label="Has Stats?", - value=True, - include=False, - description="Indicates whether there are statistics for this table", - ), - "row_count": StatsItem( - id="row_count", - label="Row Count", - value=row.get("rows"), - include=True, - description="Number of rows in the table as reported by Snowflake", - ), - "bytes": StatsItem( - id="bytes", - label="Approximate Size", - value=row.get("bytes"), - include=True, - description="Size of the table as reported by Snowflake", - ), - } + is_dynamic = row.get("is_dynamic") + kind = row.get("kind") - return table_metadata, stats_dict + if is_dynamic == "YES" and kind == "BASE TABLE": + table_type = "DYNAMIC TABLE" + else: + table_type = kind + + table_metadata = TableMetadata( + type=table_type, + schema=row.get("schema_name"), + name=row.get("name"), + database=row.get("database_name"), + comment=row.get("comment"), + owner=row.get("owner"), + ) + + stats_dict: StatsDict = { + "has_stats": StatsItem( + id="has_stats", + label="Has Stats?", + value=True, + include=False, + description="Indicates whether there are statistics for this table", + ), + "row_count": StatsItem( + id="row_count", + label="Row Count", + value=row.get("rows"), + include=True, + description="Number of rows in the table as reported by Snowflake", + ), + "bytes": StatsItem( + id="bytes", + label="Approximate Size", + value=row.get("bytes"), + include=True, + description="Size of the table as reported by Snowflake", + ), + } + + catalog_columns = {} + columns_metadata = self.get_columns_in_relation(relation) + + for i, c in enumerate(columns_metadata): + name = c.column + catalog_columns[name] = ColumnMetadata(type=c.dtype, index=i + 1, name=name) + + return CatalogTable( + metadata=table_metadata, + columns=catalog_columns, + stats=stats_dict, + ) + else: + return None def list_relations_without_caching( self, schema_relation: SnowflakeRelation diff --git a/dbt/include/snowflake/macros/adapters.sql b/dbt/include/snowflake/macros/adapters.sql index 827b78d78..3548b64fb 100644 --- a/dbt/include/snowflake/macros/adapters.sql +++ b/dbt/include/snowflake/macros/adapters.sql @@ -47,7 +47,7 @@ {% do return(columns) %} {% endmacro %} -{% macro snowflake__get_relation_metadata(relation) %} +{% macro snowflake__show_object_metadata(relation) %} {%- set sql -%} show objects like '{{ relation.identifier }}' in {{ relation.database }}.{{ relation.schema }} limit 1 {%- endset -%} diff --git a/tests/unit/mock_adapter.py b/tests/unit/mock_adapter.py index e0f0f8057..93394bedc 100644 --- a/tests/unit/mock_adapter.py +++ b/tests/unit/mock_adapter.py @@ -1,11 +1,7 @@ from contextlib import contextmanager -from typing import Tuple, Optional from unittest import mock from dbt.adapters.base import BaseAdapter -from dbt_common.contracts.metadata import TableMetadata, StatsDict - -from dbt.adapters.snowflake import SnowflakeRelation def adapter_factory(): @@ -37,10 +33,8 @@ def rename_relation(self, *args, **kwargs): def get_columns_in_relation(self, *args, **kwargs): return self.responder.get_columns_in_relation(*args, **kwargs) - def get_relation_metadata( - self, relation: SnowflakeRelation - ) -> Tuple[Optional[TableMetadata], StatsDict]: - return self.responder.get_relation_metadata(relation) + def get_catalog_for_single_relation(self, *args, **kwargs): + return self.responder.get_catalog_for_single_relation(*args, **kwargs) def expand_column_types(self, *args, **kwargs): return self.responder.expand_column_types(*args, **kwargs) From 9cf34971864d08a07805e7bc40fe103db54e4dec Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Tue, 18 Jun 2024 15:56:35 +0100 Subject: [PATCH 12/21] wip --- dbt/adapters/snowflake/impl.py | 14 +++-- tests/functional/adapter/test_basic.py | 86 +++++++++++++++++++++++++- 2 files changed, 91 insertions(+), 9 deletions(-) diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index f7715c9bd..0bea43bc5 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -6,6 +6,9 @@ from dbt.adapters.base.impl import AdapterConfig, ConstraintSupport from dbt.adapters.base.meta import available from dbt.adapters.capability import CapabilityDict, CapabilitySupport, Support, Capability +from dbt.adapters.snowflake import SnowflakeColumn +from dbt.adapters.snowflake import SnowflakeConnectionManager +from dbt.adapters.snowflake import SnowflakeRelation from dbt.adapters.sql import SQLAdapter from dbt.adapters.sql.impl import ( LIST_SCHEMAS_MACRO_NAME, @@ -22,10 +25,6 @@ from dbt_common.exceptions import CompilationError, DbtDatabaseError, DbtRuntimeError from dbt_common.utils import filter_null_values -from dbt.adapters.snowflake import SnowflakeColumn -from dbt.adapters.snowflake import SnowflakeConnectionManager -from dbt.adapters.snowflake import SnowflakeRelation - SHOW_OBJECT_METADATA_MACRO_NAME = "snowflake__show_object_metadata" @@ -168,6 +167,9 @@ def get_catalog_for_single_relation(self, relation: BaseRelation) -> Optional[Ca else: table_type = kind + # https://docs.snowflake.com/en/sql-reference/sql/show-views#output + is_view = kind in ("VIEW", "MATERIALIZED_VIEW") + table_metadata = TableMetadata( type=table_type, schema=row.get("schema_name"), @@ -189,14 +191,14 @@ def get_catalog_for_single_relation(self, relation: BaseRelation) -> Optional[Ca id="row_count", label="Row Count", value=row.get("rows"), - include=True, + include=(not is_view), description="Number of rows in the table as reported by Snowflake", ), "bytes": StatsItem( id="bytes", label="Approximate Size", value=row.get("bytes"), - include=True, + include=(not is_view), description="Size of the table as reported by Snowflake", ), } diff --git a/tests/functional/adapter/test_basic.py b/tests/functional/adapter/test_basic.py index 8602a6a2f..e55100c43 100644 --- a/tests/functional/adapter/test_basic.py +++ b/tests/functional/adapter/test_basic.py @@ -17,6 +17,9 @@ from dbt.tests.adapter.basic.test_adapter_methods import BaseAdapterMethod from dbt.tests.adapter.basic.test_docs_generate import BaseDocsGenerate from dbt.tests.adapter.basic.expected_catalog import base_expected_catalog, no_stats +from dbt_common.contracts.metadata import CatalogTable, TableMetadata, ColumnMetadata, StatsItem + +from dbt.adapters.snowflake.relation_configs import SnowflakeRelationType from tests.functional.adapter.expected_stats import snowflake_stats @@ -29,7 +32,86 @@ class TestSingularTestsSnowflake(BaseSingularTests): class TestGetCatalogForSingleRelationSnowflake(BaseGetCatalogForSingleRelation): - pass + @pytest.fixture(scope="class") + def current_role(self, project): + return project.run_sql("select current_role()", fetch="one")[0] + + @pytest.fixture(scope="class") + def expected_catalog_my_seed(self, project, current_role): + return CatalogTable( + metadata=TableMetadata( + type=SnowflakeRelationType.Table.upper(), + schema=project.test_schema.upper(), + name="MY_SEED", + database=project.database, + comment="", + owner=current_role, + ), + columns={ + "ID": ColumnMetadata(type="NUMBER", index=1, name="ID", comment=None), + "FIRST_NAME": ColumnMetadata( + type="VARCHAR", index=2, name="FIRST_NAME", comment=None + ), + "EMAIL": ColumnMetadata(type="VARCHAR", index=3, name="EMAIL", comment=None), + "IP_ADDRESS": ColumnMetadata( + type="VARCHAR", index=4, name="IP_ADDRESS", comment=None + ), + "UPDATED_AT": ColumnMetadata( + type="TIMESTAMP_NTZ", index=5, name="UPDATED_AT", comment=None + ), + }, + stats={ + "has_stats": StatsItem( + id="has_stats", + label="Has Stats?", + value=True, + include=False, + description="Indicates whether there are statistics for this table", + ), + "row_count": StatsItem( + id="row_count", + label="Row Count", + value=1, + include=True, + description="Number of rows in the table as reported by Snowflake", + ), + "bytes": StatsItem( + id="bytes", + label="Approximate Size", + value=2048, + include=True, + description="Size of the table as reported by Snowflake", + ), + }, + unique_id=None, + ) + + @pytest.fixture(scope="class") + def expected_catalog_my_model(self, project, current_role): + return CatalogTable( + metadata=TableMetadata( + type=SnowflakeRelationType.View, + schema=project.test_schema.upper(), + name="MY_MODEL", + database=project.database, + comment="", + owner=current_role, + ), + columns={ + "ID": ColumnMetadata(type="NUMBER", index=1, name="ID", comment=None), + "FIRST_NAME": ColumnMetadata( + type="VARCHAR", index=2, name="FIRST_NAME", comment=None + ), + "EMAIL": ColumnMetadata(type="VARCHAR", index=3, name="EMAIL", comment=None), + "IP_ADDRESS": ColumnMetadata( + type="VARCHAR", index=4, name="IP_ADDRESS", comment=None + ), + "UPDATED_AT": ColumnMetadata( + type="TIMESTAMP_NTZ", index=5, name="UPDATED_AT", comment=None + ), + }, + stats=snowflake_stats(), + ) class TestSingularTestsEphemeralSnowflake(BaseSingularTestsEphemeral): @@ -81,8 +163,6 @@ def expected_catalog(self, project, get_role): time_type="TIMESTAMP_NTZ", view_type="VIEW", table_type="BASE TABLE", - model_stats=no_stats(), - seed_stats=snowflake_stats(), case=lambda x: x.upper(), case_columns=False, ) From ab513512366ff2fd437dc169306ba1c38e971d48 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Tue, 18 Jun 2024 16:09:40 +0100 Subject: [PATCH 13/21] hardcode macro name --- dbt/adapters/snowflake/impl.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index d14d56fc6..e238cb5dd 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -8,7 +8,6 @@ from dbt.adapters.snowflake import SnowflakeColumn from dbt.adapters.snowflake import SnowflakeConnectionManager from dbt.adapters.snowflake import SnowflakeRelation -from dbt.adapters.snowflake.impl import SHOW_OBJECT_METADATA_MACRO_NAME from dbt.adapters.sql import SQLAdapter from dbt.adapters.sql.impl import ( LIST_SCHEMAS_MACRO_NAME, @@ -28,6 +27,8 @@ if TYPE_CHECKING: import agate +SHOW_OBJECT_METADATA_MACRO_NAME = "snowflake__show_object_metadata" + @dataclass class SnowflakeConfig(AdapterConfig): From 2f4510b3c4f1bf8ec5ab3940ac5994e5f605f4e4 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Tue, 18 Jun 2024 16:17:14 +0100 Subject: [PATCH 14/21] fix docs_generate test --- tests/functional/adapter/test_basic.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/functional/adapter/test_basic.py b/tests/functional/adapter/test_basic.py index e55100c43..c7b0d0b95 100644 --- a/tests/functional/adapter/test_basic.py +++ b/tests/functional/adapter/test_basic.py @@ -87,7 +87,7 @@ def expected_catalog_my_seed(self, project, current_role): ) @pytest.fixture(scope="class") - def expected_catalog_my_model(self, project, current_role): + def expected_catalog_my_view_model(self, project, current_role): return CatalogTable( metadata=TableMetadata( type=SnowflakeRelationType.View, @@ -163,6 +163,8 @@ def expected_catalog(self, project, get_role): time_type="TIMESTAMP_NTZ", view_type="VIEW", table_type="BASE TABLE", + model_stats=no_stats(), + seed_stats=snowflake_stats(), case=lambda x: x.upper(), case_columns=False, ) From 5aacd01f4f373ee5b18b85908bc2c3bbaca57a2b Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Tue, 18 Jun 2024 18:08:32 +0100 Subject: [PATCH 15/21] address pr comments --- dbt/adapters/snowflake/impl.py | 112 +++++++++++----------- dbt/include/snowflake/macros/adapters.sql | 2 +- tests/functional/adapter/test_basic.py | 79 ++++++++++++++- 3 files changed, 132 insertions(+), 61 deletions(-) diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index e238cb5dd..3434d9e7e 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -158,67 +158,65 @@ def _show_object_metadata(self, relation): def get_catalog_for_single_relation(self, relation: BaseRelation) -> Optional[CatalogTable]: object_metadata = self._show_object_metadata(relation) - if object_metadata: - row = object_metadata[0] + if not object_metadata: + return None - is_dynamic = row.get("is_dynamic") - kind = row.get("kind") + row = object_metadata[0] - if is_dynamic == "YES" and kind == "BASE TABLE": - table_type = "DYNAMIC TABLE" - else: - table_type = kind - - # https://docs.snowflake.com/en/sql-reference/sql/show-views#output - is_view = kind in ("VIEW", "MATERIALIZED_VIEW") - - table_metadata = TableMetadata( - type=table_type, - schema=row.get("schema_name"), - name=row.get("name"), - database=row.get("database_name"), - comment=row.get("comment"), - owner=row.get("owner"), - ) + is_dynamic = row.get("is_dynamic") + kind = row.get("kind") - stats_dict: StatsDict = { - "has_stats": StatsItem( - id="has_stats", - label="Has Stats?", - value=True, - include=False, - description="Indicates whether there are statistics for this table", - ), - "row_count": StatsItem( - id="row_count", - label="Row Count", - value=row.get("rows"), - include=(not is_view), - description="Number of rows in the table as reported by Snowflake", - ), - "bytes": StatsItem( - id="bytes", - label="Approximate Size", - value=row.get("bytes"), - include=(not is_view), - description="Size of the table as reported by Snowflake", - ), - } - - catalog_columns = {} - columns_metadata = self.get_columns_in_relation(relation) - - for i, c in enumerate(columns_metadata): - name = c.column - catalog_columns[name] = ColumnMetadata(type=c.dtype, index=i + 1, name=name) - - return CatalogTable( - metadata=table_metadata, - columns=catalog_columns, - stats=stats_dict, - ) + if is_dynamic == "YES" and kind == "BASE TABLE": + table_type = "DYNAMIC TABLE" else: - return None + table_type = kind + + # https://docs.snowflake.com/en/sql-reference/sql/show-views#output + is_view = kind in ("VIEW", "MATERIALIZED_VIEW") + + table_metadata = TableMetadata( + type=table_type, + schema=row.get("schema_name"), + name=row.get("name"), + database=row.get("database_name"), + comment=row.get("comment"), + owner=row.get("owner"), + ) + + stats_dict: StatsDict = { + "has_stats": StatsItem( + id="has_stats", + label="Has Stats?", + value=True, + include=False, + description="Indicates whether there are statistics for this table", + ), + "row_count": StatsItem( + id="row_count", + label="Row Count", + value=row.get("rows"), + include=(not is_view), + description="Number of rows in the table as reported by Snowflake", + ), + "bytes": StatsItem( + id="bytes", + label="Approximate Size", + value=row.get("bytes"), + include=(not is_view), + description="Size of the table as reported by Snowflake", + ), + } + + catalog_columns = { + c.column: ColumnMetadata(type=c.dtype, index=i + 1, name=c.column) + for i, c in enumerate(self.get_columns_in_relation(relation)) + } + + return CatalogTable( + metadata=table_metadata, + columns=catalog_columns, + stats=stats_dict, + ) def list_relations_without_caching( self, schema_relation: SnowflakeRelation diff --git a/dbt/include/snowflake/macros/adapters.sql b/dbt/include/snowflake/macros/adapters.sql index 3548b64fb..b858aec11 100644 --- a/dbt/include/snowflake/macros/adapters.sql +++ b/dbt/include/snowflake/macros/adapters.sql @@ -49,7 +49,7 @@ {% macro snowflake__show_object_metadata(relation) %} {%- set sql -%} - show objects like '{{ relation.identifier }}' in {{ relation.database }}.{{ relation.schema }} limit 1 + show objects like '{{ relation.identifier }}' in {{ relation.include(identifier=False) }} limit 1 {%- endset -%} {%- set result = run_query(sql) -%} diff --git a/tests/functional/adapter/test_basic.py b/tests/functional/adapter/test_basic.py index c7b0d0b95..1a79f672b 100644 --- a/tests/functional/adapter/test_basic.py +++ b/tests/functional/adapter/test_basic.py @@ -90,9 +90,9 @@ def expected_catalog_my_seed(self, project, current_role): def expected_catalog_my_view_model(self, project, current_role): return CatalogTable( metadata=TableMetadata( - type=SnowflakeRelationType.View, + type=SnowflakeRelationType.View.upper(), schema=project.test_schema.upper(), - name="MY_MODEL", + name="MY_VIEW_MODEL", database=project.database, comment="", owner=current_role, @@ -110,7 +110,80 @@ def expected_catalog_my_view_model(self, project, current_role): type="TIMESTAMP_NTZ", index=5, name="UPDATED_AT", comment=None ), }, - stats=snowflake_stats(), + stats={ + "has_stats": StatsItem( + id="has_stats", + label="Has Stats?", + value=True, + include=False, + description="Indicates whether there are statistics for this table", + ), + "row_count": StatsItem( + id="row_count", + label="Row Count", + value=0, + include=False, + description="Number of rows in the table as reported by Snowflake", + ), + "bytes": StatsItem( + id="bytes", + label="Approximate Size", + value=0, + include=False, + description="Size of the table as reported by Snowflake", + ), + }, + unique_id=None, + ) + + @pytest.fixture(scope="class") + def expected_catalog_my_table_model(self, project, current_role): + return CatalogTable( + metadata=TableMetadata( + type=SnowflakeRelationType.Table.upper(), + schema=project.test_schema.upper(), + name="MY_TABLE_MODEL", + database=project.database, + comment="", + owner=current_role, + ), + columns={ + "ID": ColumnMetadata(type="NUMBER", index=1, name="ID", comment=None), + "FIRST_NAME": ColumnMetadata( + type="VARCHAR", index=2, name="FIRST_NAME", comment=None + ), + "EMAIL": ColumnMetadata(type="VARCHAR", index=3, name="EMAIL", comment=None), + "IP_ADDRESS": ColumnMetadata( + type="VARCHAR", index=4, name="IP_ADDRESS", comment=None + ), + "UPDATED_AT": ColumnMetadata( + type="TIMESTAMP_NTZ", index=5, name="UPDATED_AT", comment=None + ), + }, + stats={ + "has_stats": StatsItem( + id="has_stats", + label="Has Stats?", + value=True, + include=False, + description="Indicates whether there are statistics for this table", + ), + "row_count": StatsItem( + id="row_count", + label="Row Count", + value=1, + include=True, + description="Number of rows in the table as reported by Snowflake", + ), + "bytes": StatsItem( + id="bytes", + label="Approximate Size", + value=2048, + include=True, + description="Size of the table as reported by Snowflake", + ), + }, + unique_id=None, ) From 126ba3283508c574d453430ada70e46f016a7c77 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Tue, 18 Jun 2024 18:15:25 +0100 Subject: [PATCH 16/21] update changie, add type sigs --- .../unreleased/Fixes-20240516-224134.yaml | 2 +- dbt/adapters/snowflake/impl.py | 23 +++++++++---------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/.changes/unreleased/Fixes-20240516-224134.yaml b/.changes/unreleased/Fixes-20240516-224134.yaml index 794b5d4a6..011ecb449 100644 --- a/.changes/unreleased/Fixes-20240516-224134.yaml +++ b/.changes/unreleased/Fixes-20240516-224134.yaml @@ -1,5 +1,5 @@ kind: Fixes -body: Implement get_relation_metadata macro +body: Get catalog metadata for a single relation in the most optimized way using the get_catalog_for_single_relation macro and capability time: 2024-05-16T22:41:34.256095+01:00 custom: Author: aranke diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index 3434d9e7e..816c090a5 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -1,13 +1,9 @@ from dataclasses import dataclass from typing import Mapping, Any, Optional, List, Union, Dict, FrozenSet, Tuple, TYPE_CHECKING -from dbt.adapters.base import BaseRelation from dbt.adapters.base.impl import AdapterConfig, ConstraintSupport from dbt.adapters.base.meta import available from dbt.adapters.capability import CapabilityDict, CapabilitySupport, Support, Capability -from dbt.adapters.snowflake import SnowflakeColumn -from dbt.adapters.snowflake import SnowflakeConnectionManager -from dbt.adapters.snowflake import SnowflakeRelation from dbt.adapters.sql import SQLAdapter from dbt.adapters.sql.impl import ( LIST_SCHEMAS_MACRO_NAME, @@ -24,6 +20,10 @@ from dbt_common.exceptions import CompilationError, DbtDatabaseError, DbtRuntimeError from dbt_common.utils import filter_null_values +from dbt.adapters.snowflake import SnowflakeColumn +from dbt.adapters.snowflake import SnowflakeConnectionManager +from dbt.adapters.snowflake import SnowflakeRelation + if TYPE_CHECKING: import agate @@ -140,22 +140,21 @@ def get_columns_in_relation(self, relation): else: raise - def _show_object_metadata(self, relation): + def _show_object_metadata(self, relation: SnowflakeRelation) -> Optional[dict]: try: kwargs = {"relation": relation} results = self.execute_macro(SHOW_OBJECT_METADATA_MACRO_NAME, kwargs=kwargs) - except DbtDatabaseError as exc: - # If we don't have permissions to the object, return empty - if "Object does not exist" in str(exc): - return None - else: - # If we don't get any results, return empty + if len(results) == 0: return None return results + except DbtDatabaseError: + return None - def get_catalog_for_single_relation(self, relation: BaseRelation) -> Optional[CatalogTable]: + def get_catalog_for_single_relation( + self, relation: SnowflakeRelation + ) -> Optional[CatalogTable]: object_metadata = self._show_object_metadata(relation) if not object_metadata: From a6cc1c7721e6e0ec2a3557db6b5c244e4c8de901 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Tue, 18 Jun 2024 19:19:27 +0100 Subject: [PATCH 17/21] Use enum values for comparison --- dbt/adapters/snowflake/impl.py | 12 ++++++++---- dbt/adapters/snowflake/relation_configs/policies.py | 1 + 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index 816c090a5..5da9a2cdb 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -20,6 +20,7 @@ from dbt_common.exceptions import CompilationError, DbtDatabaseError, DbtRuntimeError from dbt_common.utils import filter_null_values +from dbt.adapters.snowflake.relation_configs import SnowflakeRelationType from dbt.adapters.snowflake import SnowflakeColumn from dbt.adapters.snowflake import SnowflakeConnectionManager from dbt.adapters.snowflake import SnowflakeRelation @@ -162,16 +163,19 @@ def get_catalog_for_single_relation( row = object_metadata[0] - is_dynamic = row.get("is_dynamic") + is_dynamic = row.get("is_dynamic") in ("Y", "YES") kind = row.get("kind") - if is_dynamic == "YES" and kind == "BASE TABLE": - table_type = "DYNAMIC TABLE" + if is_dynamic and kind == str(SnowflakeRelationType.Table).upper(): + table_type = str(SnowflakeRelationType.DynamicTable).upper() else: table_type = kind # https://docs.snowflake.com/en/sql-reference/sql/show-views#output - is_view = kind in ("VIEW", "MATERIALIZED_VIEW") + is_view = kind in ( + str(SnowflakeRelationType.View).upper(), + str(SnowflakeRelationType.MaterializedView).upper(), + ) table_metadata = TableMetadata( type=table_type, diff --git a/dbt/adapters/snowflake/relation_configs/policies.py b/dbt/adapters/snowflake/relation_configs/policies.py index 75195f9a3..ecd08b18c 100644 --- a/dbt/adapters/snowflake/relation_configs/policies.py +++ b/dbt/adapters/snowflake/relation_configs/policies.py @@ -7,6 +7,7 @@ class SnowflakeRelationType(StrEnum): Table = "table" View = "view" + MaterializedView = "materialized_view" CTE = "cte" External = "external" DynamicTable = "dynamic_table" From 943be20a637bce1ef2fa2f9d5baccd589d3d2d53 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Tue, 18 Jun 2024 21:12:41 +0100 Subject: [PATCH 18/21] remove mv --- dbt/adapters/snowflake/impl.py | 6 ++---- dbt/adapters/snowflake/relation_configs/policies.py | 1 - 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index 5da9a2cdb..092510e8a 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -172,10 +172,8 @@ def get_catalog_for_single_relation( table_type = kind # https://docs.snowflake.com/en/sql-reference/sql/show-views#output - is_view = kind in ( - str(SnowflakeRelationType.View).upper(), - str(SnowflakeRelationType.MaterializedView).upper(), - ) + # Note: we don't support materialized views in dbt-snowflake + is_view = kind == str(SnowflakeRelationType.View).upper() table_metadata = TableMetadata( type=table_type, diff --git a/dbt/adapters/snowflake/relation_configs/policies.py b/dbt/adapters/snowflake/relation_configs/policies.py index ecd08b18c..75195f9a3 100644 --- a/dbt/adapters/snowflake/relation_configs/policies.py +++ b/dbt/adapters/snowflake/relation_configs/policies.py @@ -7,7 +7,6 @@ class SnowflakeRelationType(StrEnum): Table = "table" View = "view" - MaterializedView = "materialized_view" CTE = "cte" External = "external" DynamicTable = "dynamic_table" From 7a3ea2635b0fcf892c7f774927a04ab2a016a5fb Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Tue, 18 Jun 2024 21:28:38 +0100 Subject: [PATCH 19/21] Discard changes to dev-requirements.txt --- dev-requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index f0ce2409d..0906cba2e 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,8 +1,8 @@ # install latest changes in dbt-core git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-adapters.git@get_relation_metadata -git+https://github.com/dbt-labs/dbt-adapters.git@get_relation_metadata#subdirectory=dbt-tests-adapter -git+https://github.com/dbt-labs/dbt-common.git@main +git+https://github.com/dbt-labs/dbt-adapters.git +git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter +git+https://github.com/dbt-labs/dbt-common.git # dev ipdb~=0.13.13 From a269ae2919d2bf5476dd8e26f9d75005719a10f4 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Tue, 18 Jun 2024 21:31:54 +0100 Subject: [PATCH 20/21] Discard changes to setup.py --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 468097e9a..f8ff363ed 100644 --- a/setup.py +++ b/setup.py @@ -57,8 +57,8 @@ def _plugin_version() -> str: packages=find_namespace_packages(include=["dbt", "dbt.*"]), include_package_data=True, install_requires=[ - # "dbt-common>=1.2.0,<2.0", - "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@get_relation_metadata", + "dbt-common>=1.0.4,<2.0", + "dbt-adapters>=1.1.1,<2.0", "snowflake-connector-python[secure-local-storage]~=3.0", # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency "dbt-core>=1.8.0", From 55ee5c3a1fc3b6b08d920d6f661e0e590b408519 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Fri, 21 Jun 2024 16:51:48 +0100 Subject: [PATCH 21/21] Bump pins on dbt-common and dbt-adapters --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index f8ff363ed..aa7b3772c 100644 --- a/setup.py +++ b/setup.py @@ -57,8 +57,8 @@ def _plugin_version() -> str: packages=find_namespace_packages(include=["dbt", "dbt.*"]), include_package_data=True, install_requires=[ - "dbt-common>=1.0.4,<2.0", - "dbt-adapters>=1.1.1,<2.0", + "dbt-common>=1.3.0,<2.0", + "dbt-adapters>=1.3.1,<2.0", "snowflake-connector-python[secure-local-storage]~=3.0", # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency "dbt-core>=1.8.0",