dbt-fabric 1.7.1 release commit

.github/workflows/publish-docker.yml

@@ -24,14 +24,14 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Log in to the Container registry
-        uses: docker/login-action@v2.2.0
+        uses: docker/login-action@v3.0.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@v4.2.1
+        uses: docker/build-push-action@v5.1.0
         with:
           context: devops
           build-args: PYTHON_VERSION=${{ matrix.python_version }}

CHANGELOG.md

@@ -1,5 +1,23 @@
 # Changelog
 
+### v1.7.1
+
+## Features
+
+* Added capability support structure in fabric adapter
+* Added metadata freshness checks
+* Updated catalog fetch performance improvements to handle relations with many pre-existing objects
+* Added dbt-show support to 1.7.1
+
+## Enhancements
+
+* improve connection manager logging
+* Added metadata freshness checks tests
+* Added capability support tests
+* Added catalog fetch performance improvements
+* Added dbt show's --limit flag tests
+* Added storing test failures tests
+
 ### v1.6.1
 
 ## Features

dbt/adapters/fabric/fabric_adapter.py

@@ -9,6 +9,7 @@
 from dbt.adapters.base.meta import available
 from dbt.adapters.base.relation import BaseRelation
 from dbt.adapters.cache import _make_ref_key_dict
+from dbt.adapters.capability import Capability, CapabilityDict, CapabilitySupport, Support
 
 # from dbt.adapters.cache import _make_ref_key_msg
 from dbt.adapters.sql import SQLAdapter
@@ -27,6 +28,12 @@ class FabricAdapter(SQLAdapter):
     Column = FabricColumn
     AdapterSpecificConfigs = FabricConfigs
 
+    _capabilities: CapabilityDict = CapabilityDict(
+        {
+            Capability.SchemaMetadataByRelations: CapabilitySupport(support=Support.Full),
+            Capability.TableLastModifiedMetadata: CapabilitySupport(support=Support.Full),
+        }
+    )
     CONSTRAINT_SUPPORT = {
         ConstraintType.check: ConstraintSupport.NOT_SUPPORTED,
         ConstraintType.not_null: ConstraintSupport.ENFORCED,

dbt/adapters/fabric/fabric_connection_manager.py

@@ -14,6 +14,10 @@
 from dbt.clients.agate_helper import empty_table
 from dbt.contracts.connection import AdapterResponse, Connection, ConnectionState
 from dbt.events import AdapterLogger
+from dbt.events.contextvars import get_node_info
+from dbt.events.functions import fire_event
+from dbt.events.types import ConnectionUsed, SQLQuery, SQLQueryStatus
+from dbt.utils import cast_to_str
 
 from dbt.adapters.fabric import __version__
 from dbt.adapters.fabric.fabric_credentials import FabricCredentials
@@ -391,13 +395,26 @@ def add_query(
         if auto_begin and connection.transaction_open is False:
             self.begin()
 
-        logger.debug('Using {} connection "{}".'.format(self.TYPE, connection.name))
+        fire_event(
+            ConnectionUsed(
+                conn_type=self.TYPE,
+                conn_name=cast_to_str(connection.name),
+                node_info=get_node_info(),
+            )
+        )
 
         with self.exception_handler(sql):
             if abridge_sql_log:
-                logger.debug("On {}: {}....".format(connection.name, sql[0:512]))
+                log_sql = "{}...".format(sql[:512])
             else:
-                logger.debug("On {}: {}".format(connection.name, sql))
+                log_sql = sql
+
+            fire_event(
+                SQLQuery(
+                    conn_name=cast_to_str(connection.name), sql=log_sql, node_info=get_node_info()
+                )
+            )
+
             pre = time.time()
 
             cursor = connection.handle.cursor()
@@ -416,9 +433,11 @@ def add_query(
             # https://github.com/mkleehammer/pyodbc/issues/134#issuecomment-281739794
             connection.handle.add_output_converter(-155, byte_array_to_datetime)
 
-            logger.debug(
-                "SQL status: {} in {:0.2f} seconds".format(
-                    self.get_response(cursor), (time.time() - pre)
+            fire_event(
+                SQLQueryStatus(
+                    status=str(self.get_response(cursor)),
+                    elapsed=round((time.time() - pre)),
+                    node_info=get_node_info(),
                 )
             )
 
@@ -454,6 +473,7 @@ def data_type_code_to_name(cls, type_code: Union[str, str]) -> str:
     def execute(
         self, sql: str, auto_begin: bool = True, fetch: bool = False, limit: Optional[int] = None
     ) -> Tuple[AdapterResponse, agate.Table]:
+        sql = self._add_query_comment(sql)
         _, cursor = self.add_query(sql, auto_begin)
         response = self.get_response(cursor)
         if fetch:

dbt/include/fabric/macros/adapters/catalog.sql

@@ -0,0 +1,260 @@
+{% macro fabric__get_catalog(information_schemas, schemas) -%}
+
+  {%- call statement('catalog', fetch_result=True) -%}
+
+    with
+    principals as (
+        select
+            name as principal_name,
+            principal_id as principal_id
+        from
+            sys.database_principals {{ information_schema_hints() }}
+    ),
+
+    schemas as (
+        select
+            name as schema_name,
+            schema_id as schema_id,
+            principal_id as principal_id
+        from
+            sys.schemas {{ information_schema_hints() }}
+    ),
+
+    tables as (
+        select
+            name as table_name,
+            schema_id as schema_id,
+            principal_id as principal_id,
+            'BASE TABLE' as table_type
+        from
+            sys.tables {{ information_schema_hints() }}
+    ),
+
+    tables_with_metadata as (
+        select
+            table_name,
+            schema_name,
+            coalesce(tables.principal_id, schemas.principal_id) as owner_principal_id,
+            table_type
+        from
+            tables
+        join schemas on tables.schema_id = schemas.schema_id
+    ),
+
+    views as (
+        select
+            name as table_name,
+            schema_id as schema_id,
+            principal_id as principal_id,
+            'VIEW' as table_type
+        from
+            sys.views {{ information_schema_hints() }}
+    ),
+
+    views_with_metadata as (
+        select
+            table_name,
+            schema_name,
+            coalesce(views.principal_id, schemas.principal_id) as owner_principal_id,
+            table_type
+        from
+            views
+        join schemas on views.schema_id = schemas.schema_id
+    ),
+
+    tables_and_views as (
+        select
+            table_name,
+            schema_name,
+            principal_name,
+            table_type
+        from
+            tables_with_metadata
+        join principals on tables_with_metadata.owner_principal_id = principals.principal_id
+        union all
+        select
+            table_name,
+            schema_name,
+            principal_name,
+            table_type
+        from
+            views_with_metadata
+        join principals on views_with_metadata.owner_principal_id = principals.principal_id
+    ),
+
+    cols as (
+
+        select
+            table_catalog as table_database,
+            table_schema,
+            table_name,
+            column_name,
+            ordinal_position as column_index,
+            data_type as column_type
+        from INFORMATION_SCHEMA.COLUMNS {{ information_schema_hints() }}
+
+    )
+
+    select
+        cols.table_database,
+        tv.schema_name as table_schema,
+        tv.table_name,
+        tv.table_type,
+        null as table_comment,
+        tv.principal_name as table_owner,
+        cols.column_name,
+        cols.column_index,
+        cols.column_type,
+        null as column_comment
+    from tables_and_views tv
+    join cols on tv.schema_name = cols.table_schema and tv.table_name = cols.table_name
+    where ({%- for schema in schemas -%}
+        upper(tv.schema_name) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%}
+    {%- endfor -%})
+
+    order by column_index
+
+    {%- endcall -%}
+
+  {{ return(load_result('catalog').table) }}
+
+{%- endmacro %}
+
+{% macro fabric__get_catalog_relations(information_schema, relations) -%}
+
+    {%- call statement('catalog', fetch_result=True) -%}
+
+    with
+    principals as (
+        select
+            name as principal_name,
+            principal_id as principal_id
+        from
+            sys.database_principals {{ information_schema_hints() }}
+    ),
+
+    schemas as (
+        select
+            name as schema_name,
+            schema_id as schema_id,
+            principal_id as principal_id
+        from
+            sys.schemas {{ information_schema_hints() }}
+    ),
+
+    tables as (
+        select
+            name as table_name,
+            schema_id as schema_id,
+            principal_id as principal_id,
+            'BASE TABLE' as table_type
+        from
+            sys.tables {{ information_schema_hints() }}
+    ),
+
+    tables_with_metadata as (
+        select
+            table_name,
+            schema_name,
+            coalesce(tables.principal_id, schemas.principal_id) as owner_principal_id,
+            table_type
+        from
+            tables
+        join schemas on tables.schema_id = schemas.schema_id
+    ),
+
+    views as (
+        select
+            name as table_name,
+            schema_id as schema_id,
+            principal_id as principal_id,
+            'VIEW' as table_type
+        from
+            sys.views {{ information_schema_hints() }}
+    ),
+
+    views_with_metadata as (
+        select
+            table_name,
+            schema_name,
+            coalesce(views.principal_id, schemas.principal_id) as owner_principal_id,
+            table_type
+        from
+            views
+        join schemas on views.schema_id = schemas.schema_id
+    ),
+
+    tables_and_views as (
+        select
+            table_name,
+            schema_name,
+            principal_name,
+            table_type
+        from
+            tables_with_metadata
+        join principals on tables_with_metadata.owner_principal_id = principals.principal_id
+        union all
+        select
+            table_name,
+            schema_name,
+            principal_name,
+            table_type
+        from
+            views_with_metadata
+        join principals on views_with_metadata.owner_principal_id = principals.principal_id
+    ),
+
+    cols as (
+
+        select
+            table_catalog as table_database,
+            table_schema,
+            table_name,
+            column_name,
+            ordinal_position as column_index,
+            data_type as column_type
+        from INFORMATION_SCHEMA.COLUMNS {{ information_schema_hints() }}
+
+    )
+
+    select
+        cols.table_database,
+        tv.schema_name as table_schema,
+        tv.table_name,
+        tv.table_type,
+        null as table_comment,
+        tv.principal_name as table_owner,
+        cols.column_name,
+        cols.column_index,
+        cols.column_type,
+        null as column_comment
+    from tables_and_views tv
+    join cols on tv.schema_name = cols.table_schema and tv.table_name = cols.table_name
+    where (
+        {%- for relation in relations -%}
+            {% if relation.schema and relation.identifier %}
+                (
+                    upper(tv.schema_name) = upper('{{ relation.schema }}')
+                    and upper(tv.table_name) = upper('{{ relation.identifier }}')
+                )
+            {% elif relation.schema %}
+                (
+                    upper(tv.schema_name) = upper('{{ relation.schema }}')
+                )
+            {% else %}
+                {% do exceptions.raise_compiler_error(
+                    '`get_catalog_relations` requires a list of relations, each with a schema'
+                ) %}
+            {% endif %}
+
+            {%- if not loop.last %} or {% endif -%}
+        {%- endfor -%}
+    )
+
+    order by column_index
+
+    {%- endcall -%}
+
+  {{ return(load_result('catalog').table) }}
+
+{%- endmacro %}