v1.6.1 release

microsoft · Nov 22, 2023 · cf36880 · cf36880
1 parent 004ab7a
commit cf36880
Show file tree

Hide file tree

Showing 15 changed files with 100 additions and 263 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,21 @@
 # Changelog
 
+### v1.6.1
+
+## Features
+
+* Fabric DW now supports sp_rename. Starting v1.6.1 sp_rename is metadata operation
+* Enabled table clone feature
+
+## Enhancements
+
+* Addressed [Issue 53](https://github.com/microsoft/dbt-fabric/issues/53)
+* Added explicit support for [Issue 76 - ActiveDirectoryServicePrincipal authentication](https://github.com/microsoft/dbt-fabric/issues/74)
+* Removed port number support in connection string as it is no longer required in Microsoft Fabric DW
+* Removed MSI authentication as it does not make sense for Microsoft Fabric.
+* Table lock hints are not supported by Fabric DW
+* Supported authentication modes are ActiveDirectory* and AZ CLI
+
 ### v1.6.0
 
 ## Features

diff --git a/dbt/adapters/fabric/__version__.py b/dbt/adapters/fabric/__version__.py
@@ -1 +1 @@
-version = "1.6.0"
+version = "1.6.1"
diff --git a/dbt/adapters/fabric/fabric_connection_manager.py b/dbt/adapters/fabric/fabric_connection_manager.py
@@ -9,13 +9,7 @@
 import dbt.exceptions
 import pyodbc
 from azure.core.credentials import AccessToken
-from azure.identity import (
-    AzureCliCredential,
-    ClientSecretCredential,
-    DefaultAzureCredential,
-    EnvironmentCredential,
-    ManagedIdentityCredential,
-)
+from azure.identity import AzureCliCredential, DefaultAzureCredential, EnvironmentCredential
 from dbt.adapters.sql import SQLConnectionManager
 from dbt.clients.agate_helper import empty_table
 from dbt.contracts.connection import AdapterResponse, Connection, ConnectionState
@@ -113,24 +107,6 @@ def get_cli_access_token(credentials: FabricCredentials) -> AccessToken:
     return token
 
 
-def get_msi_access_token(credentials: FabricCredentials) -> AccessToken:
-    """
-    Get an Azure access token from the system's managed identity
-
-    Parameters
-    -----------
-    credentials: FabricCredentials
-        Credentials.
-
-    Returns
-    -------
-    out : AccessToken
-        The access token.
-    """
-    token = ManagedIdentityCredential().get_token(AZURE_CREDENTIAL_SCOPE)
-    return token
-
-
 def get_auto_access_token(credentials: FabricCredentials) -> AccessToken:
     """
     Get an Azure access token automatically through azure-identity
@@ -167,30 +143,8 @@ def get_environment_access_token(credentials: FabricCredentials) -> AccessToken:
     return token
 
 
-def get_sp_access_token(credentials: FabricCredentials) -> AccessToken:
-    """
-    Get an Azure access token using the SP credentials.
-
-    Parameters
-    ----------
-    credentials : FabricCredentials
-        Credentials.
-
-    Returns
-    -------
-    out : AccessToken
-        The access token.
-    """
-    token = ClientSecretCredential(
-        str(credentials.tenant_id), str(credentials.client_id), str(credentials.client_secret)
-    ).get_token(AZURE_CREDENTIAL_SCOPE)
-    return token
-
-
 AZURE_AUTH_FUNCTIONS: Mapping[str, AZURE_AUTH_FUNCTION_TYPE] = {
-    "serviceprincipal": get_sp_access_token,
     "cli": get_cli_access_token,
-    "msi": get_msi_access_token,
     "auto": get_auto_access_token,
     "environment": get_environment_access_token,
 }
@@ -335,7 +289,7 @@ def open(cls, connection: Connection) -> Connection:
             # SQL Server named instance. In this case then port number has to be omitted.
             con_str.append(f"SERVER={credentials.host}")
         else:
-            con_str.append(f"SERVER={credentials.host},{credentials.port}")
+            con_str.append(f"SERVER={credentials.host}")
 
         con_str.append(f"Database={credentials.database}")
 
@@ -347,14 +301,16 @@ def open(cls, connection: Connection) -> Connection:
             if credentials.authentication == "ActiveDirectoryPassword":
                 con_str.append(f"UID={{{credentials.UID}}}")
                 con_str.append(f"PWD={{{credentials.PWD}}}")
+            if credentials.authentication == "ActiveDirectoryServicePrincipal":
+                con_str.append(f"UID={{{credentials.client_id}}}")
+                con_str.append(f"PWD={{{credentials.client_secret}}}")
             elif credentials.authentication == "ActiveDirectoryInteractive":
                 con_str.append(f"UID={{{credentials.UID}}}")
 
         elif credentials.windows_login:
             con_str.append("trusted_connection=Yes")
         elif credentials.authentication == "sql":
-            con_str.append(f"UID={{{credentials.UID}}}")
-            con_str.append(f"PWD={{{credentials.PWD}}}")
+            raise pyodbc.DatabaseError("SQL Authentication is not supported by Microsoft Fabric")
 
         # https://docs.microsoft.com/en-us/sql/relational-databases/native-client/features/using-encryption-without-validation?view=sql-server-ver15
         assert credentials.encrypt is not None

diff --git a/dbt/adapters/fabric/fabric_credentials.py b/dbt/adapters/fabric/fabric_credentials.py
@@ -10,14 +10,13 @@ class FabricCredentials(Credentials):
     host: str
     database: str
     schema: str
-    port: Optional[int] = 1433
     UID: Optional[str] = None
     PWD: Optional[str] = None
     windows_login: Optional[bool] = False
     tenant_id: Optional[str] = None
     client_id: Optional[str] = None
     client_secret: Optional[str] = None
-    authentication: Optional[str] = "sql"
+    authentication: Optional[str] = "ActiveDirectoryServicePrincipal"
     encrypt: Optional[bool] = True  # default value in MS ODBC Driver 18 as well
     trust_cert: Optional[bool] = False  # default value in MS ODBC Driver 18 as well
     retries: int = 1
@@ -53,7 +52,6 @@ def _connection_keys(self):
             "server",
             "database",
             "schema",
-            "port",
             "UID",
             "client_id",
             "authentication",

diff --git a/dbt/include/fabric/macros/adapters/metadata.sql b/dbt/include/fabric/macros/adapters/metadata.sql
@@ -1,12 +1,3 @@
-{% macro use_database_hint() %}
-    {{ return(adapter.dispatch('use_database_hint')()) }}
-{% endmacro %}
-
-{% macro default__use_database_hint() %}{% endmacro %}
-{% macro fabric__use_database_hint() %}
-    {# USE [{{ relation.database }}]; #}
-{% endmacro %}
-
 {% macro information_schema_hints() %}
     {{ return(adapter.dispatch('information_schema_hints')()) }}
 {% endmacro %}
@@ -24,7 +15,7 @@
             name as principal_name,
             principal_id as principal_id
         from
-            sys.database_principals
+            sys.database_principals {{ information_schema_hints() }}
     ),
 
     schemas as (
@@ -33,7 +24,7 @@
             schema_id as schema_id,
             principal_id as principal_id
         from
-            sys.schemas
+            sys.schemas {{ information_schema_hints() }}
     ),
 
     tables as (
@@ -43,7 +34,7 @@
             principal_id as principal_id,
             'BASE TABLE' as table_type
         from
-            sys.tables
+            sys.tables {{ information_schema_hints() }}
     ),
 
     tables_with_metadata as (
@@ -64,7 +55,7 @@
             principal_id as principal_id,
             'VIEW' as table_type
         from
-            sys.views
+            sys.views {{ information_schema_hints() }}
     ),
 
     views_with_metadata as (
@@ -107,7 +98,7 @@
             column_name,
             ordinal_position as column_index,
             data_type as column_type
-        from INFORMATION_SCHEMA.COLUMNS
+        from INFORMATION_SCHEMA.COLUMNS {{ information_schema_hints() }}
 
     )
 
@@ -138,9 +129,9 @@
 
 {% macro fabric__list_schemas(database) %}
   {% call statement('list_schemas', fetch_result=True, auto_begin=False) -%}
-    {{ use_database_hint() }}
+
     select  name as [schema]
-    from sys.schemas
+    from sys.schemas {{ information_schema_hints() }}
   {% endcall %}
   {{ return(load_result('list_schemas').table) }}
 {% endmacro %}
@@ -164,7 +155,7 @@
            else table_type
       end as table_type
 
-    from [{{ schema_relation.database }}].INFORMATION_SCHEMA.TABLES
+    from [{{ schema_relation.database }}].INFORMATION_SCHEMA.TABLES {{ information_schema_hints() }}
     where table_schema like '{{ schema_relation.schema }}'
   {% endcall %}
   {{ return(load_result('list_relations_without_caching').table) }}

diff --git a/dbt/include/fabric/macros/adapters/relation.sql b/dbt/include/fabric/macros/adapters/relation.sql
@@ -16,7 +16,7 @@
 
     {% if relation.type == 'view' -%}
       {% call statement('find_references', fetch_result=true) %}
-      {{ use_database_hint() }}
+      USE [{{ relation.database }}];
       select
           sch.name as schema_name,
           obj.name as view_name
@@ -39,117 +39,19 @@
           path={"schema": reference[0], "identifier": reference[1]})) }}
       {% endfor %}
     {% elif relation.type == 'table'%}
+      {% set object_id_type = 'U' %}
     {%- else -%}
         {{ exceptions.raise_not_implemented('Invalid relation being dropped: ' ~ relation) }}
     {% endif %}
-
-    {{ use_database_hint() }}
+    USE [{{ relation.database }}];
     EXEC('DROP {{ relation.type }} IF EXISTS {{ relation.include(database=False) }};');
-
 {% endmacro %}
 
 {% macro fabric__rename_relation(from_relation, to_relation) -%}
-  {% if to_relation.type == 'view' %}
-    {% call statement('get_view_definition', fetch_result=True) %}
-        SELECT m.[definition] AS VIEW_DEFINITION
-        FROM sys.objects o
-        INNER JOIN sys.sql_modules m
-            ON m.[object_id] = o.[object_id]
-        INNER JOIN sys.views v
-            ON o.[object_id] = v.[object_id]
-        INNER JOIN sys.schemas s
-            ON o.schema_id = s.schema_id
-            AND s.schema_id = v.schema_id
-        WHERE s.name = '{{ from_relation.schema }}'
-            AND v.name = '{{ from_relation.identifier }}'
-            AND o.[type] = 'V';
-    {% endcall %}
-
-    {% set view_def_full = load_result('get_view_definition')['data'][0][0] %}
-    {# Jinja does not allow bitwise operators and we need re.I | re.M here. So calculated manually this becomes 10. #}
-    {% set final_view_sql = modules.re.sub("create\s+view\s+.*?\s+as\s+","",view_def_full, 10) %}
-
-    {% call statement('create_new_view') %}
-        {{ create_view_as(to_relation, final_view_sql) }}
-    {% endcall %}
-    {% call statement('drop_old_view') %}
-        EXEC('DROP VIEW IF EXISTS {{ from_relation.include(database=False) }};');
-    {% endcall %}
-  {% endif %}
-  {% if to_relation.type == 'table' %}
-      {% call statement('rename_relation') %}
-        EXEC('create table {{ to_relation.include(database=False) }} as select * from {{ from_relation.include(database=False) }}');
-      {%- endcall %}
-      -- Getting constraints from the old table
-      {% call statement('get_table_constraints', fetch_result=True) %}
-        SELECT DISTINCT Contraint_statement FROM
-        (
-          SELECT DISTINCT
-          CASE
-              WHEN tc.CONSTRAINT_TYPE = 'PRIMARY KEY'
-                  THEN 'ALTER TABLE <<REPLACE TABLE>> ADD CONSTRAINT PK_<<CONSTRAINT NAME>>_'+ccu.COLUMN_NAME+' PRIMARY KEY NONCLUSTERED('+ccu.COLUMN_NAME+') NOT ENFORCED'
-              WHEN tc.CONSTRAINT_TYPE = 'UNIQUE'
-                  THEN 'ALTER TABLE <<REPLACE TABLE>> ADD CONSTRAINT UK_<<CONSTRAINT NAME>>_'+ccu.COLUMN_NAME+' UNIQUE NONCLUSTERED('+ccu.COLUMN_NAME+') NOT ENFORCED'
-              END AS Contraint_statement
-          FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS tc INNER JOIN
-              INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE ccu
-                  ON tc.CONSTRAINT_NAME = ccu.CONSTRAINT_NAME
-          WHERE tc.TABLE_NAME = '{{ from_relation.identifier }}' and tc.TABLE_SCHEMA = '{{ from_relation.schema }}'
-          UNION ALL
-          SELECT
-            'ALTER TABLE <<REPLACE TABLE>> ADD CONSTRAINT FK_<<CONSTRAINT NAME>>_'+CU.COLUMN_NAME+' FOREIGN KEY('+CU.COLUMN_NAME+') references '+PK.TABLE_SCHEMA+'.'+PK.TABLE_NAME+' ('+PT.COLUMN_NAME+') not enforced'   AS Contraint_statement
-            FROM INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS C
-            INNER JOIN INFORMATION_SCHEMA.TABLE_CONSTRAINTS FK ON C.CONSTRAINT_NAME = FK.CONSTRAINT_NAME
-            INNER JOIN INFORMATION_SCHEMA.TABLE_CONSTRAINTS PK ON C.UNIQUE_CONSTRAINT_NAME=PK.CONSTRAINT_NAME
-            INNER JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE CU ON C.CONSTRAINT_NAME = CU.CONSTRAINT_NAME
-            INNER JOIN (
-                SELECT i1.TABLE_NAME, i2.COLUMN_NAME, i1.TABLE_SCHEMA, i2.TABLE_SCHEMA AS CU_TableSchema
-                FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS i1
-                    INNER JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE i2 ON i1.CONSTRAINT_NAME =i2.CONSTRAINT_NAME
-                WHERE i1.CONSTRAINT_TYPE = 'PRIMARY KEY'
-            ) PT ON PT.TABLE_NAME = PK.TABLE_NAME AND PT.TABLE_SCHEMA = PK.TABLE_SCHEMA AND PT.CU_TableSchema = PK.TABLE_SCHEMA
-          WHERE FK.TABLE_NAME = '{{ from_relation.identifier }}' and FK.TABLE_SCHEMA = '{{ from_relation.schema }}'
-          and PK.TABLE_SCHEMA = '{{ from_relation.schema }}' and PT.TABLE_SCHEMA = '{{ from_relation.schema }}'
-        ) T WHERE Contraint_statement IS NOT NULL
-      {% endcall %}
-
-      {%call statement('drop_table_constraints', fetch_result= True)%}
-        SELECT drop_constraint_statement FROM
-        (
-          SELECT 'ALTER TABLE ['+TABLE_SCHEMA+'].['+TABLE_NAME+'] DROP CONSTRAINT ' + CONSTRAINT_NAME AS drop_constraint_statement
-          FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS
-          WHERE TABLE_NAME = '{{ from_relation.identifier }}' and TABLE_SCHEMA = '{{ from_relation.schema }}'
-        ) T WHERE drop_constraint_statement IS NOT NULL
-
-      {% endcall %}
-
-      {% set references = load_result('get_table_constraints')['data'] %}
-      {% set drop_references = load_result('drop_table_constraints')['data'] %}
-
-      {% for reference in drop_references -%}
-        {% set drop_constraint = reference[0]%}
-
-        {% call statement('Drop_Constraints') %}
-          {{ log("Constraints to drop: "~reference[0], info=True) }}
-          EXEC('{{drop_constraint}}');
-        {% endcall %}
-      {% endfor %}
-
-      {% set targetTableNameConstraint = to_relation.include(database=False)%}
-      {% set targetTableNameConstraint = (targetTableNameConstraint|string).strip().replace("\"","").replace(".","_")%}
-      {% set targetTableName = to_relation.include(database=False) %}
-
-      {% for reference in references -%}
-        {% set constraint_name = reference[0].replace("<<CONSTRAINT NAME>>",targetTableNameConstraint)%}
-        {% set alter_create_table_constraint_script = constraint_name.replace("<<REPLACE TABLE>>", (targetTableName|string).strip()) %}
-        {{ log("Constraints to create: "~alter_create_table_constraint_script, info=True) }}
-        {% call statement('Drop_Create_Constraints') %}
-          EXEC('{{alter_create_table_constraint_script}}');
-        {% endcall %}
-      {% endfor %}
-
-      {{ fabric__drop_relation(from_relation) }}
-  {% endif %}
+  {% call statement('rename_relation') -%}
+     USE [{{ from_relation.database }}];
+      EXEC sp_rename '{{ from_relation.schema }}.{{ from_relation.identifier }}', '{{ to_relation.identifier }}'
+  {%- endcall %}
 {% endmacro %}
 
 -- DROP fabric__truncate_relation when TRUNCATE TABLE is supported

diff --git a/dbt/include/fabric/macros/adapters/schema.sql b/dbt/include/fabric/macros/adapters/schema.sql
@@ -1,6 +1,6 @@
 {% macro fabric__create_schema(relation) -%}
   {% call statement('create_schema') -%}
-    {{ use_database_hint() }}
+    USE [{{ relation.database }}];
     IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = '{{ relation.schema }}')
     BEGIN
     EXEC('CREATE SCHEMA [{{ relation.schema }}]')
@@ -10,7 +10,7 @@
 
 {% macro fabric__create_schema_with_authorization(relation, schema_authorization) -%}
   {% call statement('create_schema') -%}
-    {{ use_database_hint() }}
+    USE [{{ relation.database }}];
     IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = '{{ relation.schema }}')
     BEGIN
     EXEC('CREATE SCHEMA [{{ relation.schema }}] AUTHORIZATION [{{ schema_authorization }}]')