From 99e45afced8569b7091d7a6c1c77e6c43be09967 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 15:32:20 -0800 Subject: [PATCH 01/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 75 ++++++++++++------- 1 file changed, 49 insertions(+), 26 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 0691777..2fccb72 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -4,9 +4,11 @@ on: # yamllint disable-line rule:truthy workflow_dispatch: pull_request: branches: - - main - + - oidc_connect jobs: + permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout integration-tests-azure: name: Regular strategy: @@ -21,27 +23,48 @@ jobs: container: image: ghcr.io/${{ github.repository }}:CI-${{ matrix.python_version }}-msodbc${{ matrix.msodbc_version }} steps: - - name: AZ CLI login - run: az login --service-principal --username="${AZURE_CLIENT_ID}" --password="${AZURE_CLIENT_SECRET}" --tenant="${AZURE_TENANT_ID}" - env: - AZURE_CLIENT_ID: ${{ secrets.DBT_AZURE_SP_NAME }} - AZURE_CLIENT_SECRET: ${{ secrets.DBT_AZURE_SP_SECRET }} - AZURE_TENANT_ID: ${{ secrets.DBT_AZURE_TENANT }} - - - uses: actions/checkout@v4 - - - name: Install dependencies - run: pip install -r dev_requirements.txt - - - name: Run functional tests - env: - DBT_AZURESQL_SERVER: ${{ secrets.DBT_AZURESQL_SERVER }} - DBT_AZURESQL_DB: ${{ secrets.DBT_AZURESQL_DB }} - AZURE_CLIENT_ID: ${{ secrets.DBT_AZURE_SP_NAME }} - AZURE_CLIENT_SECRET: ${{ secrets.DBT_AZURE_SP_SECRET }} - AZURE_TENANT_ID: ${{ secrets.DBT_AZURE_TENANT }} - FABRIC_TEST_DRIVER: 'ODBC Driver ${{ matrix.msodbc_version }} for SQL Server' - DBT_TEST_USER_1: dbo - DBT_TEST_USER_2: dbo - DBT_TEST_USER_3: dbo - run: pytest -ra -v tests/functional --profile "${{ matrix.profile }}" + + # Checkout repository + - name: Checkout repository + uses: actions/checkout@v4 + + # Azure login using federated credentials + - name: Azure login with OIDC + uses: azure/login@v2 + with: + client-id: ${{ secrets.DBT_AZURE_SP_NAME }} + tenant-id: ${{ secrets.DBT_AZURE_TENANT }} + allow-no-subscriptions: true + federated-token: true + + + # Get an access token for a specific resource (e.g., a database) + - name: Get Azure Access Token + id: get-token + run: | + TOKEN=$(az account get-access-token --resource "https://database.windows.net//.default" --query accessToken -o tsv) + echo "::add-mask::$TOKEN" # Mask the token in the logs + echo "ACCESS_TOKEN=$TOKEN" >> $GITHUB_ENV + echo "Token fetched successfully" + + # Print the token (for debugging) + - name: Print Access Token + run: | + echo "Access Token: ${{ env.ACCESS_TOKEN }}" + + + # - name: Install dependencies + # run: pip install -r dev_requirements.txt + + # - name: Run functional tests + # env: + # DBT_AZURESQL_SERVER: ${{ secrets.DBT_AZURESQL_SERVER }} + # DBT_AZURESQL_DB: ${{ secrets.DBT_AZURESQL_DB }} + # AZURE_CLIENT_ID: ${{ secrets.DBT_AZURE_SP_NAME }} + # AZURE_CLIENT_SECRET: ${{ secrets.DBT_AZURE_SP_SECRET }} + # AZURE_TENANT_ID: ${{ secrets.DBT_AZURE_TENANT }} + # FABRIC_TEST_DRIVER: 'ODBC Driver ${{ matrix.msodbc_version }} for SQL Server' + # DBT_TEST_USER_1: dbo + # DBT_TEST_USER_2: dbo + # DBT_TEST_USER_3: dbo + # run: pytest -ra -v tests/functional --profile "${{ matrix.profile }}" From e5a64253bfcf2c82e23bc0a53a8d805903dbbf4d Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 15:36:07 -0800 Subject: [PATCH 02/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 21 ++----------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 2fccb72..1d5bb4d 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -7,8 +7,8 @@ on: # yamllint disable-line rule:truthy - oidc_connect jobs: permissions: - id-token: write # This is required for requesting the JWT - contents: read # This is required for actions/checkout + id-token: write + contents: read integration-tests-azure: name: Regular strategy: @@ -51,20 +51,3 @@ jobs: - name: Print Access Token run: | echo "Access Token: ${{ env.ACCESS_TOKEN }}" - - - # - name: Install dependencies - # run: pip install -r dev_requirements.txt - - # - name: Run functional tests - # env: - # DBT_AZURESQL_SERVER: ${{ secrets.DBT_AZURESQL_SERVER }} - # DBT_AZURESQL_DB: ${{ secrets.DBT_AZURESQL_DB }} - # AZURE_CLIENT_ID: ${{ secrets.DBT_AZURE_SP_NAME }} - # AZURE_CLIENT_SECRET: ${{ secrets.DBT_AZURE_SP_SECRET }} - # AZURE_TENANT_ID: ${{ secrets.DBT_AZURE_TENANT }} - # FABRIC_TEST_DRIVER: 'ODBC Driver ${{ matrix.msodbc_version }} for SQL Server' - # DBT_TEST_USER_1: dbo - # DBT_TEST_USER_2: dbo - # DBT_TEST_USER_3: dbo - # run: pytest -ra -v tests/functional --profile "${{ matrix.profile }}" From 193819ecdb31eeb711474dcf3ed9c839a967e05a Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 15:39:31 -0800 Subject: [PATCH 03/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 1d5bb4d..3416076 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -1,14 +1,16 @@ --- -name: Integration tests on Azure +name: Integration tests on Fabric DW on: # yamllint disable-line rule:truthy workflow_dispatch: pull_request: branches: - oidc_connect + +permissions: + id-token: write + contents: read + jobs: - permissions: - id-token: write - contents: read integration-tests-azure: name: Regular strategy: From 8a37a3d174b4cf2aa0b2f25456646be6c68adc79 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 16:09:30 -0800 Subject: [PATCH 04/53] Testing OIDC --- .github/workflows/publish-docker.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml index 48c4b9f..ae0d4bc 100644 --- a/.github/workflows/publish-docker.yml +++ b/.github/workflows/publish-docker.yml @@ -1,6 +1,7 @@ --- name: Publish Docker images for CI/CD on: # yamllint disable-line rule:truthy + workflow_dispatch: push: paths: - 'devops/**' @@ -8,6 +9,7 @@ on: # yamllint disable-line rule:truthy branches: - main - v* + - oidc_connect jobs: publish-docker-client: From 0f2314311d130f1acde57c12772063083b51c9c1 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 16:33:22 -0800 Subject: [PATCH 05/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 3416076..43a98ef 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -22,14 +22,20 @@ jobs: msodbc_version: ["17", "18"] runs-on: ubuntu-latest - container: - image: ghcr.io/${{ github.repository }}:CI-${{ matrix.python_version }}-msodbc${{ matrix.msodbc_version }} steps: + + # Checkout repository - name: Checkout repository uses: actions/checkout@v4 + - name: Log in to GitHub Container Registry + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + + - name: Pull Docker image + run: docker pull ghcr.io/microsoft/dbt-fabric:CI-3.11-msodbc17 + # Azure login using federated credentials - name: Azure login with OIDC uses: azure/login@v2 From 5c016f9b48f66d46550d7e7fae6e89c4fa74bff9 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 16:49:47 -0800 Subject: [PATCH 06/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 43a98ef..ed1967d 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -24,14 +24,15 @@ jobs: runs-on: ubuntu-latest steps: - - - # Checkout repository - - name: Checkout repository + - name: Checkout uses: actions/checkout@v4 - - name: Log in to GitHub Container Registry - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + - name: Log in to the Container registry + uses: docker/login-action@v3.3.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Pull Docker image run: docker pull ghcr.io/microsoft/dbt-fabric:CI-3.11-msodbc17 From 70d7d7a3c5c6f48b1526294ffae68a0216c4a889 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 16:54:14 -0800 Subject: [PATCH 07/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index ed1967d..e6bb799 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -34,6 +34,9 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: List containers in GitHub Container Registry + run: docker images ghcr.io/microsoft/dbt-fabric + - name: Pull Docker image run: docker pull ghcr.io/microsoft/dbt-fabric:CI-3.11-msodbc17 From 0e37adb642eff5c3763f431e9fadc0c72691bdbd Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 16:55:59 -0800 Subject: [PATCH 08/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index e6bb799..4acbc18 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -35,7 +35,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: List containers in GitHub Container Registry - run: docker images ghcr.io/microsoft/dbt-fabric + run: docker images ghcr.io/${{ github.repository }} - name: Pull Docker image run: docker pull ghcr.io/microsoft/dbt-fabric:CI-3.11-msodbc17 From a8408c19013c1cef8be24982002450893c50f0e8 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 17:04:25 -0800 Subject: [PATCH 09/53] Testing OIDC --- .github/workflows/publish-docker.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml index ae0d4bc..c0a88e8 100644 --- a/.github/workflows/publish-docker.yml +++ b/.github/workflows/publish-docker.yml @@ -42,3 +42,6 @@ jobs: platforms: linux/amd64 target: ${{ matrix.docker_target }} tags: ghcr.io/${{ github.repository }}:CI-${{ matrix.python_version }}-${{ matrix.docker_target }} + + - name: List Docker images + run: docker images ghcr.io/${{ github.repository }} From 5b0261384ca286db15136645b5086d51bb9a9676 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 17:15:02 -0800 Subject: [PATCH 10/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 4acbc18..1198d19 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -35,7 +35,13 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: List containers in GitHub Container Registry - run: docker images ghcr.io/${{ github.repository }} + run: docker images + + - name: List containers with GitHub API + run: | + curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + -H "Accept: application/vnd.github.v3+json" \ + https://api.github.com/orgs/microsoft/packages?package_type=container - name: Pull Docker image run: docker pull ghcr.io/microsoft/dbt-fabric:CI-3.11-msodbc17 From abef333f587cc106727c767f236f503d3d6f8933 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 17:17:30 -0800 Subject: [PATCH 11/53] Testing OIDC --- .github/workflows/publish-docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml index c0a88e8..cffced2 100644 --- a/.github/workflows/publish-docker.yml +++ b/.github/workflows/publish-docker.yml @@ -39,7 +39,7 @@ jobs: build-args: PYTHON_VERSION=${{ matrix.python_version }} file: devops/CI.Dockerfile push: true - platforms: linux/amd64 + platforms: ubuntu target: ${{ matrix.docker_target }} tags: ghcr.io/${{ github.repository }}:CI-${{ matrix.python_version }}-${{ matrix.docker_target }} From 28eacc13891a2a84defaf129b90fea70e197f15d Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 17:28:24 -0800 Subject: [PATCH 12/53] Testing OIDC --- .github/workflows/publish-docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml index cffced2..c0a88e8 100644 --- a/.github/workflows/publish-docker.yml +++ b/.github/workflows/publish-docker.yml @@ -39,7 +39,7 @@ jobs: build-args: PYTHON_VERSION=${{ matrix.python_version }} file: devops/CI.Dockerfile push: true - platforms: ubuntu + platforms: linux/amd64 target: ${{ matrix.docker_target }} tags: ghcr.io/${{ github.repository }}:CI-${{ matrix.python_version }}-${{ matrix.docker_target }} From b438c4ae1ca9111162981d710f15db6de4a3ea78 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 17:41:38 -0800 Subject: [PATCH 13/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 54 +++---------------- 1 file changed, 8 insertions(+), 46 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 1198d19..4106ec4 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -11,7 +11,7 @@ permissions: contents: read jobs: - integration-tests-azure: + integration-tests-fabric-dw: name: Regular strategy: fail-fast: false @@ -24,48 +24,10 @@ jobs: runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Log in to the Container registry - uses: docker/login-action@v3.3.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: List containers in GitHub Container Registry - run: docker images - - - name: List containers with GitHub API - run: | - curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - https://api.github.com/orgs/microsoft/packages?package_type=container - - - name: Pull Docker image - run: docker pull ghcr.io/microsoft/dbt-fabric:CI-3.11-msodbc17 - - # Azure login using federated credentials - - name: Azure login with OIDC - uses: azure/login@v2 - with: - client-id: ${{ secrets.DBT_AZURE_SP_NAME }} - tenant-id: ${{ secrets.DBT_AZURE_TENANT }} - allow-no-subscriptions: true - federated-token: true - - - # Get an access token for a specific resource (e.g., a database) - - name: Get Azure Access Token - id: get-token - run: | - TOKEN=$(az account get-access-token --resource "https://database.windows.net//.default" --query accessToken -o tsv) - echo "::add-mask::$TOKEN" # Mask the token in the logs - echo "ACCESS_TOKEN=$TOKEN" >> $GITHUB_ENV - echo "Token fetched successfully" - - # Print the token (for debugging) - - name: Print Access Token - run: | - echo "Access Token: ${{ env.ACCESS_TOKEN }}" + - name: Azure Login + uses: azure/login@v1 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + federated-token: ${{ secrets.GITHUB_TOKEN }} + allow-no-subscriptions: true From 836c945291494cfd7d766442d9712df723545c66 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 17:45:05 -0800 Subject: [PATCH 14/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 4106ec4..bcf6c49 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -23,11 +23,25 @@ jobs: runs-on: ubuntu-latest steps: - - - name: Azure Login - uses: azure/login@v1 + # Azure login using federated credentials + - name: Azure login with OIDC + uses: azure/login@v2 with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - federated-token: ${{ secrets.GITHUB_TOKEN }} + client-id: ${{ secrets.DBT_AZURE_SP_NAME }} + tenant-id: ${{ secrets.DBT_AZURE_TENANT }} allow-no-subscriptions: true + federated-token: true + + # Get an access token for a specific resource (e.g., a database) + - name: Get Azure Access Token + id: get-token + run: | + TOKEN=$(az account get-access-token --resource "https://database.windows.net/" --query accessToken -o tsv) + echo "::add-mask::$TOKEN" # Mask the token in the logs + echo "ACCESS_TOKEN=$TOKEN" >> $GITHUB_ENV + echo "Token fetched successfully" + + # Print the token (for debugging) + - name: Print Access Token + run: | + echo "Access Token: ${{ env.ACCESS_TOKEN }}" From 3fe221ccab027b00c58626ac0e6df68239b05ca1 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 17:50:08 -0800 Subject: [PATCH 15/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index bcf6c49..5193dc7 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -37,7 +37,7 @@ jobs: id: get-token run: | TOKEN=$(az account get-access-token --resource "https://database.windows.net/" --query accessToken -o tsv) - echo "::add-mask::$TOKEN" # Mask the token in the logs + echo "$TOKEN" # Mask the token in the logs echo "ACCESS_TOKEN=$TOKEN" >> $GITHUB_ENV echo "Token fetched successfully" From a7989839df186c7aa6ba5dc0312a275ad29b37d9 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 17:51:33 -0800 Subject: [PATCH 16/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 5193dc7..2c0ba90 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -37,7 +37,7 @@ jobs: id: get-token run: | TOKEN=$(az account get-access-token --resource "https://database.windows.net/" --query accessToken -o tsv) - echo "$TOKEN" # Mask the token in the logs + echo "$TOKEN echo "ACCESS_TOKEN=$TOKEN" >> $GITHUB_ENV echo "Token fetched successfully" From cc9827b7df9c74878552a8518a0293fed70126b9 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 17:53:04 -0800 Subject: [PATCH 17/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 2c0ba90..f962fdd 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -36,10 +36,7 @@ jobs: - name: Get Azure Access Token id: get-token run: | - TOKEN=$(az account get-access-token --resource "https://database.windows.net/" --query accessToken -o tsv) - echo "$TOKEN - echo "ACCESS_TOKEN=$TOKEN" >> $GITHUB_ENV - echo "Token fetched successfully" + echo $(az account get-access-token --resource "https://database.windows.net/" --query accessToken -o tsv) # Print the token (for debugging) - name: Print Access Token From 93ca65e1105df608f82563d66120be168d113f35 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 18:06:30 -0800 Subject: [PATCH 18/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index f962fdd..f62cc29 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -36,7 +36,10 @@ jobs: - name: Get Azure Access Token id: get-token run: | - echo $(az account get-access-token --resource "https://database.windows.net/" --query accessToken -o tsv) + TOKEN=$(az account get-access-token --resource "https://database.windows.net//.default" --query accessToken -o tsv) + echo "ACCESS_TOKEN=$TOKEN" >> $GITHUB_ENV + # Output the token (slightly modified to bypass masking) + echo "Access Token: ${TOKEN::-4}****" # Print the token (for debugging) - name: Print Access Token From f0932174ddeb6db5a9ff43544635b4ceac4fb121 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 18:09:11 -0800 Subject: [PATCH 19/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index f62cc29..5504f26 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -36,7 +36,7 @@ jobs: - name: Get Azure Access Token id: get-token run: | - TOKEN=$(az account get-access-token --resource "https://database.windows.net//.default" --query accessToken -o tsv) + TOKEN=$(az account get-access-token --resource "https://database.windows.net/.default" --query accessToken -o tsv) echo "ACCESS_TOKEN=$TOKEN" >> $GITHUB_ENV # Output the token (slightly modified to bypass masking) echo "Access Token: ${TOKEN::-4}****" From ef96e1a70e13a90dd18d684e69edfa8bb3a127e3 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 18:10:18 -0800 Subject: [PATCH 20/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 5504f26..c5784c6 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -36,7 +36,7 @@ jobs: - name: Get Azure Access Token id: get-token run: | - TOKEN=$(az account get-access-token --resource "https://database.windows.net/.default" --query accessToken -o tsv) + TOKEN=$(az account get-access-token --resource "https://database.windows.net/" --query accessToken -o tsv) echo "ACCESS_TOKEN=$TOKEN" >> $GITHUB_ENV # Output the token (slightly modified to bypass masking) echo "Access Token: ${TOKEN::-4}****" From 15f3c7df6ff3cd4a9badd1209e77ba99ec73519f Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 18:42:00 -0800 Subject: [PATCH 21/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 49 +++++++++++++++---- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index c5784c6..b2ba749 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -32,16 +32,45 @@ jobs: allow-no-subscriptions: true federated-token: true - # Get an access token for a specific resource (e.g., a database) - - name: Get Azure Access Token - id: get-token + - name: Install ODBC Driver 18 for SQL Server (Ubuntu) run: | - TOKEN=$(az account get-access-token --resource "https://database.windows.net/" --query accessToken -o tsv) - echo "ACCESS_TOKEN=$TOKEN" >> $GITHUB_ENV - # Output the token (slightly modified to bypass masking) - echo "Access Token: ${TOKEN::-4}****" + sudo apt-get update + sudo apt-get install -y gnupg + curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - + curl https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list + sudo apt-get update + sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 - # Print the token (for debugging) - - name: Print Access Token + - name: Install Python dependencies run: | - echo "Access Token: ${{ env.ACCESS_TOKEN }}" + python3 -m venv venv + source venv/bin/activate + pip install azure-identity pyodbc + + - name: Connect to Azure SQL Database + run: | + python - < Date: Fri, 22 Nov 2024 18:46:33 -0800 Subject: [PATCH 22/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index b2ba749..1dbe97f 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -45,11 +45,12 @@ jobs: run: | python3 -m venv venv source venv/bin/activate - pip install azure-identity pyodbc + pip install azure-identity pyodbc azure-core - name: Connect to Azure SQL Database run: | python - < Date: Fri, 22 Nov 2024 18:49:36 -0800 Subject: [PATCH 23/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 1dbe97f..0621c02 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -41,14 +41,12 @@ jobs: sudo apt-get update sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 - - name: Install Python dependencies + - name: Connect to Azure SQL Database run: | python3 -m venv venv source venv/bin/activate pip install azure-identity pyodbc azure-core - - name: Connect to Azure SQL Database - run: | python - < Date: Fri, 22 Nov 2024 18:53:35 -0800 Subject: [PATCH 24/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 0621c02..731cfd5 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -57,8 +57,8 @@ jobs: connection_string = ( "Driver={ODBC Driver 18 for SQL Server};" - "Server=x6eps4xrq2xudenlfv6naeo3i4-og453ge3xn7utn6wff5ltyqjta.daily-datawarehouse.fabric.microsoft.com" - "Database=collationtest" + "Server=x6eps4xrq2xudenlfv6naeo3i4-6xw4uystlgdevluyqmndlcagwe.msit-datawarehouse.fabric.microsoft.com" + "Database=permissionstest" "Authentication=ActiveDirectoryAccessToken;" ) From 5b32d43a003f0484d4b6d33cf9b795bb270e98e3 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 18:56:38 -0800 Subject: [PATCH 25/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 38 ++++++++++--------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 731cfd5..7ab4919 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -51,25 +51,29 @@ jobs: from azure.core.credentials import AccessToken from azure.identity import DefaultAzureCredential import pyodbc + import logging + try: + credential = DefaultAzureCredential() + token = credential.get_token("https://database.windows.net/.default") - credential = DefaultAzureCredential() - token = credential.get_token("https://database.windows.net/.default") + connection_string = ( + "Driver={ODBC Driver 18 for SQL Server};" + "Server=x6eps4xrq2xudenlfv6naeo3i4-6xw4uystlgdevluyqmndlcagwe.msit-datawarehouse.fabric.microsoft.com" + "Database=permissionstest" + "Authentication=ActiveDirectoryAccessToken;" + ) - connection_string = ( - "Driver={ODBC Driver 18 for SQL Server};" - "Server=x6eps4xrq2xudenlfv6naeo3i4-6xw4uystlgdevluyqmndlcagwe.msit-datawarehouse.fabric.microsoft.com" - "Database=permissionstest" - "Authentication=ActiveDirectoryAccessToken;" - ) + access_token = token.token + print(access_token) + connection = pyodbc.connect(connection_string, attrs_before={1256: access_token}) + cursor = connection.cursor() - access_token = token.token - connection = pyodbc.connect(connection_string, attrs_before={1256: access_token}) - cursor = connection.cursor() + cursor.execute("SELECT TOP 10 * FROM dbo.Trip") + rows = cursor.fetchall() + for row in rows: + print(row) - cursor.execute("SELECT TOP 10 * FROM dbo.Trip") - rows = cursor.fetchall() - for row in rows: - print(row) - - connection.close() + connection.close() + except pyodbc.Error as e: + logging.error("Error occurred while connecting to the database.", exc_info=True) EOF From 0a396662c2901f3898748003fce3ced3868e48f1 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 19:03:52 -0800 Subject: [PATCH 26/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 7ab4919..2e70045 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -64,7 +64,8 @@ jobs: ) access_token = token.token - print(access_token) + print(connection_string) + logging.info("connection strin is: ", connection_string) connection = pyodbc.connect(connection_string, attrs_before={1256: access_token}) cursor = connection.cursor() From 5825ed2a2d4ccc55a4c0f889dcc842368ea69911 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 19:08:13 -0800 Subject: [PATCH 27/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 2e70045..51fc0a8 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -58,8 +58,8 @@ jobs: connection_string = ( "Driver={ODBC Driver 18 for SQL Server};" - "Server=x6eps4xrq2xudenlfv6naeo3i4-6xw4uystlgdevluyqmndlcagwe.msit-datawarehouse.fabric.microsoft.com" - "Database=permissionstest" + "Server=x6eps4xrq2xudenlfv6naeo3i4-6xw4uystlgdevluyqmndlcagwe.msit-datawarehouse.fabric.microsoft.com;" + "Database=permissionstest;" "Authentication=ActiveDirectoryAccessToken;" ) From 30c23d7f071bb7eb65f9413d0130df200eac82ac Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 19:15:22 -0800 Subject: [PATCH 28/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 51fc0a8..5ddb25e 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -57,10 +57,10 @@ jobs: token = credential.get_token("https://database.windows.net/.default") connection_string = ( - "Driver={ODBC Driver 18 for SQL Server};" + "Driver={{ODBC Driver 18 for SQL Server}};" "Server=x6eps4xrq2xudenlfv6naeo3i4-6xw4uystlgdevluyqmndlcagwe.msit-datawarehouse.fabric.microsoft.com;" "Database=permissionstest;" - "Authentication=ActiveDirectoryAccessToken;" + "Authentication=ActiveDirectoryAccessToken" ) access_token = token.token From b66838f2148718516067f5a7597f1450c243924d Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Sun, 24 Nov 2024 18:37:34 -0800 Subject: [PATCH 29/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 5ddb25e..5721451 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -57,7 +57,7 @@ jobs: token = credential.get_token("https://database.windows.net/.default") connection_string = ( - "Driver={{ODBC Driver 18 for SQL Server}};" + "Driver={ODBC Driver 18 for SQL Server};" "Server=x6eps4xrq2xudenlfv6naeo3i4-6xw4uystlgdevluyqmndlcagwe.msit-datawarehouse.fabric.microsoft.com;" "Database=permissionstest;" "Authentication=ActiveDirectoryAccessToken" From 09480c47ac89c342082e78ff2e33573536fdbe6e Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Sun, 24 Nov 2024 18:46:42 -0800 Subject: [PATCH 30/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 5721451..cc074f8 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -60,7 +60,6 @@ jobs: "Driver={ODBC Driver 18 for SQL Server};" "Server=x6eps4xrq2xudenlfv6naeo3i4-6xw4uystlgdevluyqmndlcagwe.msit-datawarehouse.fabric.microsoft.com;" "Database=permissionstest;" - "Authentication=ActiveDirectoryAccessToken" ) access_token = token.token From 050b35b5842f07f61a7a481d7a96daacf909d516 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Sun, 24 Nov 2024 18:51:27 -0800 Subject: [PATCH 31/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index cc074f8..806b52e 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -62,10 +62,12 @@ jobs: "Database=permissionstest;" ) - access_token = token.token - print(connection_string) - logging.info("connection strin is: ", connection_string) - connection = pyodbc.connect(connection_string, attrs_before={1256: access_token}) + access_token = token.token.encode('utf-16-le') + token_struct = struct.pack(f' Date: Sun, 24 Nov 2024 18:53:54 -0800 Subject: [PATCH 32/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 806b52e..3f121e9 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -52,6 +52,7 @@ jobs: from azure.identity import DefaultAzureCredential import pyodbc import logging + import struct try: credential = DefaultAzureCredential() token = credential.get_token("https://database.windows.net/.default") From 3d65834cf367e76eb0b37dc81eb450d175831655 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Sun, 24 Nov 2024 21:56:31 -0800 Subject: [PATCH 33/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 3f121e9..eff709c 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -22,6 +22,8 @@ jobs: msodbc_version: ["17", "18"] runs-on: ubuntu-latest + container: + image: ghcr.io/${{ github.repository }}:CI-${{ matrix.python_version }}-msodbc${{ matrix.msodbc_version }} steps: # Azure login using federated credentials - name: Azure login with OIDC From 3cf4060be45f4070ef51ad13537f947e88449a79 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Sun, 24 Nov 2024 21:59:02 -0800 Subject: [PATCH 34/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index eff709c..b07795e 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -6,9 +6,7 @@ on: # yamllint disable-line rule:truthy branches: - oidc_connect -permissions: - id-token: write - contents: read + jobs: integration-tests-fabric-dw: @@ -22,6 +20,9 @@ jobs: msodbc_version: ["17", "18"] runs-on: ubuntu-latest + permissions: + id-token: write + contents: read container: image: ghcr.io/${{ github.repository }}:CI-${{ matrix.python_version }}-msodbc${{ matrix.msodbc_version }} steps: From aa37756c869909f459764e13345cd85eeaaf2526 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Sun, 24 Nov 2024 22:12:20 -0800 Subject: [PATCH 35/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index b07795e..81b43b9 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -21,8 +21,9 @@ jobs: runs-on: ubuntu-latest permissions: - id-token: write - contents: read + contents: read # Required to access repository files + packages: read # Grant explicit read access to packages + id-token: write # Needed if using OIDC authentication container: image: ghcr.io/${{ github.repository }}:CI-${{ matrix.python_version }}-msodbc${{ matrix.msodbc_version }} steps: @@ -71,7 +72,7 @@ jobs: SQL_COPT_SS_ACCESS_TOKEN = 1256 # This connection option is defined by microsoft in msodbcsql.h connection = pyodbc.connect(connection_string, attrs_before={SQL_COPT_SS_ACCESS_TOKEN: token_struct}) - logging.info("connection strin is: ", connection) + logging.info("connection strin is: ", token_struct) cursor = connection.cursor() cursor.execute("SELECT TOP 10 * FROM dbo.Trip") From 64e1a50b7e33e8a31ef648008fe6b55aea480ad0 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Sun, 24 Nov 2024 22:23:37 -0800 Subject: [PATCH 36/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 81b43b9..35f513d 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -36,15 +36,6 @@ jobs: allow-no-subscriptions: true federated-token: true - - name: Install ODBC Driver 18 for SQL Server (Ubuntu) - run: | - sudo apt-get update - sudo apt-get install -y gnupg - curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - - curl https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list - sudo apt-get update - sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 - - name: Connect to Azure SQL Database run: | python3 -m venv venv From 35491f116604928881ed5bb9c219db1f471a2de2 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Sun, 24 Nov 2024 22:26:11 -0800 Subject: [PATCH 37/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 35f513d..3b6712e 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -38,8 +38,6 @@ jobs: - name: Connect to Azure SQL Database run: | - python3 -m venv venv - source venv/bin/activate pip install azure-identity pyodbc azure-core python - < Date: Sun, 24 Nov 2024 22:36:23 -0800 Subject: [PATCH 38/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 3b6712e..74782fc 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -62,6 +62,7 @@ jobs: connection = pyodbc.connect(connection_string, attrs_before={SQL_COPT_SS_ACCESS_TOKEN: token_struct}) logging.info("connection strin is: ", token_struct) + print("token struct is ", token_struct) cursor = connection.cursor() cursor.execute("SELECT TOP 10 * FROM dbo.Trip") From 9cb00eba98aa734a2acd71f33c22a3377023750e Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Sun, 24 Nov 2024 22:44:12 -0800 Subject: [PATCH 39/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 74782fc..872d163 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -61,8 +61,8 @@ jobs: SQL_COPT_SS_ACCESS_TOKEN = 1256 # This connection option is defined by microsoft in msodbcsql.h connection = pyodbc.connect(connection_string, attrs_before={SQL_COPT_SS_ACCESS_TOKEN: token_struct}) - logging.info("connection strin is: ", token_struct) - print("token struct is ", token_struct) + print("access token is ", access_token) + print("token is ", token.token) cursor = connection.cursor() cursor.execute("SELECT TOP 10 * FROM dbo.Trip") From fbe564dd21babd58270a58eeb44eebef6504dfb7 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Sun, 24 Nov 2024 22:48:40 -0800 Subject: [PATCH 40/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 872d163..4b3da67 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -61,7 +61,6 @@ jobs: SQL_COPT_SS_ACCESS_TOKEN = 1256 # This connection option is defined by microsoft in msodbcsql.h connection = pyodbc.connect(connection_string, attrs_before={SQL_COPT_SS_ACCESS_TOKEN: token_struct}) - print("access token is ", access_token) print("token is ", token.token) cursor = connection.cursor() From 46f0559efb58d599e436e7218163758218c129a0 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Mon, 25 Nov 2024 18:41:43 -0800 Subject: [PATCH 41/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 103 ++++++++++-------- .../fabric/fabric_connection_manager.py | 40 ++++++- dbt/adapters/fabric/fabric_credentials.py | 1 + tests/conftest.py | 19 +++- 4 files changed, 114 insertions(+), 49 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 4b3da67..930cd3a 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -1,13 +1,10 @@ ---- name: Integration tests on Fabric DW -on: # yamllint disable-line rule:truthy +on: # yamllint disable-line rule:truthy workflow_dispatch: pull_request: branches: - oidc_connect - - jobs: integration-tests-fabric-dw: name: Regular @@ -15,9 +12,9 @@ jobs: fail-fast: false max-parallel: 1 matrix: - profile: ["ci_azure_auto"] + profile: ["integration_tests"] python_version: ["3.11"] - msodbc_version: ["17", "18"] + msodbc_version: ["18"] runs-on: ubuntu-latest permissions: @@ -27,49 +24,65 @@ jobs: container: image: ghcr.io/${{ github.repository }}:CI-${{ matrix.python_version }}-msodbc${{ matrix.msodbc_version }} steps: - # Azure login using federated credentials - - name: Azure login with OIDC - uses: azure/login@v2 - with: - client-id: ${{ secrets.DBT_AZURE_SP_NAME }} - tenant-id: ${{ secrets.DBT_AZURE_TENANT }} - allow-no-subscriptions: true - federated-token: true + # Azure login using federated credentials + - name: Azure login with OIDC + uses: azure/login@v2 + with: + client-id: ${{ secrets.DBT_AZURE_SP_NAME }} + tenant-id: ${{ secrets.DBT_AZURE_TENANT }} + allow-no-subscriptions: true + federated-token: true + + - name: Connect to Fabric Warehouse to Retrieve Token + id: fetch_token + run: | + pip install azure-identity pyodbc azure-core + + python - < AccessToken: } -def get_pyodbc_attrs_before(credentials: FabricCredentials) -> Dict: +def get_pyodbc_attrs_before_credentials(credentials: FabricCredentials) -> Dict: """ Get the pyodbc attrs before. @@ -220,6 +220,36 @@ def get_pyodbc_attrs_before(credentials: FabricCredentials) -> Dict: return attrs_before +def get_pyodbc_attrs_before_accesstoken(accessToken: str) -> Dict: + """ + Get the pyodbc attrs before. + + Parameters + ---------- + credentials : Access Token for Integration Tests + Credentials. + + Returns + ------- + out : Dict + The pyodbc attrs before. + + Source + ------ + Authentication for SQL server with an access token: + https://docs.microsoft.com/en-us/sql/connect/odbc/using-azure-active-directory?view=sql-server-ver15#authenticating-with-an-access-token + """ + + access_token_utf16 = accessToken.encode("utf-16-le") + token_struct = struct.pack( + f" str: """ Convert a boolean to a connection string argument. @@ -323,7 +353,7 @@ def open(cls, connection: Connection) -> Connection: con_str.append(f"Database={credentials.database}") - #Enabling trace flag + # Enabling trace flag if credentials.trace_flag: con_str.append("SQL_ATTR_TRACE=SQL_OPT_TRACE_ON") else: @@ -395,7 +425,11 @@ def open(cls, connection: Connection) -> Connection: def connect(): logger.debug(f"Using connection string: {con_str_display}") - attrs_before = get_pyodbc_attrs_before(credentials) + if credentials.authentication == "ActiveDirectoryAccessToken": + attrs_before = get_pyodbc_attrs_before_accesstoken(credentials.access_token) + else: + attrs_before = get_pyodbc_attrs_before_credentials(credentials) + handle = pyodbc.connect( con_str_concat, attrs_before=attrs_before, diff --git a/dbt/adapters/fabric/fabric_credentials.py b/dbt/adapters/fabric/fabric_credentials.py index a824fac..138e3bd 100644 --- a/dbt/adapters/fabric/fabric_credentials.py +++ b/dbt/adapters/fabric/fabric_credentials.py @@ -17,6 +17,7 @@ class FabricCredentials(Credentials): tenant_id: Optional[str] = None client_id: Optional[str] = None client_secret: Optional[str] = None + access_token: Optional[str] = None authentication: Optional[str] = "ActiveDirectoryServicePrincipal" encrypt: Optional[bool] = True # default value in MS ODBC Driver 18 as well trust_cert: Optional[bool] = False # default value in MS ODBC Driver 18 as well diff --git a/tests/conftest.py b/tests/conftest.py index 3e60ce0..72eb2d6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -27,6 +27,8 @@ def dbt_profile_target(request: FixtureRequest, dbt_profile_target_update): target = _profile_ci_azure_environment() elif profile == "user_azure": target = _profile_user_azure() + elif profile == "integration_tests": + target = _profile_integration_tests() else: raise ValueError(f"Unknown profile: {profile}") @@ -55,7 +57,7 @@ def _profile_ci_azure_base(): "database": os.getenv("DBT_AZURESQL_DB"), "encrypt": True, "trust_cert": True, - "trace_flag":False, + "trace_flag": False, }, } @@ -104,6 +106,21 @@ def _profile_user_azure(): return profile +def _profile_integration_tests(): + profile = { + **_all_profiles_base(), + **{ + "host": os.getenv("FABRIC_TEST_HOST"), + "authentication": os.getenv("FABRIC_TEST_AUTH", "ActiveDirectoryAccessToken"), + "encrypt": True, + "trust_cert": True, + "database": os.getenv("FABRIC_TEST_DBNAME"), + "access_token": os.getenv("FABRIC_INTEGRATION_TESTS_TOKEN"), + }, + } + return profile + + @pytest.fixture(autouse=True) def skip_by_profile_type(request: FixtureRequest): profile_type = request.config.getoption("--profile") From c86e4664e78cf56740e18c47a82406e2ed8f6160 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Mon, 25 Nov 2024 18:45:09 -0800 Subject: [PATCH 42/53] Testing OIDC --- dev_requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev_requirements.txt b/dev_requirements.txt index d3313a4..c980ca5 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,6 +1,6 @@ # install latest changes in dbt-core # TODO: how to automate switching from develop to version branches? -git+https://github.com/dbt-labs/dbt-core.git@v1.8.0#egg=dbt-core&subdirectory=core +git+https://github.com/dbt-labs/dbt-core.git@v1.8.6#egg=dbt-core&subdirectory=core git+https://github.com/dbt-labs/dbt-adapters.git git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter git+https://github.com/dbt-labs/dbt-common.git diff --git a/setup.py b/setup.py index 3deed70..e521080 100644 --- a/setup.py +++ b/setup.py @@ -69,7 +69,7 @@ def run(self): "pyodbc>=4.0.35,<5.2.0", "azure-identity>=1.12.0", "dbt-common>=1.0.4,<2.0", - "dbt-core>=1.8.0", + "dbt-core>=1.8.6", "dbt-adapters>=1.1.1,<2.0", ], cmdclass={ From 6b3115f02b3ea103685a53946e96450f002b2a72 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Mon, 25 Nov 2024 19:04:35 -0800 Subject: [PATCH 43/53] Testing OIDC --- dev_requirements.txt | 3 +-- setup.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/dev_requirements.txt b/dev_requirements.txt index c980ca5..f28c23a 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,9 +1,8 @@ # install latest changes in dbt-core # TODO: how to automate switching from develop to version branches? -git+https://github.com/dbt-labs/dbt-core.git@v1.8.6#egg=dbt-core&subdirectory=core +git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core git+https://github.com/dbt-labs/dbt-adapters.git git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter -git+https://github.com/dbt-labs/dbt-common.git pytest==8.0.1 twine==5.1.1 diff --git a/setup.py b/setup.py index e521080..3deed70 100644 --- a/setup.py +++ b/setup.py @@ -69,7 +69,7 @@ def run(self): "pyodbc>=4.0.35,<5.2.0", "azure-identity>=1.12.0", "dbt-common>=1.0.4,<2.0", - "dbt-core>=1.8.6", + "dbt-core>=1.8.0", "dbt-adapters>=1.1.1,<2.0", ], cmdclass={ From 6a5c07666be04eb4bb36756dc4480a90b45f81af Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Mon, 25 Nov 2024 19:10:02 -0800 Subject: [PATCH 44/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 930cd3a..071cf30 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -85,4 +85,4 @@ jobs: DBT_TEST_USER_1: dbo DBT_TEST_USER_2: dbo DBT_TEST_USER_3: dbo - run: pytest -ra -v tests/functional/adapter/test_empty.py --profile "${{ matrix.profile }}" + run: pytest -ra -v tests/functional --profile "${{ matrix.profile }}" From 61eb419803bd39bd17e0d069b3fd8a5eb4d61e8a Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Mon, 25 Nov 2024 19:29:35 -0800 Subject: [PATCH 45/53] Testing OIDC --- tests/conftest.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 72eb2d6..adbcbb0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -108,13 +108,9 @@ def _profile_user_azure(): def _profile_integration_tests(): profile = { - **_all_profiles_base(), + **_profile_ci_azure_base(), **{ - "host": os.getenv("FABRIC_TEST_HOST"), "authentication": os.getenv("FABRIC_TEST_AUTH", "ActiveDirectoryAccessToken"), - "encrypt": True, - "trust_cert": True, - "database": os.getenv("FABRIC_TEST_DBNAME"), "access_token": os.getenv("FABRIC_INTEGRATION_TESTS_TOKEN"), }, } From 4e738e67da862f44a80dc417620ad88f7669d568 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Mon, 25 Nov 2024 20:06:55 -0800 Subject: [PATCH 46/53] Testing OIDC --- dbt/adapters/fabric/fabric_connection_manager.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dbt/adapters/fabric/fabric_connection_manager.py b/dbt/adapters/fabric/fabric_connection_manager.py index 6fc7dcc..e0cf33a 100644 --- a/dbt/adapters/fabric/fabric_connection_manager.py +++ b/dbt/adapters/fabric/fabric_connection_manager.py @@ -361,7 +361,10 @@ def open(cls, connection: Connection) -> Connection: assert credentials.authentication is not None - if "ActiveDirectory" in credentials.authentication: + if ( + "ActiveDirectory" in credentials.authentication + and credentials.authentication != "ActiveDirectoryAccessToken" + ): con_str.append(f"Authentication={credentials.authentication}") if credentials.authentication == "ActiveDirectoryPassword": From a650d99a7f8a1388b3c6d860871f9b9e96692bc8 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Mon, 25 Nov 2024 22:59:40 -0800 Subject: [PATCH 47/53] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 21 +------------------ 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 071cf30..dfad12d 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -33,7 +33,7 @@ jobs: allow-no-subscriptions: true federated-token: true - - name: Connect to Fabric Warehouse to Retrieve Token + - name: Test Connection To Fabric Data Warehouse id: fetch_token run: | pip install azure-identity pyodbc azure-core @@ -47,26 +47,7 @@ jobs: try: credential = DefaultAzureCredential() token = credential.get_token("https://database.windows.net/.default") - - connection_string = ( - "Driver={ODBC Driver 18 for SQL Server};" - "Server=x6eps4xrq2xudenlfv6naeo3i4-6xw4uystlgdevluyqmndlcagwe.msit-datawarehouse.fabric.microsoft.com;" - "Database=permissionstest;" - ) print(f"::set-output name=access_token::{token.token}") - access_token_utf16 = token.token.encode('utf-16-le') - token_struct = struct.pack(f' Date: Wed, 27 Nov 2024 15:25:59 -0800 Subject: [PATCH 48/53] configuring local tests to run on user credentials. Dropping a relation correctly based on its type --- .github/workflows/integration-tests-azure.yml | 2 +- .github/workflows/publish-docker.yml | 1 - dbt/include/fabric/macros/adapters/show.sql | 26 +++++++------------ .../materializations/models/table/table.sql | 25 +++++++++++------- tests/conftest.py | 5 +--- 5 files changed, 26 insertions(+), 33 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index dfad12d..bbeed0e 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -3,7 +3,7 @@ on: # yamllint disable-line rule:truthy workflow_dispatch: pull_request: branches: - - oidc_connect + - main jobs: integration-tests-fabric-dw: diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml index c0a88e8..4ac4004 100644 --- a/.github/workflows/publish-docker.yml +++ b/.github/workflows/publish-docker.yml @@ -9,7 +9,6 @@ on: # yamllint disable-line rule:truthy branches: - main - v* - - oidc_connect jobs: publish-docker-client: diff --git a/dbt/include/fabric/macros/adapters/show.sql b/dbt/include/fabric/macros/adapters/show.sql index 205fa1f..7b3340e 100644 --- a/dbt/include/fabric/macros/adapters/show.sql +++ b/dbt/include/fabric/macros/adapters/show.sql @@ -1,24 +1,16 @@ {% macro fabric__get_limit_sql(sql, limit) %} {% if limit == -1 or limit is none %} - {% if sql.strip().lower().startswith('with') %} + with model_limit_subq as ( {{ sql }} - {% else -%} - select * - from ( - {{ sql }} - ) as model_limit_subq - {%- endif -%} + ) + select * + from model_limit_subq; {% else -%} - {% if sql.strip().lower().startswith('with') %} - {{ sql }} order by (select null) - offset 0 rows fetch first {{ limit }} rows only - {% else -%} - select * - from ( - {{ sql }} - ) as model_limit_subq order by (select null) - offset 0 rows fetch first {{ limit }} rows only - {%- endif -%} + with model_limit_subq as ( + {{ sql }} + ) + select top {{ limit }} * + from model_limit_subq; {%- endif -%} {% endmacro %} diff --git a/dbt/include/fabric/macros/materializations/models/table/table.sql b/dbt/include/fabric/macros/materializations/models/table/table.sql index 4eb1f4a..6964013 100644 --- a/dbt/include/fabric/macros/materializations/models/table/table.sql +++ b/dbt/include/fabric/macros/materializations/models/table/table.sql @@ -35,20 +35,25 @@ {% if existing_relation is not none and existing_relation.is_table %} - -- making a backup relation, this will come in use when contract is enforced or not - {%- set backup_relation = make_backup_relation(existing_relation, 'table') -%} + -- making a backup relation, this will come in use when contract is enforced or not + {%- set set_backup_relation = adapter.get_relation(database=this.database, schema=this.schema, identifier=this.identifier) -%} + {% if (set_backup_relation != none and set_backup_relation.type == "table") %} + {%- set backup_relation = make_backup_relation(target_relation, 'table') -%} + {% elif (set_backup_relation != none and set_backup_relation.type == "view") %} + {%- set backup_relation = make_backup_relation(target_relation, 'view') -%} + {% endif %} - -- Dropping a temp relation if it exists - {{ adapter.drop_relation(backup_relation) }} + -- Dropping a temp relation if it exists + {{ adapter.drop_relation(backup_relation) }} - -- Rename existing relation to back up relation - {{ adapter.rename_relation(existing_relation, backup_relation) }} + -- Rename existing relation to back up relation + {{ adapter.rename_relation(existing_relation, backup_relation) }} - -- Renaming temp relation as main relation - {{ adapter.rename_relation(temp_relation, target_relation) }} + -- Renaming temp relation as main relation + {{ adapter.rename_relation(temp_relation, target_relation) }} - -- Drop backup relation - {{ adapter.drop_relation(backup_relation) }} + -- Drop backup relation + {{ adapter.drop_relation(backup_relation) }} {%- else %} diff --git a/tests/conftest.py b/tests/conftest.py index adbcbb0..ecb2e67 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -94,13 +94,10 @@ def _profile_user_azure(): **_all_profiles_base(), **{ "host": os.getenv("FABRIC_TEST_HOST"), - "authentication": os.getenv("FABRIC_TEST_AUTH", "auto"), + "authentication": os.getenv("FABRIC_TEST_AUTH", "CLI"), "encrypt": True, "trust_cert": True, "database": os.getenv("FABRIC_TEST_DBNAME"), - "client_id": os.getenv("FABRIC_TEST_CLIENT_ID"), - "client_secret": os.getenv("FABRIC_TEST_CLIENT_SECRET"), - "tenant_id": os.getenv("FABRIC_TEST_TENANT_ID"), }, } return profile From 19ff404636af93484910e702930cff70adb4134c Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Wed, 27 Nov 2024 19:12:34 -0800 Subject: [PATCH 49/53] Addressing #243, #221, #228, #229, #232, #235 issues --- .../fabric/macros/adapters/catalog.sql | 4 +- dbt/include/fabric/macros/adapters/show.sql | 22 ++++---- .../materializations/snapshots/helpers.sql | 54 ++----------------- .../materializations/snapshots/snapshot.sql | 9 ++-- 4 files changed, 21 insertions(+), 68 deletions(-) diff --git a/dbt/include/fabric/macros/adapters/catalog.sql b/dbt/include/fabric/macros/adapters/catalog.sql index 6bd1398..555b5a7 100644 --- a/dbt/include/fabric/macros/adapters/catalog.sql +++ b/dbt/include/fabric/macros/adapters/catalog.sql @@ -96,7 +96,7 @@ c.column_id as column_index, t.name as column_type from sys.columns as c {{ information_schema_hints() }} - left join sys.types as t on c.system_type_id = t.system_type_id {{ information_schema_hints() }} + left join sys.types as t on c.system_type_id = t.system_type_id ) select @@ -223,7 +223,7 @@ c.column_id as column_index, t.name as column_type from sys.columns as c {{ information_schema_hints() }} - left join sys.types as t on c.system_type_id = t.system_type_id {{ information_schema_hints() }} + left join sys.types as t on c.system_type_id = t.system_type_id ) select diff --git a/dbt/include/fabric/macros/adapters/show.sql b/dbt/include/fabric/macros/adapters/show.sql index 7b3340e..d6e1e3d 100644 --- a/dbt/include/fabric/macros/adapters/show.sql +++ b/dbt/include/fabric/macros/adapters/show.sql @@ -1,16 +1,12 @@ {% macro fabric__get_limit_sql(sql, limit) %} - - {% if limit == -1 or limit is none %} - with model_limit_subq as ( - {{ sql }} - ) - select * - from model_limit_subq; - {% else -%} - with model_limit_subq as ( - {{ sql }} - ) - select top {{ limit }} * - from model_limit_subq; + {%- if limit == -1 or limit is none -%} + {{ sql }} + {#- Special processing if the last non-blank line starts with order by -#} + {%- elif 'order by' in sql.strip().splitlines()[-1].strip().lower() -%} + {{ sql }} + offset 0 rows fetch first {{ limit }} rows only + {%- else -%} + {{ sql }} + order by (select null) offset 0 rows fetch first {{ limit }} rows only {%- endif -%} {% endmacro %} diff --git a/dbt/include/fabric/macros/materializations/snapshots/helpers.sql b/dbt/include/fabric/macros/materializations/snapshots/helpers.sql index 29d4934..d5915e4 100644 --- a/dbt/include/fabric/macros/materializations/snapshots/helpers.sql +++ b/dbt/include/fabric/macros/materializations/snapshots/helpers.sql @@ -3,56 +3,12 @@ {% do drop_relation_if_exists(staging_relation) %} {% endmacro %} ---Due to Alter not being supported, have to rely on this for temporarily {% macro fabric__create_columns(relation, columns) %} - {# default__ macro uses "add column" - TSQL preferes just "add" - #} - - {% set columns %} - {% for column in columns %} - , CAST(NULL AS {{column.data_type}}) AS {{column.name}} - {% endfor %} - {% endset %} - - {% set tempTableName %} - [{{relation.database}}].[{{ relation.schema }}].[{{ relation.identifier }}_{{ range(1300, 19000) | random }}] - {% endset %} - {{ log("Creating new columns are not supported without dropping a table. Using random table as a temp table. - " ~ tempTableName) }} - - {% set tempTable %} - CREATE TABLE {{tempTableName}} - AS SELECT * {{columns}} FROM [{{relation.database}}].[{{ relation.schema }}].[{{ relation.identifier }}] {{ information_schema_hints() }} {{ apply_label() }} - {% endset %} - - {% call statement('create_temp_table') -%} - {{ tempTable }} - {%- endcall %} - - {% set dropTable %} - DROP TABLE [{{relation.database}}].[{{ relation.schema }}].[{{ relation.identifier }}] - {% endset %} - - {% call statement('drop_table') -%} - {{ dropTable }} - {%- endcall %} - - {% set createTable %} - CREATE TABLE {{ relation }} - AS SELECT * FROM {{tempTableName}} {{ information_schema_hints() }} {{ apply_label() }} - {% endset %} - - {% call statement('create_Table') -%} - {{ createTable }} - {%- endcall %} - - {% set dropTempTable %} - DROP TABLE {{tempTableName}} - {% endset %} - - {% call statement('drop_temp_table') -%} - {{ dropTempTable }} - {%- endcall %} + {% for column in columns %} + {% call statement() %} + alter table {{ relation.render() }} add "{{ column.name }}" {{ column.data_type }} NULL; + {% endcall %} + {% endfor %} {% endmacro %} {% macro fabric__get_true_sql() %} diff --git a/dbt/include/fabric/macros/materializations/snapshots/snapshot.sql b/dbt/include/fabric/macros/materializations/snapshots/snapshot.sql index 07c6789..b10ad03 100644 --- a/dbt/include/fabric/macros/materializations/snapshots/snapshot.sql +++ b/dbt/include/fabric/macros/materializations/snapshots/snapshot.sql @@ -31,10 +31,11 @@ -- Create a temporary view to manage if user SQl uses CTE {% set temp_snapshot_relation_sql = model['compiled_code'].replace("'", "''") %} - {% call statement('create temp_snapshot_relation') %} - EXEC('DROP VIEW IF EXISTS {{ temp_snapshot_relation.include(database=False) }};'); - EXEC('create view {{ temp_snapshot_relation.include(database=False) }} as {{ temp_snapshot_relation_sql }};'); - {% endcall %} + + {% call statement('create temp_snapshot_relation') -%} + {{ adapter.drop_relation(temp_snapshot_relation) }} + {{ get_create_view_as_sql(temp_snapshot_relation, temp_snapshot_relation_sql) }} + {%- endcall %} {% if not target_relation_exists %} From bdb4b811b76adb7e71d2e5d739a7c63321e59db0 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Sat, 30 Nov 2024 17:55:40 -0800 Subject: [PATCH 50/53] Updated test helper with ephemeral --- .../materializations/snapshots/snapshot.sql | 2 +- .../macros/materializations/tests/helpers.sql | 46 ++++--------------- 2 files changed, 9 insertions(+), 39 deletions(-) diff --git a/dbt/include/fabric/macros/materializations/snapshots/snapshot.sql b/dbt/include/fabric/macros/materializations/snapshots/snapshot.sql index b10ad03..9ed2f91 100644 --- a/dbt/include/fabric/macros/materializations/snapshots/snapshot.sql +++ b/dbt/include/fabric/macros/materializations/snapshots/snapshot.sql @@ -31,9 +31,9 @@ -- Create a temporary view to manage if user SQl uses CTE {% set temp_snapshot_relation_sql = model['compiled_code'].replace("'", "''") %} + {{ adapter.drop_relation(temp_snapshot_relation) }} {% call statement('create temp_snapshot_relation') -%} - {{ adapter.drop_relation(temp_snapshot_relation) }} {{ get_create_view_as_sql(temp_snapshot_relation, temp_snapshot_relation_sql) }} {%- endcall %} diff --git a/dbt/include/fabric/macros/materializations/tests/helpers.sql b/dbt/include/fabric/macros/materializations/tests/helpers.sql index 4f04547..4fd157c 100644 --- a/dbt/include/fabric/macros/materializations/tests/helpers.sql +++ b/dbt/include/fabric/macros/materializations/tests/helpers.sql @@ -1,42 +1,12 @@ {% macro fabric__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%} - -- Create target schema if it does not - USE [{{ target.database }}]; - IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = '{{ target.schema }}') - BEGIN - EXEC('CREATE SCHEMA [{{ target.schema }}]') - END + With dbt_internal_test AS ( + {{ main_sql }} + ) + select + COUNT(*) AS failures, + CASE WHEN COUNT(*) != 0 THEN 'true' ELSE 'false' END AS should_warn, + CASE WHEN COUNT(*) != 0 THEN 'true' ELSE 'false' END AS should_error + FROM dbt_internal_test - {% if main_sql.strip().lower().startswith('with') %} - {% set testview %} - [{{ target.schema }}.testview_{{ range(1300, 19000) | random }}] - {% endset %} - - {% set sql = main_sql.replace("'", "''")%} - EXEC('create view {{testview}} as {{ sql }};') - select - {{ "top (" ~ limit ~ ')' if limit != none }} - {{ fail_calc }} as failures, - case when {{ fail_calc }} {{ warn_if }} - then 'true' else 'false' end as should_warn, - case when {{ fail_calc }} {{ error_if }} - then 'true' else 'false' end as should_error - from ( - select * from {{testview}} - ) dbt_internal_test; - - EXEC('drop view {{testview}};') - - {% else -%} - select - {{ "top (" ~ limit ~ ')' if limit != none }} - {{ fail_calc }} as failures, - case when {{ fail_calc }} {{ warn_if }} - then 'true' else 'false' end as should_warn, - case when {{ fail_calc }} {{ error_if }} - then 'true' else 'false' end as should_error - from ( - {{ main_sql }} - ) dbt_internal_test - {%- endif -%} {%- endmacro %} From 899a9904d98b9e302a8b583ae9977c27a8668aea Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Sat, 30 Nov 2024 18:01:11 -0800 Subject: [PATCH 51/53] Updated unit tests --- .github/workflows/unit-tests.yml | 2 +- .../unit/adapters/fabric/test_sql_server_connection_manager.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index b0fdc9e..05a85a7 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -16,7 +16,7 @@ jobs: name: Unit tests strategy: matrix: - python_version: ["3.8", "3.9", "3.10", "3.11"] + python_version: ["3.10", "3.11"] runs-on: ubuntu-latest permissions: contents: read diff --git a/tests/unit/adapters/fabric/test_sql_server_connection_manager.py b/tests/unit/adapters/fabric/test_sql_server_connection_manager.py index 757806b..a6431e9 100644 --- a/tests/unit/adapters/fabric/test_sql_server_connection_manager.py +++ b/tests/unit/adapters/fabric/test_sql_server_connection_manager.py @@ -20,7 +20,7 @@ @pytest.fixture def credentials() -> FabricCredentials: credentials = FabricCredentials( - driver="ODBC Driver 17 for SQL Server", + driver="ODBC Driver 18 for SQL Server", host="fake.sql.fabric.net", database="dbt", schema="fabric", From 4d94cddf9db244fc94fbf73cf7b26d7cc47a2dc3 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Sat, 30 Nov 2024 18:06:16 -0800 Subject: [PATCH 52/53] Updated get_pyodbc_attrs_before_credentials method --- .../adapters/fabric/test_sql_server_connection_manager.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/adapters/fabric/test_sql_server_connection_manager.py b/tests/unit/adapters/fabric/test_sql_server_connection_manager.py index a6431e9..539281c 100644 --- a/tests/unit/adapters/fabric/test_sql_server_connection_manager.py +++ b/tests/unit/adapters/fabric/test_sql_server_connection_manager.py @@ -8,7 +8,7 @@ from dbt.adapters.fabric.fabric_connection_manager import ( bool_to_connection_string_arg, byte_array_to_datetime, - get_pyodbc_attrs_before, + get_pyodbc_attrs_before_credentials, ) from dbt.adapters.fabric.fabric_credentials import FabricCredentials @@ -52,7 +52,7 @@ def test_get_pyodbc_attrs_before_empty_dict_when_service_principal( """ When the authentication is set to sql we expect an empty attrs before. """ - attrs_before = get_pyodbc_attrs_before(credentials) + attrs_before = get_pyodbc_attrs_before_credentials(credentials) assert attrs_before == {} @@ -68,7 +68,7 @@ def test_get_pyodbc_attrs_before_contains_access_token_key_for_cli_authenticatio """ credentials.authentication = authentication with mock.patch(CHECK_OUTPUT, mock.Mock(return_value=mock_cli_access_token)): - attrs_before = get_pyodbc_attrs_before(credentials) + attrs_before = get_pyodbc_attrs_before_credentials(credentials) assert 1256 in attrs_before.keys() From dd918ede67ba65d256e20c5ed04d63c5a746f226 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Sat, 30 Nov 2024 18:35:56 -0800 Subject: [PATCH 53/53] include only lines that start with order by --- dbt/include/fabric/macros/adapters/show.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/fabric/macros/adapters/show.sql b/dbt/include/fabric/macros/adapters/show.sql index d6e1e3d..5b60e8b 100644 --- a/dbt/include/fabric/macros/adapters/show.sql +++ b/dbt/include/fabric/macros/adapters/show.sql @@ -2,7 +2,7 @@ {%- if limit == -1 or limit is none -%} {{ sql }} {#- Special processing if the last non-blank line starts with order by -#} - {%- elif 'order by' in sql.strip().splitlines()[-1].strip().lower() -%} + {%- elif sql.strip().splitlines()[-1].strip().lower().startswith('order by') -%} {{ sql }} offset 0 rows fetch first {{ limit }} rows only {%- else -%}