From 0dc7a5c7b8cfa067825241dc999c7264e9e79922 Mon Sep 17 00:00:00 2001 From: Petro Tiurin <93913847+ptiurin@users.noreply.github.com> Date: Wed, 6 Dec 2023 11:16:22 +0000 Subject: [PATCH] feat: Firebolt backwards compatibility (#112) --- .../unreleased/Added-20231204-143652.yaml | 3 + .github/workflows/integration-tests-v1.yml | 110 ++++++++++++++++++ .github/workflows/integration-tests-v2.yml | 110 ++++++++++++++++++ .github/workflows/integration-tests.yml | 106 ++++------------- .github/workflows/jaffle-shop-v1.yml | 82 +++++++++++++ .github/workflows/jaffle-shop-v2.yml | 89 ++++++++++++++ .github/workflows/jaffle_shop.yml | 86 ++++---------- dbt/adapters/firebolt/connections.py | 45 ++++++- .../models/incremental/merge.sql | 13 +-- .../macros/utils/cast_bool_to_text.sql | 4 +- .../firebolt/macros/utils/datediff.sql | 2 +- setup.cfg | 2 +- tests/conftest.py | 19 ++- tests/unit/test_firebolt_adapter.py | 41 +++++++ 14 files changed, 548 insertions(+), 164 deletions(-) create mode 100644 .changes/unreleased/Added-20231204-143652.yaml create mode 100644 .github/workflows/integration-tests-v1.yml create mode 100644 .github/workflows/integration-tests-v2.yml create mode 100644 .github/workflows/jaffle-shop-v1.yml create mode 100644 .github/workflows/jaffle-shop-v2.yml diff --git a/.changes/unreleased/Added-20231204-143652.yaml b/.changes/unreleased/Added-20231204-143652.yaml new file mode 100644 index 000000000..059cc0bbd --- /dev/null +++ b/.changes/unreleased/Added-20231204-143652.yaml @@ -0,0 +1,3 @@ +kind: Added +body: Ability to connect to new and old Firebolt +time: 2023-12-04T14:36:52.025965Z diff --git a/.github/workflows/integration-tests-v1.yml b/.github/workflows/integration-tests-v1.yml new file mode 100644 index 000000000..de9a16713 --- /dev/null +++ b/.github/workflows/integration-tests-v1.yml @@ -0,0 +1,110 @@ +name: Run integration tests V1 + +on: + workflow_dispatch: + inputs: + environment: + description: 'Environment to run the tests against' + type: choice + required: true + default: 'dev' + options: + - dev + - staging + workflow_call: + inputs: + environment: + default: 'staging' + required: false + type: string + secrets: + FIREBOLT_USERNAME_STAGING: + required: true + FIREBOLT_PASSWORD_STAGING: + required: true + FIREBOLT_USERNAME_DEV: + required: true + FIREBOLT_PASSWORD_DEV: + required: true + +jobs: + tests: + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v2 + + - name: Set up Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: 3.7 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install ".[dev]" --no-cache-dir + + - name: Determine env variables + run: | + if [ "${{ inputs.environment }}" == 'staging' ]; then + echo "USERNAME=${{ secrets.FIREBOLT_USERNAME_STAGING }}" >> "$GITHUB_ENV" + echo "PASSWORD=${{ secrets.FIREBOLT_PASSWORD_STAGING }}" >> "$GITHUB_ENV" + else + echo "USERNAME=${{ secrets.FIREBOLT_USERNAME_DEV }}" >> "$GITHUB_ENV" + echo "PASSWORD=${{ secrets.FIREBOLT_PASSWORD_DEV }}" >> "$GITHUB_ENV" + fi + + - name: Keep environment name in the summary + run: echo '### Ran integration tests against ${{ inputs.environment }} ' >> $GITHUB_STEP_SUMMARY + + - name: Setup database and engine + id: setup + uses: firebolt-db/integration-testing-setup@v1 + with: + firebolt-username: ${{ env.USERNAME }} + firebolt-password: ${{ env.PASSWORD }} + api-endpoint: "api.${{ inputs.environment }}.firebolt.io" + region: "us-east-1" + + - name: Restore cached failed tests + id: cache-tests-restore + uses: actions/cache/restore@v3 + with: + path: | + .pytest_cache/v/cache/lastfailed + key: ${{ runner.os }}-pytest-restore-failed-${{ github.ref }}-${{ github.sha }}-v1 + + - name: Run integration tests + env: + USER_NAME: ${{ env.USERNAME }} + PASSWORD: ${{ env.PASSWORD }} + DATABASE_NAME: ${{ steps.setup.outputs.database_name }} + ENGINE_NAME: ${{ steps.setup.outputs.engine_name }} + API_ENDPOINT: "api.${{ inputs.environment }}.firebolt.io" + ACCOUNT_NAME: "firebolt" + run: | + pytest --last-failed -o log_cli=true -o log_cli_level=INFO tests/functional/ --alluredir=allure-results + + - name: Save failed tests + id: cache-tests-save + uses: actions/cache/save@v3 + if: failure() + with: + path: | + .pytest_cache/v/cache/lastfailed + key: ${{ steps.cache-tests-restore.outputs.cache-primary-key }} + + - name: Get Allure history + uses: actions/checkout@v2 + if: always() + continue-on-error: true + with: + ref: gh-pages + path: gh-pages + + - name: Allure Report + uses: firebolt-db/action-allure-report@v1 + if: always() + with: + github-key: ${{ secrets.GITHUB_TOKEN }} + test-type: integration-v1 diff --git a/.github/workflows/integration-tests-v2.yml b/.github/workflows/integration-tests-v2.yml new file mode 100644 index 000000000..626c1cd59 --- /dev/null +++ b/.github/workflows/integration-tests-v2.yml @@ -0,0 +1,110 @@ +name: Run integration tests V2 + +on: + workflow_dispatch: + inputs: + environment: + description: 'Environment to run the tests against' + type: choice + required: true + default: 'dev' + options: + - dev + - staging + workflow_call: + inputs: + environment: + default: 'staging' + required: false + type: string + secrets: + FIREBOLT_CLIENT_ID_DEV_NEW_IDN: + required: true + FIREBOLT_CLIENT_SECRET_DEV_NEW_IDN: + required: true + FIREBOLT_CLIENT_ID_STG_NEW_IDN: + required: true + FIREBOLT_CLIENT_SECRET_STG_NEW_IDN: + required: true + +jobs: + tests: + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v2 + + - name: Set up Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: 3.7 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install ".[dev]" --no-cache-dir + + - name: Determine env variables + run: | + if [ "${{ inputs.environment }}" == 'staging' ]; then + echo "CLIENT_ID=${{ secrets.FIREBOLT_CLIENT_ID_STG_NEW_IDN }}" >> "$GITHUB_ENV" + echo "CLIENT_SECRET=${{ secrets.FIREBOLT_CLIENT_SECRET_STG_NEW_IDN }}" >> "$GITHUB_ENV" + else + echo "CLIENT_ID=${{ secrets.FIREBOLT_CLIENT_ID_DEV_NEW_IDN }}" >> "$GITHUB_ENV" + echo "CLIENT_SECRET=${{ secrets.FIREBOLT_CLIENT_SECRET_DEV_NEW_IDN }}" >> "$GITHUB_ENV" + fi + + - name: Keep environment name in the summary + run: echo '### Ran integration tests against ${{ inputs.environment }} ' >> $GITHUB_STEP_SUMMARY + + - name: Setup database and engine + id: setup + uses: firebolt-db/integration-testing-setup@v2 + with: + firebolt-client-id: ${{ env.CLIENT_ID }} + firebolt-client-secret: ${{ env.CLIENT_SECRET }} + account: "automation" + api-endpoint: "api.${{ inputs.environment }}.firebolt.io" + + - name: Restore cached failed tests + id: cache-tests-restore + uses: actions/cache/restore@v3 + with: + path: | + .pytest_cache/v/cache/lastfailed + key: ${{ runner.os }}-pytest-restore-failed-${{ github.ref }}-${{ github.sha }}-v2 + + - name: Run integration tests + env: + CLIENT_ID: ${{ env.CLIENT_ID }} + CLIENT_SECRET: ${{ env.CLIENT_SECRET }} + DATABASE_NAME: ${{ steps.setup.outputs.database_name }} + ENGINE_NAME: ${{ steps.setup.outputs.engine_name }} + API_ENDPOINT: "api.${{ inputs.environment }}.firebolt.io" + ACCOUNT_NAME: "automation" + run: | + pytest --last-failed -o log_cli=true -o log_cli_level=INFO tests/functional/ --alluredir=allure-results + + - name: Save failed tests + id: cache-tests-save + uses: actions/cache/save@v3 + if: failure() + with: + path: | + .pytest_cache/v/cache/lastfailed + key: ${{ steps.cache-tests-restore.outputs.cache-primary-key }} + + - name: Get Allure history + uses: actions/checkout@v2 + if: always() + continue-on-error: true + with: + ref: gh-pages + path: gh-pages + + - name: Allure Report + uses: firebolt-db/action-allure-report@v1 + if: always() + with: + github-key: ${{ secrets.GITHUB_TOKEN }} + test-type: integration-v2 diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 5b7ab89cd..0f0ed1640 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -19,92 +19,30 @@ on: type: string secrets: FIREBOLT_USERNAME_STAGING: - required: false + required: true FIREBOLT_PASSWORD_STAGING: - required: false + required: true FIREBOLT_USERNAME_DEV: - required: false + required: true FIREBOLT_PASSWORD_DEV: - required: false + required: true + FIREBOLT_CLIENT_ID_DEV_NEW_IDN: + required: true + FIREBOLT_CLIENT_SECRET_DEV_NEW_IDN: + required: true + FIREBOLT_CLIENT_ID_STG_NEW_IDN: + required: true + FIREBOLT_CLIENT_SECRET_STG_NEW_IDN: + required: true jobs: - tests: - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@v2 - - - name: Set up Python 3.7 - uses: actions/setup-python@v2 - with: - python-version: 3.7 - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install ".[dev]" --no-cache-dir - - - name: Determine env variables - run: | - if [ "${{ inputs.environment }}" == 'staging' ]; then - echo "USERNAME=${{ secrets.FIREBOLT_USERNAME_STAGING }}" >> "$GITHUB_ENV" - echo "PASSWORD=${{ secrets.FIREBOLT_PASSWORD_STAGING }}" >> "$GITHUB_ENV" - else - echo "USERNAME=${{ secrets.FIREBOLT_USERNAME_DEV }}" >> "$GITHUB_ENV" - echo "PASSWORD=${{ secrets.FIREBOLT_PASSWORD_DEV }}" >> "$GITHUB_ENV" - fi - - - name: Keep environment name in the summary - run: echo '### Ran integration tests against ${{ inputs.environment }} ' >> $GITHUB_STEP_SUMMARY - - - name: Setup database and engine - id: setup - uses: firebolt-db/integration-testing-setup@v1 - with: - firebolt-username: ${{ env.USERNAME }} - firebolt-password: ${{ env.PASSWORD }} - api-endpoint: "api.${{ inputs.environment }}.firebolt.io" - region: "us-east-1" - - - name: Restore cached failed tests - id: cache-tests-restore - uses: actions/cache/restore@v3 - with: - path: | - .pytest_cache/v/cache/lastfailed - key: ${{ runner.os }}-pytest-restore-failed-${{ github.ref }}-${{ github.sha }} - - - name: Run integration tests - env: - USER_NAME: ${{ env.USERNAME }} - PASSWORD: ${{ env.PASSWORD }} - DATABASE_NAME: ${{ steps.setup.outputs.database_name }} - ENGINE_NAME: ${{ steps.setup.outputs.engine_name }} - API_ENDPOINT: "api.${{ inputs.environment }}.firebolt.io" - ACCOUNT_NAME: "firebolt" - run: | - pytest --last-failed -o log_cli=true -o log_cli_level=INFO tests/functional/ --alluredir=allure-results - - - name: Save failed tests - id: cache-tests-save - uses: actions/cache/save@v3 - if: failure() - with: - path: | - .pytest_cache/v/cache/lastfailed - key: ${{ steps.cache-tests-restore.outputs.cache-primary-key }} - - - name: Get Allure history - uses: actions/checkout@v2 - if: always() - continue-on-error: true - with: - ref: gh-pages - path: gh-pages - - - name: Allure Report - uses: firebolt-db/action-allure-report@v1 - if: always() - with: - github-key: ${{ secrets.GITHUB_TOKEN }} - test-type: integration + integration-test-v1: + uses: ./.github/workflows/integration-tests-v1.yml + with: + environment: ${{ inputs.environment }} + secrets: inherit + integration-test-v2: + uses: ./.github/workflows/integration-tests-v2.yml + with: + environment: ${{ inputs.environment }} + secrets: inherit diff --git a/.github/workflows/jaffle-shop-v1.yml b/.github/workflows/jaffle-shop-v1.yml new file mode 100644 index 000000000..8a0ba5e47 --- /dev/null +++ b/.github/workflows/jaffle-shop-v1.yml @@ -0,0 +1,82 @@ +name: Run Jaffle shop tests V1 +on: + workflow_dispatch: + inputs: + environment: + description: 'Environment to run the tests against' + type: choice + required: true + default: 'dev' + options: + - dev + - staging + workflow_call: + inputs: + environment: + default: 'staging' + required: false + type: string + secrets: + FIREBOLT_USERNAME: + required: true + FIREBOLT_PASSWORD: + required: true +jobs: + tests: + runs-on: ubuntu-latest + steps: + - name: Check out dbt-adapter code + uses: actions/checkout@v2 + with: + path: dbt-firebolt + + - name: Check out Jaffle Shop code + uses: actions/checkout@v2 + with: + repository: firebolt-db/jaffle_shop_firebolt + path: jaffle-shop + + - name: Set up Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: 3.7 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install "dbt-firebolt/.[dev]" + + - name: Determine env variables + run: | + if [ "${{ inputs.environment }}" == 'staging' ]; then + echo "USERNAME=${{ secrets.FIREBOLT_USERNAME_STAGING }}" >> "$GITHUB_ENV" + echo "PASSWORD=${{ secrets.FIREBOLT_PASSWORD_STAGING }}" >> "$GITHUB_ENV" + else + echo "USERNAME=${{ secrets.FIREBOLT_USERNAME_DEV }}" >> "$GITHUB_ENV" + echo "PASSWORD=${{ secrets.FIREBOLT_PASSWORD_DEV }}" >> "$GITHUB_ENV" + fi + + - name: Keep environment name in the summary + run: echo '### Ran integration tests against ${{ inputs.environment }} ' >> $GITHUB_STEP_SUMMARY + + - name: Setup database and engine + id: setup + uses: firebolt-db/integration-testing-setup@v1 + with: + firebolt-username: ${{ env.USERNAME }} + firebolt-password: ${{ env.PASSWORD }} + api-endpoint: "api.${{ inputs.environment }}.firebolt.io" + region: "us-east-1" + + - name: Run Jaffle Shop test workflow + env: + USER_NAME: ${{ env.USERNAME }} + PASSWORD: ${{ env.PASSWORD }} + DATABASE_NAME: ${{ steps.setup.outputs.database_name }} + ENGINE_NAME: ${{ steps.setup.outputs.engine_name }} + API_ENDPOINT: "api.${{ inputs.environment }}.firebolt.io" + ACCOUNT_NAME: "firebolt" + DBT_PROFILES_DIR: "../dbt-firebolt/.github/workflows/jaffle_shop" + working-directory: jaffle-shop + run: + ../dbt-firebolt/.github/workflows/jaffle_shop/run_test_workflow.sh diff --git a/.github/workflows/jaffle-shop-v2.yml b/.github/workflows/jaffle-shop-v2.yml new file mode 100644 index 000000000..c1982bd06 --- /dev/null +++ b/.github/workflows/jaffle-shop-v2.yml @@ -0,0 +1,89 @@ +name: Run Jaffle shop tests V2 +on: + workflow_dispatch: + inputs: + environment: + description: 'Environment to run the tests against' + type: choice + required: true + default: 'dev' + options: + - dev + - staging + workflow_call: + inputs: + environment: + default: 'staging' + required: false + type: string + secrets: + FIREBOLT_CLIENT_ID_DEV_NEW_IDN: + required: true + FIREBOLT_CLIENT_SECRET_DEV_NEW_IDN: + required: true + FIREBOLT_CLIENT_ID_STG_NEW_IDN: + required: true + FIREBOLT_CLIENT_SECRET_STG_NEW_IDN: + required: true +jobs: + tests: + runs-on: ubuntu-latest + steps: + - name: Check out dbt-adapter code + uses: actions/checkout@v2 + with: + path: dbt-firebolt + + - name: Check out Jaffle Shop code + uses: actions/checkout@v2 + with: + repository: firebolt-db/jaffle_shop_firebolt + path: jaffle-shop + + - name: Set up Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: 3.7 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install "dbt-firebolt/.[dev]" + + - name: Determine env variables + run: | + if [ "${{ inputs.environment }}" == 'staging' ]; then + echo "KEY=${{ secrets.FIREBOLT_CLIENT_ID_STG_NEW_IDN }}" >> "$GITHUB_ENV" + echo "SECRET=${{ secrets.FIREBOLT_CLIENT_SECRET_STG_NEW_IDN }}" >> "$GITHUB_ENV" + else + echo "KEY=${{ secrets.FIREBOLT_CLIENT_ID_DEV_NEW_IDN }}" >> "$GITHUB_ENV" + echo "SECRET=${{ secrets.FIREBOLT_CLIENT_SECRET_DEV_NEW_IDN }}" >> "$GITHUB_ENV" + fi + + - name: Keep environment name in the summary + run: echo '### Ran integration tests against ${{ inputs.environment }} ' >> $GITHUB_STEP_SUMMARY + + - name: Setup database and engine + id: setup + uses: firebolt-db/integration-testing-setup@v2 + with: + firebolt-client-id: ${{ env.KEY }} + firebolt-client-secret: ${{ env.SECRET }} + account: "automation" + api-endpoint: "api.${{ inputs.environment }}.firebolt.io" + + + - name: Run Jaffle Shop test workflow + env: + USER_NAME: ${{ env.KEY }} + PASSWORD: ${{ env.SECRET }} + DATABASE_NAME: ${{ steps.setup.outputs.database_name }} + ENGINE_NAME: ${{ steps.setup.outputs.engine_name }} + API_ENDPOINT: "api.${{ inputs.environment }}.firebolt.io" + ACCOUNT_NAME: "automation" + DBT_PROFILES_DIR: "../dbt-firebolt/.github/workflows/jaffle_shop" + working-directory: jaffle-shop + run: + ../dbt-firebolt/.github/workflows/jaffle_shop/run_test_workflow.sh + + diff --git a/.github/workflows/jaffle_shop.yml b/.github/workflows/jaffle_shop.yml index 8af5eaeba..6c7c65dd8 100644 --- a/.github/workflows/jaffle_shop.yml +++ b/.github/workflows/jaffle_shop.yml @@ -17,68 +17,30 @@ on: required: false type: string secrets: - FIREBOLT_USERNAME: + FIREBOLT_USERNAME_STAGING: required: true - FIREBOLT_PASSWORD: + FIREBOLT_PASSWORD_STAGING: + required: true + FIREBOLT_USERNAME_DEV: + required: true + FIREBOLT_PASSWORD_DEV: + required: true + FIREBOLT_CLIENT_ID_DEV_NEW_IDN: + required: true + FIREBOLT_CLIENT_SECRET_DEV_NEW_IDN: + required: true + FIREBOLT_CLIENT_ID_STG_NEW_IDN: + required: true + FIREBOLT_CLIENT_SECRET_STG_NEW_IDN: required: true jobs: - tests: - runs-on: ubuntu-latest - steps: - - name: Check out dbt-adapter code - uses: actions/checkout@v2 - with: - path: dbt-firebolt - - - name: Check out Jaffle Shop code - uses: actions/checkout@v2 - with: - repository: firebolt-db/jaffle_shop_firebolt - path: jaffle-shop - - - name: Set up Python 3.7 - uses: actions/setup-python@v2 - with: - python-version: 3.7 - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install "dbt-firebolt/.[dev]" - - - name: Determine env variables - run: | - if [ "${{ inputs.environment }}" == 'staging' ]; then - echo "USERNAME=${{ secrets.FIREBOLT_USERNAME_STAGING }}" >> "$GITHUB_ENV" - echo "PASSWORD=${{ secrets.FIREBOLT_PASSWORD_STAGING }}" >> "$GITHUB_ENV" - else - echo "USERNAME=${{ secrets.FIREBOLT_USERNAME_DEV }}" >> "$GITHUB_ENV" - echo "PASSWORD=${{ secrets.FIREBOLT_PASSWORD_DEV }}" >> "$GITHUB_ENV" - fi - - - name: Keep environment name in the summary - run: echo '### Ran integration tests against ${{ inputs.environment }} ' >> $GITHUB_STEP_SUMMARY - - - name: Setup database and engine - id: setup - uses: firebolt-db/integration-testing-setup@v1 - with: - firebolt-username: ${{ env.USERNAME }} - firebolt-password: ${{ env.PASSWORD }} - api-endpoint: "api.${{ inputs.environment }}.firebolt.io" - region: "us-east-1" - - - name: Run Jaffle Shop test workflow - env: - USER_NAME: ${{ env.USERNAME }} - PASSWORD: ${{ env.PASSWORD }} - DATABASE_NAME: ${{ steps.setup.outputs.database_name }} - ENGINE_NAME: ${{ steps.setup.outputs.engine_name }} - API_ENDPOINT: "api.${{ inputs.environment }}.firebolt.io" - ACCOUNT_NAME: "firebolt" - DBT_PROFILES_DIR: "../dbt-firebolt/.github/workflows/jaffle_shop" - working-directory: jaffle-shop - run: - ../dbt-firebolt/.github/workflows/jaffle_shop/run_test_workflow.sh - - + jaffle-shop-v1: + uses: ./.github/workflows/jaffle-shop-v1.yml + with: + environment: ${{ github.event.inputs.environment }} + secrets: inherit + jaffle-shop-v2: + uses: ./.github/workflows/jaffle-shop-v2.yml + with: + environment: ${{ github.event.inputs.environment }} + secrets: inherit diff --git a/dbt/adapters/firebolt/connections.py b/dbt/adapters/firebolt/connections.py index 8718e2172..9b5aaeb04 100644 --- a/dbt/adapters/firebolt/connections.py +++ b/dbt/adapters/firebolt/connections.py @@ -14,7 +14,7 @@ from dbt.events import AdapterLogger # type: ignore from dbt.exceptions import DbtRuntimeError from firebolt.client import DEFAULT_API_URL -from firebolt.client.auth import UsernamePassword +from firebolt.client.auth import Auth, ClientCredentials, UsernamePassword from firebolt.db import connect as sdk_connect from firebolt.db.connection import Connection as SDKConnection from firebolt.db.cursor import Cursor @@ -27,14 +27,33 @@ @dataclass class FireboltCredentials(Credentials): # These values all come from either profiles.yml or dbt_project.yml. - user: str - password: str + user: Optional[str] = None + password: Optional[str] = None + # New way to authenticate + client_id: Optional[str] = None + client_secret: Optional[str] = None api_endpoint: Optional[str] = DEFAULT_API_URL driver: str = 'com.firebolt.FireboltDriver' engine_name: Optional[str] = None account_name: Optional[str] = None retries: int = 1 + def __post_init__(self) -> None: + # If user and password are not provided, assume client_id and client_secret + # are provided instead + if not self.user and not self.password: + if not self.client_id or not self.client_secret: + raise dbt.exceptions.DbtProfileError( + 'Either user and password or client_id and client_secret' + ' must be provided' + ) + else: + if self.client_id or self.client_secret: + raise dbt.exceptions.DbtProfileError( + 'Either user and password or client_id and client_secret' + ' must be provided' + ) + @property def type(self) -> str: return 'firebolt' @@ -94,10 +113,11 @@ def open(cls, connection: Connection) -> Connection: if connection.state == 'open': return connection credentials = connection.credentials + auth: Auth = _determine_auth(credentials) def connect() -> SDKConnection: handle = sdk_connect( - auth=UsernamePassword(credentials.user, credentials.password), + auth=auth, engine_name=credentials.engine_name, database=credentials.database, api_endpoint=credentials.api_endpoint, @@ -162,3 +182,20 @@ def cancel(self, connection: Connection) -> None: raise dbt.exceptions.NotImplementedError( '`cancel` is not implemented for this adapter!' ) + + +def _determine_auth(credentials: FireboltCredentials) -> Auth: + if credentials.client_id and credentials.client_secret: + return ClientCredentials(credentials.client_id, credentials.client_secret) + elif '@' in credentials.user: # type: ignore # checked in the dataclass + # email auth can only be used with UsernamePassword + return UsernamePassword( + credentials.user, # type: ignore[arg-type] + credentials.password, # type: ignore[arg-type] + ) + else: + # assume user provided id and secret in the user/password fields + return ClientCredentials( + credentials.user, # type: ignore[arg-type] + credentials.password, # type: ignore[arg-type] + ) diff --git a/dbt/include/firebolt/macros/materializations/models/incremental/merge.sql b/dbt/include/firebolt/macros/materializations/models/incremental/merge.sql index 6e76ab208..51b46d7c5 100644 --- a/dbt/include/firebolt/macros/materializations/models/incremental/merge.sql +++ b/dbt/include/firebolt/macros/materializations/models/incremental/merge.sql @@ -5,17 +5,16 @@ {% if unique_key %} {% if unique_key is sequence and unique_key is not string %} delete from {{ target }} - where ( - {% for key in unique_key %} - {{ target }}.{{ key }} in (select {{ key }} from {{ source }}) - {{ "and " if not loop.last }} - {% endfor %} + where + ({{ get_quoted_csv(unique_key) }}) in ( + select {{ get_quoted_csv(unique_key) }} + from {{ source }} + ) {% if incremental_predicates %} {% for predicate in incremental_predicates %} and {{ predicate }} {% endfor %} - {% endif %} - ); + {% endif %}; {% else %} delete from {{ target }} where ( diff --git a/dbt/include/firebolt/macros/utils/cast_bool_to_text.sql b/dbt/include/firebolt/macros/utils/cast_bool_to_text.sql index 296ddf1de..28b02f31d 100644 --- a/dbt/include/firebolt/macros/utils/cast_bool_to_text.sql +++ b/dbt/include/firebolt/macros/utils/cast_bool_to_text.sql @@ -1,7 +1,7 @@ {% macro firebolt__cast_bool_to_text(field) %} CASE - WHEN {{ field }} = 0 THEN 'false' - WHEN {{ field }} = 1 THEN 'true' + WHEN {{ field }} = false THEN 'false' + WHEN {{ field }} = true THEN 'true' ELSE NULL END {% endmacro %} diff --git a/dbt/include/firebolt/macros/utils/datediff.sql b/dbt/include/firebolt/macros/utils/datediff.sql index 76f24ed03..c466ddf8c 100644 --- a/dbt/include/firebolt/macros/utils/datediff.sql +++ b/dbt/include/firebolt/macros/utils/datediff.sql @@ -1,6 +1,6 @@ {% macro firebolt__datediff(first_date, second_date, datepart) -%} - datediff( + date_diff( '{{ datepart }}', {{ first_date }} :: TIMESTAMP, {{ second_date }} :: TIMESTAMP diff --git a/setup.cfg b/setup.cfg index d90bd9d9f..5b712dc38 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,7 +24,7 @@ project_urls = packages = find_namespace: install_requires = dbt-core~=1.4 - firebolt-sdk>=0.10.0 + firebolt-sdk>=1.1.0 python_requires = >=3.7 include_package_data = True package_dir = diff --git a/tests/conftest.py b/tests/conftest.py index d765960c2..e6ff64fd0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,17 +11,25 @@ # dbt will supply a unique schema per test, so we do not specify 'schema' here @pytest.fixture(scope='class') def dbt_profile_target(): - return { + profile = { 'type': 'firebolt', 'threads': 2, 'api_endpoint': os.getenv('API_ENDPOINT'), 'account_name': os.getenv('ACCOUNT_NAME'), 'database': os.getenv('DATABASE_NAME'), 'engine_name': os.getenv('ENGINE_NAME'), - 'user': os.getenv('USER_NAME'), - 'password': os.getenv('PASSWORD'), 'port': 443, } + # add credentials to the profile keys + if os.getenv('USER_NAME') and os.getenv('PASSWORD'): + profile['user'] = os.getenv('USER_NAME') + profile['password'] = os.getenv('PASSWORD') + elif os.getenv('CLIENT_ID') and os.getenv('CLIENT_SECRET'): + profile['client_id'] = os.getenv('CLIENT_ID') + profile['client_secret'] = os.getenv('CLIENT_SECRET') + else: + raise Exception('No credentials found in environment') + return profile # Overriding dbt_profile_data in order to set the schema to public. @@ -45,3 +53,8 @@ def dbt_profile_data(dbt_profile_target, profiles_config_update): if profiles_config_update: profile.update(profiles_config_update) return profile + + +@pytest.fixture(scope='class') +def profile_user(dbt_profile_target): + return dbt_profile_target.get('user', dbt_profile_target.get('client_id')) diff --git a/tests/unit/test_firebolt_adapter.py b/tests/unit/test_firebolt_adapter.py index 8b7acaef5..680cedb95 100644 --- a/tests/unit/test_firebolt_adapter.py +++ b/tests/unit/test_firebolt_adapter.py @@ -1,6 +1,7 @@ from unittest.mock import MagicMock, patch from dbt.contracts.connection import Connection +from firebolt.client.auth import ClientCredentials, UsernamePassword from firebolt.db.connection import Connection as SDKConnection from firebolt.utils.exception import InterfaceError from pytest import fixture, mark @@ -11,6 +12,7 @@ FireboltCredentials, ) from dbt.adapters.firebolt.column import FireboltColumn +from dbt.adapters.firebolt.connections import _determine_auth @fixture @@ -111,3 +113,42 @@ def test_column_class_init(adapter): def test_column_string_type(adapter): Column = adapter.get_column_class() assert Column.string_type(111) == 'text' + + +def test_determine_auth_with_client_credentials(): + credentials = FireboltCredentials( + client_id='your_client_id', + client_secret='your_client_secret', + database='your_database', + schema='your_schema', + ) + auth = _determine_auth(credentials) + assert isinstance(auth, ClientCredentials) + assert auth.client_id == 'your_client_id' + assert auth.client_secret == 'your_client_secret' + + +def test_determine_auth_with_email_auth(): + credentials = FireboltCredentials( + user='your_email@example.com', + password='your_password', + database='your_database', + schema='your_schema', + ) + auth = _determine_auth(credentials) + assert isinstance(auth, UsernamePassword) + assert auth.username == 'your_email@example.com' + assert auth.password == 'your_password' + + +def test_determine_auth_with_id_and_secret(): + credentials = FireboltCredentials( + user='your_user_id', + password='your_user_secret', + database='your_database', + schema='your_schema', + ) + auth = _determine_auth(credentials) + assert isinstance(auth, ClientCredentials) + assert auth.client_id == 'your_user_id' + assert auth.client_secret == 'your_user_secret'