From 82b2acbf36099ea28a22032e98e95dd888e90bbd Mon Sep 17 00:00:00 2001 From: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Tue, 19 Sep 2023 12:56:52 -0700 Subject: [PATCH 1/4] allow testing against any core branch (#771) --- .github/scripts/update_dbt_core_branch.sh | 20 ++++++++++++++++++++ .github/workflows/integration.yml | 11 +++++++++++ 2 files changed, 31 insertions(+) create mode 100755 .github/scripts/update_dbt_core_branch.sh diff --git a/.github/scripts/update_dbt_core_branch.sh b/.github/scripts/update_dbt_core_branch.sh new file mode 100755 index 000000000..d28a40c35 --- /dev/null +++ b/.github/scripts/update_dbt_core_branch.sh @@ -0,0 +1,20 @@ +#!/bin/bash -e +set -e + +git_branch=$1 +target_req_file="dev-requirements.txt" +core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${git_branch}#egg=dbt-core|g" +postgres_req_sed_pattern="s|dbt-core.git.*#egg=dbt-postgres|dbt-core.git@${git_branch}#egg=dbt-postgres|g" +tests_req_sed_pattern="s|dbt-core.git.*#egg=dbt-tests|dbt-core.git@${git_branch}#egg=dbt-tests|g" +if [[ "$OSTYPE" == darwin* ]]; then + # mac ships with a different version of sed that requires a delimiter arg + sed -i "" "$core_req_sed_pattern" $target_req_file + sed -i "" "$postgres_req_sed_pattern" $target_req_file + sed -i "" "$tests_req_sed_pattern" $target_req_file +else + sed -i "$core_req_sed_pattern" $target_req_file + sed -i "$postgres_req_sed_pattern" $target_req_file + sed -i "$tests_req_sed_pattern" $target_req_file +fi +core_version=$(curl "https://raw.githubusercontent.com/dbt-labs/dbt-core/${git_branch}/core/dbt/version.py" | grep "__version__ = *"|cut -d'=' -f2) +bumpversion --allow-dirty --new-version "$core_version" major diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 866c20b07..698982c0f 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -33,6 +33,11 @@ on: pull_request_target: # manual trigger workflow_dispatch: + inputs: + dbt-core-branch: + description: "branch of dbt-core to use in dev-requirements.txt" + required: false + type: string # explicitly turn off permissions for `GITHUB_TOKEN` permissions: read-all @@ -159,6 +164,12 @@ jobs: python -m pip --version tox --version + - name: Update dev_requirements.txt + if: inputs.dbt-core-branch != '' + run: | + pip install bumpversion + ./.github/scripts/update_dbt_core_branch.sh ${{ inputs.dbt-core-branch }} + - name: Run tox (snowflake) if: matrix.adapter == 'snowflake' env: From a181a7d13c82192500bfcc8c365a846c0664d2c9 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 26 Sep 2023 11:49:26 -0500 Subject: [PATCH 2/4] automate repo cleanup (#789) --- .github/workflows/repository-cleanup.yml | 30 ++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/repository-cleanup.yml diff --git a/.github/workflows/repository-cleanup.yml b/.github/workflows/repository-cleanup.yml new file mode 100644 index 000000000..c1d780281 --- /dev/null +++ b/.github/workflows/repository-cleanup.yml @@ -0,0 +1,30 @@ +# **what?** +# Cleanup branches left over from automation and testing. Also cleanup +# draft releases from release testing. + +# **why?** +# The automations are leaving behind branches and releases that clutter +# the repository. Sometimes we need them to debug processes so we don't +# want them immediately deleted. Running on Saturday to avoid running +# at the same time as an actual release to prevent breaking a release +# mid-release. + +# **when?** +# Mainly on a schedule of 12:00 Saturday. +# Manual trigger can also run on demand + +name: Repository Cleanup + +on: + schedule: + - cron: '0 12 * * SAT' # At 12:00 on Saturday - details in `why` above + + workflow_dispatch: # for manual triggering + +permissions: + contents: write + +jobs: + cleanup-repo: + uses: dbt-labs/actions/.github/workflows/repository-cleanup.yml@main + secrets: inherit From cd789b0ff3f6f73037b073cf74c82452a7ad12be Mon Sep 17 00:00:00 2001 From: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Tue, 26 Sep 2023 13:15:01 -0700 Subject: [PATCH 3/4] Add debug logging (#768) * set snowflake connector log level on env variable * set snowflake connector log level on env variable * set snowflake connector log level on env variable * add boto packages * add logging about stting up logging * merge paw/type-fix * add changie * fix event import * add unit tests * add unit tests --- .../Under the Hood-20230925-120144.yaml | 6 +++++ dbt/adapters/snowflake/connections.py | 8 ++++++ tests/unit/test_connections.py | 25 +++++++++++++++++++ 3 files changed, 39 insertions(+) create mode 100644 .changes/unreleased/Under the Hood-20230925-120144.yaml create mode 100644 tests/unit/test_connections.py diff --git a/.changes/unreleased/Under the Hood-20230925-120144.yaml b/.changes/unreleased/Under the Hood-20230925-120144.yaml new file mode 100644 index 000000000..717f415f2 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20230925-120144.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: allow for adding snowflake-python-collector logs to dbt output +time: 2023-09-25T12:01:44.778426-07:00 +custom: + Author: colin-rogers-dbt + Issue: "768" diff --git a/dbt/adapters/snowflake/connections.py b/dbt/adapters/snowflake/connections.py index 4bb00f4d8..b5fa30002 100644 --- a/dbt/adapters/snowflake/connections.py +++ b/dbt/adapters/snowflake/connections.py @@ -1,5 +1,7 @@ import base64 import datetime +import os + import pytz import re from contextlib import contextmanager @@ -47,6 +49,12 @@ logger = AdapterLogger("Snowflake") + +if os.getenv("DBT_SNOWFLAKE_CONNECTOR_DEBUG_LOGGING"): + for logger_name in ["snowflake.connector", "botocore", "boto3"]: + logger.debug(f"Setting {logger_name} to DEBUG") + logger.set_adapter_dependency_log_level(logger_name, "DEBUG") + _TOKEN_REQUEST_URL = "https://{}.snowflakecomputing.com/oauth/token-request" ERROR_REDACTION_PATTERNS = { diff --git a/tests/unit/test_connections.py b/tests/unit/test_connections.py new file mode 100644 index 000000000..87b0cf4c2 --- /dev/null +++ b/tests/unit/test_connections.py @@ -0,0 +1,25 @@ +import os +from importlib import reload +from unittest.mock import Mock +import dbt.adapters.snowflake.connections as connections +import dbt.events + + +def test_connections_sets_logs_in_response_to_env_var(monkeypatch): + """Test that setting the DBT_SNOWFLAKE_CONNECTOR_DEBUG_LOGGING environment variable happens on import""" + log_mock = Mock() + monkeypatch.setattr(dbt.events, "AdapterLogger", Mock(return_value=log_mock)) + monkeypatch.setattr(os, "environ", {"DBT_SNOWFLAKE_CONNECTOR_DEBUG_LOGGING": "true"}) + reload(connections) + + assert log_mock.debug.call_count == 3 + assert log_mock.set_adapter_dependency_log_level.call_count == 3 + + +def test_connections_does_not_set_logs_in_response_to_env_var(monkeypatch): + log_mock = Mock() + monkeypatch.setattr(dbt.events, "AdapterLogger", Mock(return_value=log_mock)) + reload(connections) + + assert log_mock.debug.call_count == 0 + assert log_mock.set_adapter_dependency_log_level.call_count == 0 From 658e16ea01d1eb61a586dffd8134fd9d6a57e7cd Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Thu, 28 Sep 2023 19:54:29 +0100 Subject: [PATCH 4/4] add dbt show tests (#779) * add dbt show tests * changelog entry * repoint to core main * reuse core fixture for dbt show sql header test * point to @improve-show-fixture dbt-core branch * undo dev-requirements changes --------- Co-authored-by: Matthew McKnight <91097623+McKnight-42@users.noreply.github.com> Co-authored-by: Matthew McKnight --- .changes/unreleased/Under the Hood-20230925-144814.yaml | 6 ++++++ tests/functional/adapter/dbt_show/test_dbt_show.py | 9 +++++++++ 2 files changed, 15 insertions(+) create mode 100644 .changes/unreleased/Under the Hood-20230925-144814.yaml create mode 100644 tests/functional/adapter/dbt_show/test_dbt_show.py diff --git a/.changes/unreleased/Under the Hood-20230925-144814.yaml b/.changes/unreleased/Under the Hood-20230925-144814.yaml new file mode 100644 index 000000000..df7e17255 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20230925-144814.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Add tests for inlined limit + sql_header in dbt show query +time: 2023-09-25T14:48:14.663178+01:00 +custom: + Author: michelleark + Issue: "786" diff --git a/tests/functional/adapter/dbt_show/test_dbt_show.py b/tests/functional/adapter/dbt_show/test_dbt_show.py new file mode 100644 index 000000000..c60a26aec --- /dev/null +++ b/tests/functional/adapter/dbt_show/test_dbt_show.py @@ -0,0 +1,9 @@ +from dbt.tests.adapter.dbt_show.test_dbt_show import BaseShowSqlHeader, BaseShowLimit + + +class TestBigQueryShowLimit(BaseShowLimit): + pass + + +class TestBigQueryShowSqlHeader(BaseShowSqlHeader): + pass