From 9385b49c15373b4843943780c1f20b7d587ca360 Mon Sep 17 00:00:00 2001 From: Wen Date: Mon, 25 Sep 2023 14:10:09 -0400 Subject: [PATCH 1/7] Avoid nested loop in query planner execution (#612) * Adjust not equals application order - Move relation name not equals from outer SELECT `where` clause to dependency CTE - Seems to more reliably push query planner to avoid a nested loop - Should preserve original intent, i.e. to exclude self-references from final result - Execution plan suggests this reduces the row estimate for the outer SELECT - Real-world testing on internal company cluster under conditions where nested loop is used for original, shows that the new approach continues to perform and behave as expected (no nested loop) * Add changie entry --- .changes/unreleased/Fixes-20230923-091155.yaml | 6 ++++++ dbt/include/redshift/macros/relations.sql | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Fixes-20230923-091155.yaml diff --git a/.changes/unreleased/Fixes-20230923-091155.yaml b/.changes/unreleased/Fixes-20230923-091155.yaml new file mode 100644 index 000000000..055d4dc7e --- /dev/null +++ b/.changes/unreleased/Fixes-20230923-091155.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: avoid nested loop in query planner execution of redshift__get_relations +time: 2023-09-23T09:11:55.2152725-04:00 +custom: + Author: slin30 + Issue: "609" diff --git a/dbt/include/redshift/macros/relations.sql b/dbt/include/redshift/macros/relations.sql index 28c6bc377..6d83c36b9 100644 --- a/dbt/include/redshift/macros/relations.sql +++ b/dbt/include/redshift/macros/relations.sql @@ -24,6 +24,7 @@ with from pg_depend left join pg_rewrite on pg_depend.objid = pg_rewrite.oid + where coalesce(pg_rewrite.ev_class, pg_depend.objid) != pg_depend.refobjid ) select distinct @@ -36,7 +37,6 @@ join relation ref on dependency.ref_relation_id = ref.relation_id join relation dep on dependency.dep_relation_id = dep.relation_id -where ref.relation_name != dep.relation_name {%- endcall -%} From 7f7330969e2cd99d4128a55091f4b6bce6cffeaf Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 26 Sep 2023 11:49:22 -0500 Subject: [PATCH 2/7] automate repo cleanup (#618) --- .github/workflows/repository-cleanup.yml | 30 ++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/repository-cleanup.yml diff --git a/.github/workflows/repository-cleanup.yml b/.github/workflows/repository-cleanup.yml new file mode 100644 index 000000000..c1d780281 --- /dev/null +++ b/.github/workflows/repository-cleanup.yml @@ -0,0 +1,30 @@ +# **what?** +# Cleanup branches left over from automation and testing. Also cleanup +# draft releases from release testing. + +# **why?** +# The automations are leaving behind branches and releases that clutter +# the repository. Sometimes we need them to debug processes so we don't +# want them immediately deleted. Running on Saturday to avoid running +# at the same time as an actual release to prevent breaking a release +# mid-release. + +# **when?** +# Mainly on a schedule of 12:00 Saturday. +# Manual trigger can also run on demand + +name: Repository Cleanup + +on: + schedule: + - cron: '0 12 * * SAT' # At 12:00 on Saturday - details in `why` above + + workflow_dispatch: # for manual triggering + +permissions: + contents: write + +jobs: + cleanup-repo: + uses: dbt-labs/actions/.github/workflows/repository-cleanup.yml@main + secrets: inherit From 16d86aa9c570479f4a5a59ee10a6d64a38fd530d Mon Sep 17 00:00:00 2001 From: Quigley Malcolm Date: Tue, 26 Sep 2023 16:44:09 -0700 Subject: [PATCH 3/7] Add tests for `date_spine` macro, and sub macros (#617) Co-authored-by: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> --- tests/functional/adapter/utils/test_utils.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/functional/adapter/utils/test_utils.py b/tests/functional/adapter/utils/test_utils.py index 266103fbc..61a706f4d 100644 --- a/tests/functional/adapter/utils/test_utils.py +++ b/tests/functional/adapter/utils/test_utils.py @@ -8,9 +8,13 @@ from dbt.tests.adapter.utils.test_current_timestamp import BaseCurrentTimestampNaive from dbt.tests.adapter.utils.test_dateadd import BaseDateAdd from dbt.tests.adapter.utils.test_datediff import BaseDateDiff +from dbt.tests.adapter.utils.test_date_spine import BaseDateSpine from dbt.tests.adapter.utils.test_date_trunc import BaseDateTrunc from dbt.tests.adapter.utils.test_escape_single_quotes import BaseEscapeSingleQuotesQuote from dbt.tests.adapter.utils.test_except import BaseExcept +from dbt.tests.adapter.utils.test_generate_series import BaseGenerateSeries +from dbt.tests.adapter.utils.test_get_intervals_between import BaseGetIntervalsBetween +from dbt.tests.adapter.utils.test_get_powers_of_two import BaseGetPowersOfTwo from dbt.tests.adapter.utils.test_hash import BaseHash from dbt.tests.adapter.utils.test_intersect import BaseIntersect from dbt.tests.adapter.utils.test_last_day import BaseLastDay @@ -65,6 +69,10 @@ class TestDateDiff(BaseDateDiff): pass +class TestDateSpine(BaseDateSpine): + pass + + class TestDateTrunc(BaseDateTrunc): pass @@ -77,6 +85,18 @@ class TestExcept(BaseExcept): pass +class TestGenerateSeries(BaseGenerateSeries): + pass + + +class TestGetIntervalsBeteween(BaseGetIntervalsBetween): + pass + + +class TestGetPowersOfTwo(BaseGetPowersOfTwo): + pass + + class TestHash(BaseHash): pass From ca3c7b4f9e6b2df0ae6d2334edd58b6bb08a578e Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Thu, 28 Sep 2023 19:39:11 +0100 Subject: [PATCH 4/7] add dbt show tests (#611) * add dbt show tests * changelog entry * repoint to core main * change to a WITH statement in model definition for dbt_show test model * reuse core fixture for dbt show sql header test * undo dev-requirements changes --------- Co-authored-by: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> Co-authored-by: Matthew McKnight <91097623+McKnight-42@users.noreply.github.com> Co-authored-by: Matthew McKnight --- .changes/unreleased/Under the Hood-20230925-150132.yaml | 6 ++++++ tests/functional/adapter/dbt_show/test_dbt_show.py | 9 +++++++++ 2 files changed, 15 insertions(+) create mode 100644 .changes/unreleased/Under the Hood-20230925-150132.yaml create mode 100644 tests/functional/adapter/dbt_show/test_dbt_show.py diff --git a/.changes/unreleased/Under the Hood-20230925-150132.yaml b/.changes/unreleased/Under the Hood-20230925-150132.yaml new file mode 100644 index 000000000..c61a28d56 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20230925-150132.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Add tests for inlined limit + sql_header in dbt show query +time: 2023-09-25T15:01:32.025325+01:00 +custom: + Author: michelleark + Issue: "616" diff --git a/tests/functional/adapter/dbt_show/test_dbt_show.py b/tests/functional/adapter/dbt_show/test_dbt_show.py new file mode 100644 index 000000000..808a7733c --- /dev/null +++ b/tests/functional/adapter/dbt_show/test_dbt_show.py @@ -0,0 +1,9 @@ +from dbt.tests.adapter.dbt_show.test_dbt_show import BaseShowSqlHeader, BaseShowLimit + + +class TestRedshiftShowLimit(BaseShowLimit): + pass + + +class TestRedshiftShowSqlHeader(BaseShowSqlHeader): + pass From 85d37209921042e6af0e18bc11dd6017ba331622 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Oct 2023 11:38:39 -0500 Subject: [PATCH 5/7] Update ddtrace requirement from ~=1.18 to ~=1.19 (#610) * Update ddtrace requirement from ~=1.18 to ~=1.19 Updates the requirements on [ddtrace](https://github.com/DataDog/dd-trace-py) to permit the latest version. - [Release notes](https://github.com/DataDog/dd-trace-py/releases) - [Changelog](https://github.com/DataDog/dd-trace-py/blob/2.x/CHANGELOG.md) - [Commits](https://github.com/DataDog/dd-trace-py/compare/v1.18.0...v1.19.0) --- updated-dependencies: - dependency-name: ddtrace dependency-type: direct:development ... Signed-off-by: dependabot[bot] * Add automated changelog yaml from template for bot PR --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Github Build Bot Co-authored-by: Matthew McKnight <91097623+McKnight-42@users.noreply.github.com> --- .changes/unreleased/Dependencies-20230918-190833.yaml | 6 ++++++ dev-requirements.txt | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Dependencies-20230918-190833.yaml diff --git a/.changes/unreleased/Dependencies-20230918-190833.yaml b/.changes/unreleased/Dependencies-20230918-190833.yaml new file mode 100644 index 000000000..f784102ca --- /dev/null +++ b/.changes/unreleased/Dependencies-20230918-190833.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Update ddtrace requirement from ~=1.18 to ~=1.19" +time: 2023-09-18T19:08:33.00000Z +custom: + Author: dependabot[bot] + PR: 610 diff --git a/dev-requirements.txt b/dev-requirements.txt index 96f252ebe..4b3b09cda 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -9,7 +9,7 @@ git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-postgres&subdirectory=plugi black~=23.9 bumpversion~=0.6.0 click~=8.1 -ddtrace~=1.18 +ddtrace~=1.19 flake8~=6.1 flaky~=3.7 freezegun~=1.2 From 1116e4706914653d01e17c64a9e477f6cd0a5299 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Tue, 10 Oct 2023 19:05:42 -0400 Subject: [PATCH 6/7] ADAP-891: Support test results as views (#614) * implement store-failures-as tests --- .../unreleased/Features-20230921-153707.yaml | 6 +++++ .../adapter/test_store_test_failures.py | 27 ++++++++++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Features-20230921-153707.yaml diff --git a/.changes/unreleased/Features-20230921-153707.yaml b/.changes/unreleased/Features-20230921-153707.yaml new file mode 100644 index 000000000..cf2c60ad5 --- /dev/null +++ b/.changes/unreleased/Features-20230921-153707.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Support storing test failures as views +time: 2023-09-21T15:37:07.970722-04:00 +custom: + Author: mikealfare + Issue: "6914" diff --git a/tests/functional/adapter/test_store_test_failures.py b/tests/functional/adapter/test_store_test_failures.py index 5d6b70fbb..7f591654e 100644 --- a/tests/functional/adapter/test_store_test_failures.py +++ b/tests/functional/adapter/test_store_test_failures.py @@ -1,7 +1,32 @@ +from dbt.tests.adapter.store_test_failures_tests import basic from dbt.tests.adapter.store_test_failures_tests.test_store_test_failures import ( TestStoreTestFailures, ) -class RedshiftTestStoreTestFailures(TestStoreTestFailures): +class TestRedshiftTestStoreTestFailures(TestStoreTestFailures): + pass + + +class TestStoreTestFailuresAsInteractions(basic.StoreTestFailuresAsInteractions): + pass + + +class TestStoreTestFailuresAsProjectLevelOff(basic.StoreTestFailuresAsProjectLevelOff): + pass + + +class TestStoreTestFailuresAsProjectLevelView(basic.StoreTestFailuresAsProjectLevelView): + pass + + +class TestStoreTestFailuresAsGeneric(basic.StoreTestFailuresAsGeneric): + pass + + +class TestStoreTestFailuresAsProjectLevelEphemeral(basic.StoreTestFailuresAsProjectLevelEphemeral): + pass + + +class TestStoreTestFailuresAsExceptions(basic.StoreTestFailuresAsExceptions): pass From 5d4f3f5c94235d2a5cff50e6993f3c6e89d0eb16 Mon Sep 17 00:00:00 2001 From: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> Date: Tue, 10 Oct 2023 19:15:20 -0600 Subject: [PATCH 7/7] Use the PID to terminate the session (#568) * The first element of the result is the PID * Debug-level logging of high-level message + SQL * Using redshift_connector `cursor.fetchone()` returns `(,)` * Use cursor to call `select pg_terminate_backend({pid})` directly rather than using the `SQLConnectionManager` --------- Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> --- .changes/unreleased/Fixes-20230807-174409.yaml | 6 ++++++ dbt/adapters/redshift/connections.py | 10 ++++++---- tests/unit/test_redshift_adapter.py | 3 +-- 3 files changed, 13 insertions(+), 6 deletions(-) create mode 100644 .changes/unreleased/Fixes-20230807-174409.yaml diff --git a/.changes/unreleased/Fixes-20230807-174409.yaml b/.changes/unreleased/Fixes-20230807-174409.yaml new file mode 100644 index 000000000..ad4f11b42 --- /dev/null +++ b/.changes/unreleased/Fixes-20230807-174409.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Use the PID to terminate the session +time: 2023-08-07T17:44:09.15097-06:00 +custom: + Author: dbeatty10 + Issue: "553" diff --git a/dbt/adapters/redshift/connections.py b/dbt/adapters/redshift/connections.py index 83d05b587..0c9d1b7ed 100644 --- a/dbt/adapters/redshift/connections.py +++ b/dbt/adapters/redshift/connections.py @@ -240,8 +240,9 @@ class RedshiftConnectionManager(SQLConnectionManager): def _get_backend_pid(self): sql = "select pg_backend_pid()" _, cursor = self.add_query(sql) + res = cursor.fetchone() - return res + return res[0] def cancel(self, connection: Connection): try: @@ -253,9 +254,10 @@ def cancel(self, connection: Connection): raise sql = f"select pg_terminate_backend({pid})" - _, cursor = self.add_query(sql) - res = cursor.fetchone() - logger.debug(f"Cancel query '{connection.name}': {res}") + cursor = connection.handle.cursor() + logger.debug(f"Cancel query on: '{connection.name}' with PID: {pid}") + logger.debug(sql) + cursor.execute(sql) @classmethod def get_response(cls, cursor: redshift_connector.Cursor) -> AdapterResponse: diff --git a/tests/unit/test_redshift_adapter.py b/tests/unit/test_redshift_adapter.py index c31366a1e..1edea565e 100644 --- a/tests/unit/test_redshift_adapter.py +++ b/tests/unit/test_redshift_adapter.py @@ -471,14 +471,13 @@ def test_cancel_open_connections_single(self): with mock.patch.object(self.adapter.connections, "add_query") as add_query: query_result = mock.MagicMock() cursor = mock.Mock() - cursor.fetchone.return_value = 42 + cursor.fetchone.return_value = (42,) add_query.side_effect = [(None, cursor), (None, query_result)] self.assertEqual(len(list(self.adapter.cancel_open_connections())), 1) add_query.assert_has_calls( [ call("select pg_backend_pid()"), - call("select pg_terminate_backend(42)"), ] )