From cc69e6f53b697cdd0a9e286504f74fc3de2f436d Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 18 Mar 2024 20:30:17 -0400 Subject: [PATCH 1/6] add draft integration test demonstrating the issue --- .../adapter/test_anonymous_usage_stats.py | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 tests/functional/adapter/test_anonymous_usage_stats.py diff --git a/tests/functional/adapter/test_anonymous_usage_stats.py b/tests/functional/adapter/test_anonymous_usage_stats.py new file mode 100644 index 000000000..f9526d80a --- /dev/null +++ b/tests/functional/adapter/test_anonymous_usage_stats.py @@ -0,0 +1,47 @@ +from dbt.tests.util import run_dbt_and_capture +import pytest + + +ANONYMOUS_USAGE_MESSAGE = """ +sys._xoptions['snowflake_partner_attribution'].append("dbtLabs_dbtPython") +""".strip() + + +MY_PYTHON_MODEL = """ +import pandas + +def model(dbt, session): + dbt.config(materialized='table') + data = [[1,2]] * 10 + return pandas.DataFrame(data, columns=['test', 'test2']) +""" + + +class AnonymousUsageStatsBase: + @pytest.fixture(scope="class") + def models(self): + return {"my_python_model.py": MY_PYTHON_MODEL} + + +class TestAnonymousUsageStatsOn(AnonymousUsageStatsBase): + @pytest.fixture(scope="class") + def profiles_config_update(self, dbt_profile_target): + outputs = {"default": dbt_profile_target} + outputs["default"]["send_anonymous_usage_stats"] = True + return {"test": {"outputs": outputs, "target": "default"}} + + def test_stats_get_sent(self, project): + _, logs = run_dbt_and_capture(["run"]) + assert ANONYMOUS_USAGE_MESSAGE in logs + + +class TestAnonymousUsageStatsOff(AnonymousUsageStatsBase): + @pytest.fixture(scope="class") + def profiles_config_update(self, dbt_profile_target): + outputs = {"default": dbt_profile_target} + outputs["default"]["send_anonymous_usage_stats"] = False + return {"test": {"outputs": outputs, "target": "default"}} + + def test_stats_do_not_get_sent(self, project): + _, logs = run_dbt_and_capture(["run"]) + assert ANONYMOUS_USAGE_MESSAGE not in logs From a32afdc92bd8075dea8d56ff70e9de7b5f9c00c9 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 18 Mar 2024 20:36:44 -0400 Subject: [PATCH 2/6] changie --- .changes/unreleased/Fixes-20240318-203635.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Fixes-20240318-203635.yaml diff --git a/.changes/unreleased/Fixes-20240318-203635.yaml b/.changes/unreleased/Fixes-20240318-203635.yaml new file mode 100644 index 000000000..d2c1cd05f --- /dev/null +++ b/.changes/unreleased/Fixes-20240318-203635.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Acknowledge `send_anonymous_usage_stats` setting for python models +time: 2024-03-18T20:36:35.396323-04:00 +custom: + Author: mikealfare + Issue: "830" From 65fafb26631c0df9e828911db8fca4af5ea3f5b7 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 18 Mar 2024 21:08:06 -0400 Subject: [PATCH 3/6] add debug to run args for log capture --- tests/functional/adapter/test_anonymous_usage_stats.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/functional/adapter/test_anonymous_usage_stats.py b/tests/functional/adapter/test_anonymous_usage_stats.py index f9526d80a..f7502f74b 100644 --- a/tests/functional/adapter/test_anonymous_usage_stats.py +++ b/tests/functional/adapter/test_anonymous_usage_stats.py @@ -31,7 +31,7 @@ def profiles_config_update(self, dbt_profile_target): return {"test": {"outputs": outputs, "target": "default"}} def test_stats_get_sent(self, project): - _, logs = run_dbt_and_capture(["run"]) + _, logs = run_dbt_and_capture(["--debug", "run"]) assert ANONYMOUS_USAGE_MESSAGE in logs @@ -43,5 +43,5 @@ def profiles_config_update(self, dbt_profile_target): return {"test": {"outputs": outputs, "target": "default"}} def test_stats_do_not_get_sent(self, project): - _, logs = run_dbt_and_capture(["run"]) + _, logs = run_dbt_and_capture(["--debug", "run"]) assert ANONYMOUS_USAGE_MESSAGE not in logs From 3c97e87f935bdbf94e52ea5cc8703486d190113e Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 18 Mar 2024 21:20:08 -0400 Subject: [PATCH 4/6] pin ddtrace to 1.x --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index db49d0497..bd580b6d1 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -10,7 +10,7 @@ black~=23.12 bumpversion~=0.6.0 click~=8.1 cryptography~=41.0.7 -ddtrace~=2.3 +ddtrace>=1.0,<2.0 flake8~=6.1 flaky~=3.7 freezegun~=1.3 From d769d229c85b696dfac0150e04589d8ba4f057ac Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 18 Mar 2024 22:39:03 -0400 Subject: [PATCH 5/6] condition on the send_anonymous_usage_stats setting when building a python model --- dbt/adapters/snowflake/impl.py | 12 ++++++++---- .../functional/adapter/test_anonymous_usage_stats.py | 12 ++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index 4dc57a1b4..5656d7772 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -2,7 +2,6 @@ from typing import Mapping, Any, Optional, List, Union, Dict, FrozenSet, Tuple import agate - from dbt.adapters.base.impl import AdapterConfig, ConstraintSupport # type: ignore from dbt.adapters.base.meta import available from dbt.adapters.capability import CapabilityDict, CapabilitySupport, Support, Capability @@ -198,6 +197,7 @@ def timestamp_add_sql(self, add_to: str, number: int = 1, interval: str = "hour" return f"DATEADD({interval}, {number}, {add_to})" def submit_python_job(self, parsed_model: dict, compiled_code: str): + super().submit_python_job schema = parsed_model["schema"] database = parsed_model["database"] identifier = parsed_model["alias"] @@ -217,10 +217,14 @@ def submit_python_job(self, parsed_model: dict, compiled_code: str): if imports: imports = f"IMPORTS = ('{imports}')" - snowpark_telemetry_string = "dbtLabs_dbtPython" - snowpark_telemetry_snippet = f""" + if self.config.args.SEND_ANONYMOUS_USAGE_STATS: + snowpark_telemetry_string = "dbtLabs_dbtPython" + snowpark_telemetry_snippet = f""" import sys -sys._xoptions['snowflake_partner_attribution'].append("{snowpark_telemetry_string}")""" +sys._xoptions['snowflake_partner_attribution'].append("{snowpark_telemetry_string}") +""" + else: + snowpark_telemetry_snippet = "" common_procedure_code = f""" RETURNS STRING diff --git a/tests/functional/adapter/test_anonymous_usage_stats.py b/tests/functional/adapter/test_anonymous_usage_stats.py index f7502f74b..bc5a4d334 100644 --- a/tests/functional/adapter/test_anonymous_usage_stats.py +++ b/tests/functional/adapter/test_anonymous_usage_stats.py @@ -25,10 +25,8 @@ def models(self): class TestAnonymousUsageStatsOn(AnonymousUsageStatsBase): @pytest.fixture(scope="class") - def profiles_config_update(self, dbt_profile_target): - outputs = {"default": dbt_profile_target} - outputs["default"]["send_anonymous_usage_stats"] = True - return {"test": {"outputs": outputs, "target": "default"}} + def project_config_update(self): + return {"flags": {"send_anonymous_usage_stats": True}} def test_stats_get_sent(self, project): _, logs = run_dbt_and_capture(["--debug", "run"]) @@ -37,10 +35,8 @@ def test_stats_get_sent(self, project): class TestAnonymousUsageStatsOff(AnonymousUsageStatsBase): @pytest.fixture(scope="class") - def profiles_config_update(self, dbt_profile_target): - outputs = {"default": dbt_profile_target} - outputs["default"]["send_anonymous_usage_stats"] = False - return {"test": {"outputs": outputs, "target": "default"}} + def project_config_update(self, dbt_profile_target): + return {"flags": {"send_anonymous_usage_stats": False}} def test_stats_do_not_get_sent(self, project): _, logs = run_dbt_and_capture(["--debug", "run"]) From a273558c6a97df71e1a79eb0ddc0a54ad9bae8ff Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 18 Mar 2024 22:41:16 -0400 Subject: [PATCH 6/6] revert troubleshooting artifacts --- dbt/adapters/snowflake/impl.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index 5656d7772..3583c888c 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -2,6 +2,7 @@ from typing import Mapping, Any, Optional, List, Union, Dict, FrozenSet, Tuple import agate + from dbt.adapters.base.impl import AdapterConfig, ConstraintSupport # type: ignore from dbt.adapters.base.meta import available from dbt.adapters.capability import CapabilityDict, CapabilitySupport, Support, Capability @@ -197,7 +198,6 @@ def timestamp_add_sql(self, add_to: str, number: int = 1, interval: str = "hour" return f"DATEADD({interval}, {number}, {add_to})" def submit_python_job(self, parsed_model: dict, compiled_code: str): - super().submit_python_job schema = parsed_model["schema"] database = parsed_model["database"] identifier = parsed_model["alias"] @@ -221,8 +221,7 @@ def submit_python_job(self, parsed_model: dict, compiled_code: str): snowpark_telemetry_string = "dbtLabs_dbtPython" snowpark_telemetry_snippet = f""" import sys -sys._xoptions['snowflake_partner_attribution'].append("{snowpark_telemetry_string}") -""" +sys._xoptions['snowflake_partner_attribution'].append("{snowpark_telemetry_string}")""" else: snowpark_telemetry_snippet = ""