From ae890a4031b589335e037d77cc6954a561cc3c42 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 23 Apr 2024 14:29:47 -0500 Subject: [PATCH 1/4] use the secret constant from dbt-common --- core/dbt/artifacts/schemas/run/v5/run.py | 2 +- core/dbt/config/renderer.py | 3 ++- core/dbt/constants.py | 2 -- core/dbt/context/base.py | 3 ++- core/dbt/context/configured.py | 3 ++- core/dbt/context/providers.py | 3 ++- core/dbt/context/secret.py | 3 ++- core/dbt/exceptions.py | 2 +- core/dbt/parser/manifest.py | 2 +- tests/functional/context_methods/test_env_vars.py | 3 ++- tests/functional/context_methods/test_secret_env_vars.py | 2 +- tests/functional/partial_parsing/test_pp_vars.py | 2 +- 12 files changed, 17 insertions(+), 13 deletions(-) diff --git a/core/dbt/artifacts/schemas/run/v5/run.py b/core/dbt/artifacts/schemas/run/v5/run.py index 47cc0cb3b87..d3e6bfa4ece 100644 --- a/core/dbt/artifacts/schemas/run/v5/run.py +++ b/core/dbt/artifacts/schemas/run/v5/run.py @@ -5,7 +5,7 @@ from datetime import datetime -from dbt.constants import SECRET_ENV_PREFIX +from dbt_common.constants import SECRET_ENV_PREFIX from dbt.artifacts.resources import CompiledResource from dbt.artifacts.schemas.base import ( BaseArtifactMetadata, diff --git a/core/dbt/config/renderer.py b/core/dbt/config/renderer.py index ad2c547be21..75e7f690251 100644 --- a/core/dbt/config/renderer.py +++ b/core/dbt/config/renderer.py @@ -4,7 +4,8 @@ from dbt.clients.jinja import get_rendered from dbt_common.clients.jinja import catch_jinja -from dbt.constants import SECRET_ENV_PREFIX, DEPENDENCIES_FILE_NAME, SECRET_PLACEHOLDER +from dbt_common.constants import SECRET_ENV_PREFIX +from dbt.constants import DEPENDENCIES_FILE_NAME, SECRET_PLACEHOLDER from dbt.context.target import TargetContext from dbt.context.secret import SecretContext from dbt.context.base import BaseContext diff --git a/core/dbt/constants.py b/core/dbt/constants.py index 9ffa7128a69..feaf0957746 100644 --- a/core/dbt/constants.py +++ b/core/dbt/constants.py @@ -1,5 +1,3 @@ -# TODO: remove SECRET_ENV_PREFIX and import from dbt_common -SECRET_ENV_PREFIX = "DBT_ENV_SECRET_" DEFAULT_ENV_PLACEHOLDER = "DBT_DEFAULT_PLACEHOLDER" SECRET_PLACEHOLDER = "$$$DBT_SECRET_START$$${}$$$DBT_SECRET_END$$$" diff --git a/core/dbt/context/base.py b/core/dbt/context/base.py index 1c7f4cf9db4..3bc9db78740 100644 --- a/core/dbt/context/base.py +++ b/core/dbt/context/base.py @@ -11,7 +11,8 @@ from dbt import utils from dbt.clients.jinja import get_rendered from dbt.clients.yaml_helper import yaml, safe_load, SafeLoader, Loader, Dumper # noqa: F401 -from dbt.constants import SECRET_ENV_PREFIX, DEFAULT_ENV_PLACEHOLDER, SECRET_PLACEHOLDER +from dbt_common.constants import SECRET_ENV_PREFIX +from dbt.constants import DEFAULT_ENV_PLACEHOLDER, SECRET_PLACEHOLDER from dbt.contracts.graph.nodes import Resource from dbt.exceptions import ( SecretEnvVarLocationError, diff --git a/core/dbt/context/configured.py b/core/dbt/context/configured.py index c5c95bbfbcb..10a34d8394c 100644 --- a/core/dbt/context/configured.py +++ b/core/dbt/context/configured.py @@ -2,7 +2,8 @@ from dbt_common.context import get_invocation_context -from dbt.constants import SECRET_ENV_PREFIX, DEFAULT_ENV_PLACEHOLDER +from dbt_common.constants import SECRET_ENV_PREFIX +from dbt.constants import DEFAULT_ENV_PLACEHOLDER from dbt.adapters.contracts.connection import AdapterRequiredConfig from dbt.node_types import NodeType from dbt.utils import MultiDict diff --git a/core/dbt/context/providers.py b/core/dbt/context/providers.py index 90f0a508c8c..2b2ecc83f17 100644 --- a/core/dbt/context/providers.py +++ b/core/dbt/context/providers.py @@ -25,7 +25,8 @@ from dbt.adapters.factory import get_adapter, get_adapter_package_names, get_adapter_type_names from dbt.clients.jinja import get_rendered, MacroGenerator, MacroStack, UnitTestMacroGenerator from dbt.config import RuntimeConfig, Project -from dbt.constants import SECRET_ENV_PREFIX, DEFAULT_ENV_PLACEHOLDER +from dbt_common.constants import SECRET_ENV_PREFIX +from dbt.constants import DEFAULT_ENV_PLACEHOLDER from dbt.context.base import contextmember, contextproperty, Var from dbt.context.configured import FQNLookup from dbt.context.context_config import ContextConfig diff --git a/core/dbt/context/secret.py b/core/dbt/context/secret.py index 849e4903a8b..be6a5428531 100644 --- a/core/dbt/context/secret.py +++ b/core/dbt/context/secret.py @@ -4,7 +4,8 @@ from .base import BaseContext, contextmember -from dbt.constants import SECRET_ENV_PREFIX, DEFAULT_ENV_PLACEHOLDER, SECRET_PLACEHOLDER +from dbt_common.constants import SECRET_ENV_PREFIX +from dbt.constants import DEFAULT_ENV_PLACEHOLDER, SECRET_PLACEHOLDER from dbt.exceptions import EnvVarMissingError diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py index 721e65b4c27..eb4d0b97276 100644 --- a/core/dbt/exceptions.py +++ b/core/dbt/exceptions.py @@ -17,7 +17,7 @@ from dbt_common.dataclass_schema import ValidationError -from dbt.constants import SECRET_ENV_PREFIX +from dbt_common.constants import SECRET_ENV_PREFIX if TYPE_CHECKING: diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index 5e406d81d03..be41ee473a5 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -44,8 +44,8 @@ MANIFEST_FILE_NAME, PARTIAL_PARSE_FILE_NAME, SEMANTIC_MANIFEST_FILE_NAME, - SECRET_ENV_PREFIX, ) +from dbt_common.constants import SECRET_ENV_PREFIX from dbt_common.helper_types import PathSet from dbt_common.events.functions import fire_event, get_invocation_id, warn_or_error from dbt_common.events.types import ( diff --git a/tests/functional/context_methods/test_env_vars.py b/tests/functional/context_methods/test_env_vars.py index 33feb3b5de1..dcb03250fae 100644 --- a/tests/functional/context_methods/test_env_vars.py +++ b/tests/functional/context_methods/test_env_vars.py @@ -1,7 +1,8 @@ import pytest import os -from dbt.constants import SECRET_ENV_PREFIX, DEFAULT_ENV_PLACEHOLDER +from dbt_common.constants import SECRET_ENV_PREFIX +from dbt.constants import DEFAULT_ENV_PLACEHOLDER from dbt.tests.util import run_dbt, get_manifest, run_dbt_and_capture diff --git a/tests/functional/context_methods/test_secret_env_vars.py b/tests/functional/context_methods/test_secret_env_vars.py index 5319857459f..94d7e1b1f31 100644 --- a/tests/functional/context_methods/test_secret_env_vars.py +++ b/tests/functional/context_methods/test_secret_env_vars.py @@ -1,7 +1,7 @@ import pytest import os -from dbt.constants import SECRET_ENV_PREFIX +from dbt_common.constants import SECRET_ENV_PREFIX from dbt.exceptions import ParsingError, DbtInternalError from tests.functional.context_methods.first_dependency import FirstDependencyProject from dbt.tests.util import run_dbt, run_dbt_and_capture, read_file diff --git a/tests/functional/partial_parsing/test_pp_vars.py b/tests/functional/partial_parsing/test_pp_vars.py index e55592f8dd2..7a2d9a16b0c 100644 --- a/tests/functional/partial_parsing/test_pp_vars.py +++ b/tests/functional/partial_parsing/test_pp_vars.py @@ -2,7 +2,7 @@ from pathlib import Path import pytest -from dbt.constants import SECRET_ENV_PREFIX +from dbt_common.constants import SECRET_ENV_PREFIX from dbt.exceptions import ParsingError from dbt.adapters.exceptions import FailedToConnectError from dbt.tests.util import get_manifest, run_dbt, run_dbt_and_capture, write_file From a5d151b60d3ca268ffacf8467b50f598fbd252d2 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 23 Apr 2024 14:33:29 -0500 Subject: [PATCH 2/4] changelog --- .changes/unreleased/Under the Hood-20240423-143055.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Under the Hood-20240423-143055.yaml diff --git a/.changes/unreleased/Under the Hood-20240423-143055.yaml b/.changes/unreleased/Under the Hood-20240423-143055.yaml new file mode 100644 index 00000000000..691a26e5ff6 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240423-143055.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Use the SECRET_ENV_PREFIX from dbt_common instead of duplicating it in dbt-core +time: 2024-04-23T14:30:55.739378-05:00 +custom: + Author: emmyoop + Issue: "10018" From 63ec37cbefad41aa0b00148122b97405e5571f41 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 23 Apr 2024 14:43:10 -0500 Subject: [PATCH 3/4] add back the underscores to tests --- .../functional/context_methods/test_env_vars.py | 4 ++-- .../context_methods/test_secret_env_vars.py | 16 ++++++++-------- tests/functional/partial_parsing/test_pp_vars.py | 6 +++--- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/functional/context_methods/test_env_vars.py b/tests/functional/context_methods/test_env_vars.py index dcb03250fae..ebe1c9e31a3 100644 --- a/tests/functional/context_methods/test_env_vars.py +++ b/tests/functional/context_methods/test_env_vars.py @@ -56,13 +56,13 @@ def setup(self): os.environ["DBT_TEST_ENV_VAR"] = "1" os.environ["DBT_TEST_USER"] = "root" os.environ["DBT_TEST_PASS"] = "password" - os.environ[SECRET_ENV_PREFIX + "SECRET"] = "secret_variable" + os.environ[SECRET_ENV_PREFIX + "_SECRET"] = "secret_variable" os.environ["DBT_TEST_NOT_SECRET"] = "regular_variable" os.environ["DBT_TEST_IGNORE_DEFAULT"] = "ignored_default" yield del os.environ["DBT_TEST_ENV_VAR"] del os.environ["DBT_TEST_USER"] - del os.environ[SECRET_ENV_PREFIX + "SECRET"] + del os.environ[SECRET_ENV_PREFIX + "_SECRET"] del os.environ["DBT_TEST_NOT_SECRET"] del os.environ["DBT_TEST_IGNORE_DEFAULT"] diff --git a/tests/functional/context_methods/test_secret_env_vars.py b/tests/functional/context_methods/test_secret_env_vars.py index 94d7e1b1f31..b8f4991d342 100644 --- a/tests/functional/context_methods/test_secret_env_vars.py +++ b/tests/functional/context_methods/test_secret_env_vars.py @@ -73,15 +73,15 @@ def test_disallow_secret(self, project): class TestAllowSecretProfilePackage(FirstDependencyProject): @pytest.fixture(scope="class", autouse=True) def setup(self): - os.environ[SECRET_ENV_PREFIX + "USER"] = "root" - os.environ[SECRET_ENV_PREFIX + "PASS"] = "password" - os.environ[SECRET_ENV_PREFIX + "PACKAGE"] = "first_dependency" - os.environ[SECRET_ENV_PREFIX + "GIT_TOKEN"] = "abc123" + os.environ[SECRET_ENV_PREFIX + "_USER"] = "root" + os.environ[SECRET_ENV_PREFIX + "_PASS"] = "password" + os.environ[SECRET_ENV_PREFIX + "_PACKAGE"] = "first_dependency" + os.environ[SECRET_ENV_PREFIX + "_GIT_TOKEN"] = "abc123" yield - del os.environ[SECRET_ENV_PREFIX + "USER"] - del os.environ[SECRET_ENV_PREFIX + "PASS"] - del os.environ[SECRET_ENV_PREFIX + "PACKAGE"] - del os.environ[SECRET_ENV_PREFIX + "GIT_TOKEN"] + del os.environ[SECRET_ENV_PREFIX + "_USER"] + del os.environ[SECRET_ENV_PREFIX + "_PASS"] + del os.environ[SECRET_ENV_PREFIX + "_PACKAGE"] + del os.environ[SECRET_ENV_PREFIX + "_GIT_TOKEN"] @pytest.fixture(scope="class") def models(self): diff --git a/tests/functional/partial_parsing/test_pp_vars.py b/tests/functional/partial_parsing/test_pp_vars.py index 7a2d9a16b0c..10cb5c6219f 100644 --- a/tests/functional/partial_parsing/test_pp_vars.py +++ b/tests/functional/partial_parsing/test_pp_vars.py @@ -366,7 +366,7 @@ def dbt_profile_target(self): # user is secret and password is not. postgres on macos doesn't care if the password # changes so we have to change the user. related: https://github.com/dbt-labs/dbt-core/pull/4250 - os.environ[SECRET_ENV_PREFIX + "USER"] = "root" + os.environ[SECRET_ENV_PREFIX + "_USER"] = "root" os.environ["ENV_VAR_PASS"] = "password" return { "type": "postgres", @@ -381,7 +381,7 @@ def dbt_profile_target(self): def test_profile_secret_env_vars(self, project): # Initial run - os.environ[SECRET_ENV_PREFIX + "USER"] = "root" + os.environ[SECRET_ENV_PREFIX + "_USER"] = "root" os.environ["ENV_VAR_PASS"] = "password" results = run_dbt(["run"]) @@ -389,7 +389,7 @@ def test_profile_secret_env_vars(self, project): env_vars_checksum = manifest.state_check.profile_env_vars_hash.checksum # Change a secret var, it shouldn't register because we shouldn't save secrets. - os.environ[SECRET_ENV_PREFIX + "USER"] = "fake_user" + os.environ[SECRET_ENV_PREFIX + "_USER"] = "fake_user" # we just want to see if the manifest has included # the secret in the hash of environment variables. (results, log_output) = run_dbt_and_capture(["run"], expect_pass=True) From c09619547ea9ad40a038e084f41c7523e28f7f17 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 23 Apr 2024 14:52:16 -0500 Subject: [PATCH 4/4] add missing underscore --- tests/functional/context_methods/test_secret_env_vars.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/context_methods/test_secret_env_vars.py b/tests/functional/context_methods/test_secret_env_vars.py index b8f4991d342..f2447ffcf95 100644 --- a/tests/functional/context_methods/test_secret_env_vars.py +++ b/tests/functional/context_methods/test_secret_env_vars.py @@ -137,7 +137,7 @@ def test_allow_secrets(self, project, first_dependency): class TestCloneFailSecretScrubbed: @pytest.fixture(scope="class", autouse=True) def setup(self): - os.environ[SECRET_ENV_PREFIX + "GIT_TOKEN"] = "abc123" + os.environ[SECRET_ENV_PREFIX + "_GIT_TOKEN"] = "abc123" @pytest.fixture(scope="class") def models(self):