From 05c24e4c93bab5836465b98f237283bb71043058 Mon Sep 17 00:00:00 2001 From: Peter Webb Date: Wed, 10 Apr 2024 20:03:33 -0400 Subject: [PATCH] Add "Private" Environment Variables (#103) * Add concept of private env vars * Add changelog entry * Keep private env-var prefix, per review --- .../unreleased/Features-20240410-090810.yaml | 6 ++++++ dbt_common/constants.py | 5 +++++ dbt_common/context.py | 9 +++++++-- tests/unit/test_invocation_context.py | 20 ++++++++++++++++++- 4 files changed, 37 insertions(+), 3 deletions(-) create mode 100644 .changes/unreleased/Features-20240410-090810.yaml diff --git a/.changes/unreleased/Features-20240410-090810.yaml b/.changes/unreleased/Features-20240410-090810.yaml new file mode 100644 index 00000000..e9473165 --- /dev/null +++ b/.changes/unreleased/Features-20240410-090810.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Support environment variables which dbt can see but user code cannot. +time: 2024-04-10T09:08:10.068999-04:00 +custom: + Author: peterallenwebb + Issue: "103" diff --git a/dbt_common/constants.py b/dbt_common/constants.py index ca591e05..de70594a 100644 --- a/dbt_common/constants.py +++ b/dbt_common/constants.py @@ -1 +1,6 @@ +# Prefix which identifies environment variables which contains secrets. SECRET_ENV_PREFIX = "DBT_ENV_SECRET_" + +# Prefix which identifies environment variables that should not be visible +# via macros, flags, or other user-facing mechanisms. +PRIVATE_ENV_PREFIX = "DBT_ENV_PRIVATE_" diff --git a/dbt_common/context.py b/dbt_common/context.py index 840f5022..a46b1dd2 100644 --- a/dbt_common/context.py +++ b/dbt_common/context.py @@ -1,13 +1,14 @@ from contextvars import ContextVar, copy_context from typing import List, Mapping, Optional -from dbt_common.constants import SECRET_ENV_PREFIX +from dbt_common.constants import PRIVATE_ENV_PREFIX, SECRET_ENV_PREFIX class InvocationContext: def __init__(self, env: Mapping[str, str]): - self._env = env + self._env = {k: v for k, v in env.items() if not k.startswith(PRIVATE_ENV_PREFIX)} self._env_secrets: Optional[List[str]] = None + self._env_private = {k: v for k, v in env.items() if k.startswith(PRIVATE_ENV_PREFIX)} self.recorder = None # This class will also eventually manage the invocation_id, flags, event manager, etc. @@ -15,6 +16,10 @@ def __init__(self, env: Mapping[str, str]): def env(self) -> Mapping[str, str]: return self._env + @property + def env_private(self) -> Mapping[str, str]: + return self._env_private + @property def env_secrets(self) -> List[str]: if self._env_secrets is None: diff --git a/tests/unit/test_invocation_context.py b/tests/unit/test_invocation_context.py index fa63cbc4..b6697f8e 100644 --- a/tests/unit/test_invocation_context.py +++ b/tests/unit/test_invocation_context.py @@ -1,4 +1,4 @@ -from dbt_common.constants import SECRET_ENV_PREFIX +from dbt_common.constants import PRIVATE_ENV_PREFIX, SECRET_ENV_PREFIX from dbt_common.context import InvocationContext @@ -17,3 +17,21 @@ def test_invocation_context_secrets(): } ic = InvocationContext(env=test_env) assert set(ic.env_secrets) == set(["secret1", "secret2"]) + + +def test_invocation_context_private(): + test_env = { + f"{PRIVATE_ENV_PREFIX}_VAR_1": "private1", + f"{PRIVATE_ENV_PREFIX}VAR_2": "private2", + f"{PRIVATE_ENV_PREFIX}": "private3", + "NON_PRIVATE": "non-private-1", + f"foo{PRIVATE_ENV_PREFIX}": "non-private-2", + } + ic = InvocationContext(env=test_env) + assert ic.env_secrets == [] + assert ic.env_private == { + f"{PRIVATE_ENV_PREFIX}_VAR_1": "private1", + f"{PRIVATE_ENV_PREFIX}VAR_2": "private2", + f"{PRIVATE_ENV_PREFIX}": "private3", + } + assert ic.env == {"NON_PRIVATE": "non-private-1", f"foo{PRIVATE_ENV_PREFIX}": "non-private-2"}