Add "Private" Environment Variables (#103)

* Add concept of private env vars

* Add changelog entry

* Keep private env-var prefix, per review

.changes/unreleased/Features-20240410-090810.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Support environment variables which dbt can see but user code cannot.
+time: 2024-04-10T09:08:10.068999-04:00
+custom:
+  Author: peterallenwebb
+  Issue: "103"

dbt_common/constants.py

@@ -1 +1,6 @@
+# Prefix which identifies environment variables which contains secrets.
 SECRET_ENV_PREFIX = "DBT_ENV_SECRET_"
+
+# Prefix which identifies environment variables that should not be visible
+# via macros, flags, or other user-facing mechanisms.
+PRIVATE_ENV_PREFIX = "DBT_ENV_PRIVATE_"

dbt_common/context.py

@@ -1,20 +1,25 @@
 from contextvars import ContextVar, copy_context
 from typing import List, Mapping, Optional
 
-from dbt_common.constants import SECRET_ENV_PREFIX
+from dbt_common.constants import PRIVATE_ENV_PREFIX, SECRET_ENV_PREFIX
 
 
 class InvocationContext:
     def __init__(self, env: Mapping[str, str]):
-        self._env = env
+        self._env = {k: v for k, v in env.items() if not k.startswith(PRIVATE_ENV_PREFIX)}
         self._env_secrets: Optional[List[str]] = None
+        self._env_private = {k: v for k, v in env.items() if k.startswith(PRIVATE_ENV_PREFIX)}
         self.recorder = None
         # This class will also eventually manage the invocation_id, flags, event manager, etc.
 
     @property
     def env(self) -> Mapping[str, str]:
         return self._env
 
+    @property
+    def env_private(self) -> Mapping[str, str]:
+        return self._env_private
+
     @property
     def env_secrets(self) -> List[str]:
         if self._env_secrets is None:

tests/unit/test_invocation_context.py

@@ -1,4 +1,4 @@
-from dbt_common.constants import SECRET_ENV_PREFIX
+from dbt_common.constants import PRIVATE_ENV_PREFIX, SECRET_ENV_PREFIX
 from dbt_common.context import InvocationContext
 
 
@@ -17,3 +17,21 @@ def test_invocation_context_secrets():
     }
     ic = InvocationContext(env=test_env)
     assert set(ic.env_secrets) == set(["secret1", "secret2"])
+
+
+def test_invocation_context_private():
+    test_env = {
+        f"{PRIVATE_ENV_PREFIX}_VAR_1": "private1",
+        f"{PRIVATE_ENV_PREFIX}VAR_2": "private2",
+        f"{PRIVATE_ENV_PREFIX}": "private3",
+        "NON_PRIVATE": "non-private-1",
+        f"foo{PRIVATE_ENV_PREFIX}": "non-private-2",
+    }
+    ic = InvocationContext(env=test_env)
+    assert ic.env_secrets == []
+    assert ic.env_private == {
+        f"{PRIVATE_ENV_PREFIX}_VAR_1": "private1",
+        f"{PRIVATE_ENV_PREFIX}VAR_2": "private2",
+        f"{PRIVATE_ENV_PREFIX}": "private3",
+    }
+    assert ic.env == {"NON_PRIVATE": "non-private-1", f"foo{PRIVATE_ENV_PREFIX}": "non-private-2"}