[CDF-22997] 🛡️ limit env replacement to loaders with auth/credentials (…

…#1155)

* First pass introducing env replacement toggle

* testin'

* Testing that loaders with auth will replace env vars

* more tests

* testing replacements

* regen'ed tests

* testdata

* ref

* cleaner code

cognite_toolkit/_cdf_tk/loaders/_base_loaders.py

@@ -162,6 +162,7 @@ class ResourceLoader(
     support_drop = True
     filetypes = frozenset({"yaml", "yml"})
     dependencies: frozenset[type[ResourceLoader]] = frozenset()
+    do_environment_variable_injection = False
 
     # The methods that must be implemented in the subclass
     @classmethod
@@ -220,7 +221,11 @@ def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceLoader],
     def load_resource(
         self, filepath: Path, ToolGlobals: CDFToolConfig, skip_validation: bool
     ) -> T_WriteClass | T_CogniteResourceList | None:
-        raw_yaml = load_yaml_inject_variables(filepath, ToolGlobals.environment_variables())
+        use_environment_variables = (
+            ToolGlobals.environment_variables() if self.do_environment_variable_injection else {}
+        )
+        raw_yaml = load_yaml_inject_variables(filepath, use_environment_variables)
+
         if isinstance(raw_yaml, list):
             return self.list_write_cls.load(raw_yaml)
         else:

cognite_toolkit/_cdf_tk/loaders/_resource_loaders/auth_loaders.py

@@ -279,14 +279,17 @@ def _create_replace_method_by_acl_and_scope(
     def load_resource(
         self, filepath: Path, ToolGlobals: CDFToolConfig, skip_validation: bool
     ) -> GroupWrite | GroupWriteList | None:
-        raw = load_yaml_inject_variables(filepath, ToolGlobals.environment_variables())
+        use_environment_variables = (
+            ToolGlobals.environment_variables() if self.do_environment_variable_injection else {}
+        )
+        raw_yaml = load_yaml_inject_variables(filepath, use_environment_variables)
 
         group_write_list = GroupWriteList([])
 
-        if isinstance(raw, dict):
-            raw = [raw]
+        if isinstance(raw_yaml, dict):
+            raw_yaml = [raw_yaml]
 
-        for raw_group in raw:
+        for raw_group in raw_yaml:
             is_resource_scoped = any(
                 any(scope_name in capability.get(acl, {}).get("scope", {}) for scope_name in self.resource_scope_names)
                 for capability in raw_group.get("capabilities", [])

cognite_toolkit/_cdf_tk/loaders/_resource_loaders/classic_loaders.py

@@ -141,8 +141,12 @@ def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceLoader],
     def load_resource(self, filepath: Path, ToolGlobals: CDFToolConfig, skip_validation: bool) -> AssetWriteList:
         resources: list[dict[str, Any]]
         if filepath.suffix in {".yaml", ".yml"}:
-            raw = load_yaml_inject_variables(filepath, ToolGlobals.environment_variables())
-            resources = [raw] if isinstance(raw, dict) else raw
+            use_environment_variables = (
+                ToolGlobals.environment_variables() if self.do_environment_variable_injection else {}
+            )
+            raw_yaml = load_yaml_inject_variables(filepath, use_environment_variables)
+
+            resources = [raw_yaml] if isinstance(raw_yaml, dict) else raw_yaml
         elif filepath.suffix == ".csv" or filepath.suffix == ".parquet":
             if filepath.suffix == ".csv":
                 # The replacement is used to ensure that we read exactly the same file on Windows and Linux

cognite_toolkit/_cdf_tk/loaders/_resource_loaders/data_organization_loaders.py

@@ -258,7 +258,10 @@ def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceLoader],
     def load_resource(
         self, filepath: Path, ToolGlobals: CDFToolConfig, skip_validation: bool
     ) -> LabelDefinitionWrite | LabelDefinitionWriteList | None:
-        raw_yaml = load_yaml_inject_variables(filepath, ToolGlobals.environment_variables())
+        use_environment_variables = (
+            ToolGlobals.environment_variables() if self.do_environment_variable_injection else {}
+        )
+        raw_yaml = load_yaml_inject_variables(filepath, use_environment_variables)
         items: list[dict[str, Any]] = [raw_yaml] if isinstance(raw_yaml, dict) else raw_yaml
         for item in items:
             if "dataSetExternalId" in item:

cognite_toolkit/_cdf_tk/loaders/_resource_loaders/datamodel_loaders.py

@@ -276,7 +276,10 @@ def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceLoader],
     def load_resource(
         self, filepath: Path, ToolGlobals: CDFToolConfig, skip_validation: bool
     ) -> ContainerApply | ContainerApplyList | None:
-        raw_yaml = load_yaml_inject_variables(filepath, ToolGlobals.environment_variables())
+        use_environment_variables = (
+            ToolGlobals.environment_variables() if self.do_environment_variable_injection else {}
+        )
+        raw_yaml = load_yaml_inject_variables(filepath, use_environment_variables)
         dict_items = raw_yaml if isinstance(raw_yaml, list) else [raw_yaml]
         for raw_instance in dict_items:
             for prop in raw_instance.get("properties", {}).values():
@@ -660,7 +663,10 @@ def load_resource(self, filepath: Path, ToolGlobals: CDFToolConfig, skip_validat
         # However, we do not want to put this burden on the user (knowing the intricate workings of YAML),
         # so we fix it here.
         raw_str = quote_int_value_by_key_in_yaml(safe_read(filepath), key="version")
-        raw_yaml = load_yaml_inject_variables(raw_str, ToolGlobals.environment_variables())
+        use_environment_variables = (
+            ToolGlobals.environment_variables() if self.do_environment_variable_injection else {}
+        )
+        raw_yaml = load_yaml_inject_variables(raw_str, use_environment_variables)
         if isinstance(raw_yaml, list):
             return ViewApplyList.load(raw_yaml)
         else:
@@ -806,7 +812,10 @@ def load_resource(self, filepath: Path, ToolGlobals: CDFToolConfig, skip_validat
         # However, we do not want to put this burden on the user (knowing the intricate workings of YAML),
         # so we fix it here.
         raw_str = quote_int_value_by_key_in_yaml(safe_read(filepath), key="version")
-        raw_yaml = load_yaml_inject_variables(raw_str, ToolGlobals.environment_variables())
+        use_environment_variables = (
+            ToolGlobals.environment_variables() if self.do_environment_variable_injection else {}
+        )
+        raw_yaml = load_yaml_inject_variables(raw_str, use_environment_variables)
         if isinstance(raw_yaml, list):
             return DataModelApplyList.load(raw_yaml)
         else:
@@ -911,8 +920,11 @@ def _are_equal(
         return self._return_are_equal(local_dumped, cdf_dumped, return_dumped)
 
     def load_resource(self, filepath: Path, ToolGlobals: CDFToolConfig, skip_validation: bool) -> NodeApplyList:
-        raw = load_yaml_inject_variables(filepath, ToolGlobals.environment_variables())
-        raw_list = raw if isinstance(raw, list) else [raw]
+        use_environment_variables = (
+            ToolGlobals.environment_variables() if self.do_environment_variable_injection else {}
+        )
+        raw_yaml = load_yaml_inject_variables(filepath, use_environment_variables)
+        raw_list = raw_yaml if isinstance(raw_yaml, list) else [raw_yaml]
         return NodeApplyList._load(raw_list, cognite_client=self.client)
 
     def dump_resource(
@@ -1077,9 +1089,12 @@ def load_resource(
         # However, we do not want to put this burden on the user (knowing the intricate workings of YAML),
         # so we fix it here.
         raw_str = quote_int_value_by_key_in_yaml(safe_read(filepath), key="version")
+        use_environment_variables = (
+            ToolGlobals.environment_variables() if self.do_environment_variable_injection else {}
+        )
+        raw_yaml = load_yaml_inject_variables(raw_str, use_environment_variables)
 
-        raw = load_yaml_inject_variables(raw_str, ToolGlobals.environment_variables())
-        raw_list = raw if isinstance(raw, list) else [raw]
+        raw_list = raw_yaml if isinstance(raw_yaml, list) else [raw_yaml]
         models = GraphQLDataModelWriteList._load(raw_list)
 
         # Find the GraphQL files adjacent to the DML files

cognite_toolkit/_cdf_tk/loaders/_resource_loaders/extraction_pipeline_loaders.py

@@ -131,7 +131,11 @@ def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceLoader],
     def load_resource(
         self, filepath: Path, ToolGlobals: CDFToolConfig, skip_validation: bool
     ) -> ExtractionPipelineWrite | ExtractionPipelineWriteList:
-        resources = load_yaml_inject_variables(filepath, {})
+        use_environment_variables = (
+            ToolGlobals.environment_variables() if self.do_environment_variable_injection else {}
+        )
+        resources = load_yaml_inject_variables(filepath, use_environment_variables)
+
         if isinstance(resources, dict):
             resources = [resources]
 

cognite_toolkit/_cdf_tk/loaders/_resource_loaders/file_loader.py

@@ -125,7 +125,10 @@ def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceLoader],
             yield AssetLoader, asset_external_id
 
     def load_resource(self, filepath: Path, ToolGlobals: CDFToolConfig, skip_validation: bool) -> FileMetadataWriteList:
-        loaded = load_yaml_inject_variables(filepath, ToolGlobals.environment_variables())
+        use_environment_variables = (
+            ToolGlobals.environment_variables() if self.do_environment_variable_injection else {}
+        )
+        loaded = load_yaml_inject_variables(filepath, use_environment_variables)
 
         loaded_list = [loaded] if isinstance(loaded, dict) else loaded
 

cognite_toolkit/_cdf_tk/loaders/_resource_loaders/function_loaders.py

@@ -119,7 +119,10 @@ def load_resource(
             # This is to allow arbitrary YAML files inside the function code folder.
             return None
 
-        functions = load_yaml_inject_variables(filepath, ToolGlobals.environment_variables())
+        use_environment_variables = (
+            ToolGlobals.environment_variables() if self.do_environment_variable_injection else {}
+        )
+        functions = load_yaml_inject_variables(filepath, use_environment_variables)
 
         if isinstance(functions, dict):
             functions = [functions]
@@ -283,6 +286,7 @@ class FunctionScheduleLoader(
     kind = "Schedule"
     dependencies = frozenset({FunctionLoader})
     _doc_url = "Function-schedules/operation/postFunctionSchedules"
+    do_environment_variable_injection = True
 
     @property
     def display_name(self) -> str:
@@ -331,7 +335,11 @@ def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceLoader],
     def load_resource(
         self, filepath: Path, ToolGlobals: CDFToolConfig, skip_validation: bool
     ) -> FunctionScheduleWriteList:
-        schedules = load_yaml_inject_variables(filepath, ToolGlobals.environment_variables())
+        use_environment_variables = (
+            ToolGlobals.environment_variables() if self.do_environment_variable_injection else {}
+        )
+        schedules = load_yaml_inject_variables(filepath, use_environment_variables)
+
         if isinstance(schedules, dict):
             schedules = [schedules]
 

cognite_toolkit/_cdf_tk/loaders/_resource_loaders/hosted_extractors.py

@@ -46,6 +46,7 @@ class HostedExtractorSourceLoader(ResourceLoader[str, SourceWrite, Source, Sourc
     kind = "Source"
     _doc_base_url = "https://api-docs.cognite.com/20230101-alpha/tag/"
     _doc_url = "Sources/operation/create_sources"
+    do_environment_variable_injection = True
 
     @property
     def display_name(self) -> str:
@@ -129,6 +130,7 @@ class HostedExtractorDestinationLoader(
     kind = "Destination"
     _doc_base_url = "https://api-docs.cognite.com/20230101-alpha/tag/"
     _doc_url = "Destinations/operation/create_destinations"
+    do_environment_variable_injection = True
 
     def __init__(self, client: ToolkitClient, build_dir: Path | None):
         super().__init__(client, build_dir)
@@ -183,7 +185,10 @@ def iterate(self) -> Iterable[Destination]:
         return iter(self.client.hosted_extractors.destinations)
 
     def load_resource(self, filepath: Path, ToolGlobals: CDFToolConfig, skip_validation: bool) -> DestinationWriteList:
-        raw_yaml = load_yaml_inject_variables(filepath, ToolGlobals.environment_variables())
+        use_environment_variables = (
+            ToolGlobals.environment_variables() if self.do_environment_variable_injection else {}
+        )
+        raw_yaml = load_yaml_inject_variables(filepath, use_environment_variables)
 
         raw_list = raw_yaml if isinstance(raw_yaml, list) else [raw_yaml]
         loaded = DestinationWriteList([])

cognite_toolkit/_cdf_tk/loaders/_resource_loaders/location_loaders.py

@@ -89,7 +89,11 @@ def dump_id(cls, id: str) -> dict[str, Any]:
     def load_resource(
         self, filepath: Path, ToolGlobals: CDFToolConfig, skip_validation: bool
     ) -> LocationFilterWriteList:
-        raw_yaml = load_yaml_inject_variables(filepath, ToolGlobals.environment_variables())
+        use_environment_variables = (
+            ToolGlobals.environment_variables() if self.do_environment_variable_injection else {}
+        )
+        raw_yaml = load_yaml_inject_variables(filepath, use_environment_variables)
+
         raw_list = raw_yaml if isinstance(raw_yaml, list) else [raw_yaml]
         for raw in raw_list:
             if "parentExternalId" in raw:

cognite_toolkit/_cdf_tk/loaders/_resource_loaders/three_d_model_loaders.py

@@ -163,8 +163,12 @@ def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceLoader],
             yield DataSetsLoader, item["dataSetExternalId"]
 
     def load_resource(self, filepath: Path, ToolGlobals: CDFToolConfig, skip_validation: bool) -> ThreeDModelWriteList:
-        raw = load_yaml_inject_variables(filepath, ToolGlobals.environment_variables())
-        resources = raw if isinstance(raw, list) else [raw]
+        use_environment_variables = (
+            ToolGlobals.environment_variables() if self.do_environment_variable_injection else {}
+        )
+        raw_yaml = load_yaml_inject_variables(filepath, use_environment_variables)
+
+        resources = raw_yaml if isinstance(raw_yaml, list) else [raw_yaml]
 
         for resource in resources:
             if resource.get("dataSetExternalId") is not None:

cognite_toolkit/_cdf_tk/loaders/_resource_loaders/timeseries_loaders.py

@@ -121,7 +121,11 @@ def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceLoader],
             yield AssetLoader, item["assetExternalId"]
 
     def load_resource(self, filepath: Path, ToolGlobals: CDFToolConfig, skip_validation: bool) -> TimeSeriesWriteList:
-        resources = load_yaml_inject_variables(filepath, {})
+        use_environment_variables = (
+            ToolGlobals.environment_variables() if self.do_environment_variable_injection else {}
+        )
+        resources = load_yaml_inject_variables(filepath, use_environment_variables)
+
         if not isinstance(resources, list):
             resources = [resources]
         for resource in resources:

cognite_toolkit/_cdf_tk/loaders/_resource_loaders/transformation_loaders.py

@@ -109,6 +109,7 @@ class TransformationLoader(
         }
     )
     _doc_url = "Transformations/operation/createTransformations"
+    do_environment_variable_injection = True
 
     @classmethod
     def get_required_capability(
@@ -204,7 +205,11 @@ def load_resource(
         # If the destination is a DataModel or a View we need to ensure that the version is a string
         raw_str = quote_int_value_by_key_in_yaml(safe_read(filepath), key="version")
 
-        resources = load_yaml_inject_variables(raw_str, ToolGlobals.environment_variables())
+        use_environment_variables = (
+            ToolGlobals.environment_variables() if self.do_environment_variable_injection else {}
+        )
+        resources = load_yaml_inject_variables(raw_str, use_environment_variables)
+
         # The `authentication` key is custom for this template:
 
         if isinstance(resources, dict):
@@ -393,11 +398,15 @@ def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceLoader],
     def load_resource(
         self, filepath: Path, ToolGlobals: CDFToolConfig, skip_validation: bool
     ) -> TransformationScheduleWrite | TransformationScheduleWriteList | None:
-        raw = load_yaml_inject_variables(filepath, ToolGlobals.environment_variables())
-        if isinstance(raw, dict):
-            return TransformationScheduleWrite.load(raw)
+        use_environment_variables = (
+            ToolGlobals.environment_variables() if self.do_environment_variable_injection else {}
+        )
+        raw_yaml = load_yaml_inject_variables(filepath, use_environment_variables)
+
+        if isinstance(raw_yaml, dict):
+            return TransformationScheduleWrite.load(raw_yaml)
         else:
-            return TransformationScheduleWriteList.load(raw)
+            return TransformationScheduleWriteList.load(raw_yaml)
 
     def create(self, items: Sequence[TransformationScheduleWrite]) -> TransformationScheduleList:
         try:

cognite_toolkit/_cdf_tk/loaders/_resource_loaders/workflow_loaders.py

@@ -295,6 +295,7 @@ class WorkflowTriggerLoader(
     dependencies = frozenset({WorkflowLoader, WorkflowVersionLoader})
 
     _doc_url = "Workflow-triggers/operation/CreateOrUpdateTriggers"
+    do_environment_variable_injection = True
 
     def __init__(self, client: ToolkitClient, build_dir: Path | None):
         super().__init__(client, build_dir)
@@ -422,7 +423,11 @@ def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceLoader],
     def load_resource(
         self, filepath: Path, ToolGlobals: CDFToolConfig, skip_validation: bool
     ) -> WorkflowTriggerUpsertList:
-        raw_yaml = load_yaml_inject_variables(filepath, ToolGlobals.environment_variables())
+        use_environment_variables = (
+            ToolGlobals.environment_variables() if self.do_environment_variable_injection else {}
+        )
+        raw_yaml = load_yaml_inject_variables(filepath, use_environment_variables)
+
         raw_list = raw_yaml if isinstance(raw_yaml, list) else [raw_yaml]
         loaded = WorkflowTriggerUpsertList([])
         for item in raw_list:

tests/data/__init__.py

@@ -5,6 +5,7 @@
 DESCRIPTIONS_FOLDER = DATA_FOLDER / "describe_data"
 AUTH_DATA = DATA_FOLDER / "auth_data"
 PROJECT_NO_COGNITE_MODULES = DATA_FOLDER / "project_no_cognite_modules"
+RESOURCES_WITH_ENVIRONMENT_VARIABLES = DATA_FOLDER / "resources_with_environment_variables"
 PROJECT_WITH_DUPLICATES = DATA_FOLDER / "project_with_duplicates"
 PROJECT_FOR_TEST = DATA_FOLDER / "project_for_test"
 LOAD_DATA = DATA_FOLDER / "load_data"
@@ -31,4 +32,5 @@
     "BUILD_GROUP_WITH_UNKNOWN_ACL",
     "COMPLETE_ORG",
     "CDF_TOML_DATA",
+    "RESOURCES_WITH_ENVIRONMENT_VARIABLES",
 ]

tests/data/resources_with_environment_variables/modules/example_module/README.md

@@ -0,0 +1,3 @@
+# README.md
+
+This is an example of a custom module. It contains two TimeSeries with a dataset.

tests/data/resources_with_environment_variables/modules/example_module/functions/README.md

@@ -0,0 +1,4 @@
+# cdf_functions_dummy
+
+This module is an example for how to define a simple function as a template in a tookit module.
+The functions can be found below the 'functions' folder, with one folder per function.

tests/data/resources_with_environment_variables/modules/example_module/functions/first.Function.yaml

@@ -0,0 +1,22 @@
+# The dir with the function code should have the same name
+# and externalId as the function itself as defined below.
+- name: 'first:example:function'
+  externalId: 'fn_first_function'
+  owner: 'Anonymous'
+  description: 'Returns the input data, secrets, and function info.'
+  metadata:
+    version: '1'
+  secrets:
+    mysecret: '${SOME_VARIABLE}'
+  envVars:
+    # The below two environment variables are set by the Toolkit
+    ENV_TYPE: '${CDF_BUILD_TYPE}'
+    CDF_ENV: '${CDF_ENVIRON}'
+  # Number of cores, not available in Azure
+  #cpu: 0.25
+  # Not available in Azure
+  #memory: 1
+  runtime: 'py311'
+  functionPath: './src/handler.py'
+  # Data set id for the zip file with the code that is uploaded.
+  dataSetExternalId: 'ds_timeseries_{{example_variable}}'

tests/data/resources_with_environment_variables/modules/example_module/functions/first.Schedule.yaml

@@ -0,0 +1,14 @@
+- name: "daily-8am-utc"
+  functionExternalId: 'fn_first_function'
+  description: "Run every day at 8am UTC"
+  cronExpression: "0 8 * * *"
+  data:
+    breakfast: "today: peanut butter sandwich and coffee"
+    lunch: "today: greek salad and water"
+    dinner: "today: steak and red wine"
+  authentication:
+    # Credentials to use to run the function in this schedule.
+    # In this example, we just use the main deploy credentials, so the result is the same, but use a different set of
+    # credentials (env variables) if you want to run the function with different permissions.
+    clientId: ${SOME_VARIABLE}
+    clientSecret: ${ANOTHER_VARIABLE}

tests/data/resources_with_environment_variables/modules/example_module/functions/fn_first_function/arbitrary.yaml

@@ -0,0 +1,6 @@
+# This is an arbitrary YAML file that is not a function config
+# It will be copied to the build directory as is
+mapping:
+  key1: value1
+  key2: value2
+  key3: value3