Improve !terraform.output Atmos YAML function. Implement static r…

…emote state backend for `!terraform.output` and `atmos.Component` functions (#863)

* updates

* updates

* updates

* updates

* updates

* updates

* updates

* updates

* updates

* updates

* updates

* updates

* updates

* updates

* updates

* update docs

examples/quick-start-advanced/Dockerfile

@@ -6,7 +6,7 @@ ARG GEODESIC_OS=debian
 # https://atmos.tools/
 # https://github.com/cloudposse/atmos
 # https://github.com/cloudposse/atmos/releases
-ARG ATMOS_VERSION=1.127.0
+ARG ATMOS_VERSION=1.129.0
 
 # Terraform: https://github.com/hashicorp/terraform/releases
 ARG TF_VERSION=1.5.7

examples/tests/stacks/catalog/terraform/template-functions-test2/defaults.yaml

@@ -31,3 +31,17 @@ components:
         test_30: !exec atmos terraform output template-functions-test -s {{ .stack }} --skip-init -- -json test_label_id
         test_31: !exec atmos terraform output template-functions-test -s {{ .stack }} --skip-init -- -json test_map
         test_32: !exec atmos terraform output template-functions-test -s {{ .stack }} --skip-init -- -json test_list
+        # Call the `!terraform.output` function with two parameters
+        test_40: !terraform.output template-functions-test test_label_id
+        test_41: !terraform.output template-functions-test test_list
+        test_42: !terraform.output template-functions-test test_map
+        # Component `template-functions-test3` is configured with the remote state backend of type `static`
+        test_50: !terraform.output template-functions-test3 val1
+        test_51: !terraform.output template-functions-test3 {{ .stack }} val1
+        test_52: !terraform.output template-functions-test3 val2
+        test_53: !terraform.output template-functions-test3 val3
+        test_54: !terraform.output template-functions-test3 val4
+        test_55: !terraform.output template-functions-test3 val5
+        test_56: !terraform.output template-functions-test3 val6
+        # test_57: !terraform.output does_not_exist val6
+        # test_57: !terraform.output template-functions-test3 invalid-val

examples/tests/stacks/catalog/terraform/template-functions-test3/defaults.yaml

@@ -0,0 +1,20 @@
+# yaml-language-server: $schema=https://atmos.tools/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json
+
+components:
+  terraform:
+    template-functions-test3:
+      remote_state_backend_type: static
+      remote_state_backend:
+        static:
+          val1: true
+          val2: "2"
+          val3: 3
+          val4: null
+          val5:
+            - item1
+            - item2
+            - item3
+          val6:
+            i1: 1
+            i2: 2
+            i3: 3

examples/tests/stacks/orgs/cp/tenant1/prod/us-east-2.yaml

@@ -15,8 +15,9 @@ import:
   - catalog/terraform/spacelift/infrastructure-tenant1
 
   # Configurations to test `atmos.Component` template function
-  # - catalog/terraform/template-functions-test/defaults
-  # - catalog/terraform/template-functions-test2/defaults
+  #  - catalog/terraform/template-functions-test/defaults
+  #  - catalog/terraform/template-functions-test2/defaults
+  #  - catalog/terraform/template-functions-test3/defaults
 
 components:
   terraform:

internal/exec/describe_stacks.go

@@ -362,7 +362,7 @@ func ExecuteDescribeStacks(
 									u.LogErrorAndExit(cliConfig, err)
 								}
 
-								componentSectionFinal, err := ProcessCustomYamlTags(cliConfig, componentSectionConverted)
+								componentSectionFinal, err := ProcessCustomYamlTags(cliConfig, componentSectionConverted, stackName)
 								if err != nil {
 									return nil, err
 								}
@@ -547,7 +547,7 @@ func ExecuteDescribeStacks(
 									u.LogErrorAndExit(cliConfig, err)
 								}
 
-								componentSectionFinal, err := ProcessCustomYamlTags(cliConfig, componentSectionConverted)
+								componentSectionFinal, err := ProcessCustomYamlTags(cliConfig, componentSectionConverted, stackName)
 								if err != nil {
 									return nil, err
 								}

internal/exec/stack_utils.go

@@ -54,7 +54,8 @@ func BuildTerraformWorkspace(cliConfig schema.CliConfiguration, configAndStacksI
 	return strings.Replace(workspace, "/", "-", -1), nil
 }
 
-// ProcessComponentMetadata processes component metadata and returns a base component (if any) and whether the component is real or abstract and whether the component is disabled or not
+// ProcessComponentMetadata processes component metadata and returns a base component (if any) and whether
+// the component is real or abstract and whether the component is disabled or not
 func ProcessComponentMetadata(
 	component string,
 	componentSection map[string]any,
@@ -196,3 +197,27 @@ func IsComponentEnabled(varsSection map[string]any) bool {
 	}
 	return true
 }
+
+// GetComponentRemoteStateBackendStaticType returns the `remote_state_backend` section for a component in a stack
+// if the `remote_state_backend_type` is `static`
+func GetComponentRemoteStateBackendStaticType(
+	sections map[string]any,
+) (map[string]any, error) {
+	var remoteStateBackend map[string]any
+	var remoteStateBackendType string
+	var ok bool
+
+	if remoteStateBackendType, ok = sections[cfg.RemoteStateBackendTypeSectionName].(string); !ok {
+		return nil, nil
+	}
+
+	if remoteStateBackendType != "static" {
+		return nil, nil
+	}
+
+	if remoteStateBackend, ok = sections[cfg.RemoteStateBackendSectionName].(map[string]any); ok {
+		return remoteStateBackend, nil
+	}
+
+	return nil, nil
+}

internal/exec/template_funcs_component.go

@@ -44,13 +44,27 @@ func componentFunc(cliConfig schema.CliConfiguration, component string, stack st
 		return nil, err
 	}
 
-	outputProcessed, err := execTerraformOutput(cliConfig, component, stack, sections)
+	var terraformOutputs map[string]any
+
+	// Check if the component in the stack is configured with the 'static' remote state backend,
+	// in which case get the `output` from the static remote state instead of executing `terraform output`
+	remoteStateBackendStaticTypeOutputs, err := GetComponentRemoteStateBackendStaticType(sections)
 	if err != nil {
 		return nil, err
 	}
 
+	if remoteStateBackendStaticTypeOutputs != nil {
+		terraformOutputs = remoteStateBackendStaticTypeOutputs
+	} else {
+		// Execute `terraform output`
+		terraformOutputs, err = execTerraformOutput(cliConfig, component, stack, sections)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	outputs := map[string]any{
-		"outputs": outputProcessed,
+		"outputs": terraformOutputs,
 	}
 
 	sections = lo.Assign(sections, outputs)
@@ -60,7 +74,7 @@ func componentFunc(cliConfig schema.CliConfiguration, component string, stack st
 
 	if cliConfig.Logs.Level == u.LogLevelTrace {
 		u.LogTrace(cliConfig, fmt.Sprintf("Executed template function 'atmos.Component(%s, %s)'\n\n'outputs' section:", component, stack))
-		y, err2 := u.ConvertToYAML(outputProcessed)
+		y, err2 := u.ConvertToYAML(terraformOutputs)
 		if err2 != nil {
 			u.LogError(cliConfig, err2)
 		} else {

internal/exec/terraform_generate_backends.go

@@ -253,7 +253,7 @@ func ExecuteTerraformGenerateBackends(
 					u.LogErrorAndExit(cliConfig, err)
 				}
 
-				componentSectionFinal, err := ProcessCustomYamlTags(cliConfig, componentSectionConverted)
+				componentSectionFinal, err := ProcessCustomYamlTags(cliConfig, componentSectionConverted, stackName)
 				if err != nil {
 					return err
 				}

internal/exec/terraform_generate_varfiles.go

@@ -269,7 +269,7 @@ func ExecuteTerraformGenerateVarfiles(
 					u.LogErrorAndExit(cliConfig, err)
 				}
 
-				componentSectionFinal, err := ProcessCustomYamlTags(cliConfig, componentSectionConverted)
+				componentSectionFinal, err := ProcessCustomYamlTags(cliConfig, componentSectionConverted, stackName)
 				if err != nil {
 					return err
 				}

internal/exec/utils.go

@@ -398,14 +398,12 @@ func ProcessStacks(
 			}
 		}
 
-		if foundStackCount == 0 {
+		if foundStackCount == 0 && !checkStack {
 			// Allow proceeding without error if checkStack is false (e.g., for operations that don't require a stack)
-			if !checkStack {
-				return configAndStacksInfo, nil
-			}
+			return configAndStacksInfo, nil
 		}
 
-		if foundStackCount == 0 && configAndStacksInfo.ComponentIsEnabled {
+		if foundStackCount == 0 {
 			cliConfigYaml := ""
 
 			if cliConfig.Logs.Level == u.LogLevelTrace {
@@ -521,7 +519,7 @@ func ProcessStacks(
 			u.LogErrorAndExit(cliConfig, err)
 		}
 
-		componentSectionFinal, err := ProcessCustomYamlTags(cliConfig, componentSectionConverted)
+		componentSectionFinal, err := ProcessCustomYamlTags(cliConfig, componentSectionConverted, configAndStacksInfo.Stack)
 		if err != nil {
 			return configAndStacksInfo, err
 		}

internal/exec/yaml_func_exec.go

@@ -9,7 +9,11 @@ import (
 	u "github.com/cloudposse/atmos/pkg/utils"
 )
 
-func processTagExec(cliConfig schema.CliConfiguration, input string) any {
+func processTagExec(
+	cliConfig schema.CliConfiguration,
+	input string,
+	currentStack string,
+) any {
 	u.LogTrace(cliConfig, fmt.Sprintf("Executing Atmos YAML function: %s", input))
 
 	str, err := getStringAfterTag(cliConfig, input, config.AtmosYamlFuncExec)

internal/exec/yaml_func_template.go

@@ -9,7 +9,11 @@ import (
 	u "github.com/cloudposse/atmos/pkg/utils"
 )
 
-func processTagTemplate(cliConfig schema.CliConfiguration, input string) any {
+func processTagTemplate(
+	cliConfig schema.CliConfiguration,
+	input string,
+	currentStack string,
+) any {
 	u.LogTrace(cliConfig, fmt.Sprintf("Executing Atmos YAML function: %s", input))
 
 	str, err := getStringAfterTag(cliConfig, input, config.AtmosYamlFuncTemplate)

internal/exec/yaml_func_terraform_output.go

@@ -1,7 +1,6 @@
 package exec
 
 import (
-	"errors"
 	"fmt"
 	"strings"
 	"sync"
@@ -15,26 +14,42 @@ var (
 	terraformOutputFuncSyncMap = sync.Map{}
 )
 
-func processTagTerraformOutput(cliConfig schema.CliConfiguration, input string) any {
+func processTagTerraformOutput(
+	cliConfig schema.CliConfiguration,
+	input string,
+	currentStack string,
+) any {
 	u.LogTrace(cliConfig, fmt.Sprintf("Executing Atmos YAML function: %s", input))
 
 	str, err := getStringAfterTag(cliConfig, input, config.AtmosYamlFuncTerraformOutput)
-
 	if err != nil {
 		u.LogErrorAndExit(cliConfig, err)
 	}
 
-	parts := strings.Split(str, " ")
-
-	if len(parts) != 3 {
-		err := errors.New(fmt.Sprintf("invalid Atmos YAML function: %s\nthree parameters are required: component, stack, output", input))
+	var component string
+	var stack string
+	var output string
+
+	// Split the string into slices based on any whitespace (one or more spaces, tabs, or newlines),
+	// while also ignoring leading and trailing whitespace
+	parts := strings.Fields(str)
+	partsLen := len(parts)
+
+	if partsLen == 3 {
+		component = strings.TrimSpace(parts[0])
+		stack = strings.TrimSpace(parts[1])
+		output = strings.TrimSpace(parts[2])
+	} else if partsLen == 2 {
+		component = strings.TrimSpace(parts[0])
+		stack = currentStack
+		output = strings.TrimSpace(parts[1])
+		u.LogTrace(cliConfig, fmt.Sprintf("Atmos YAML function `%s` is called with two parameters 'component' and 'output'. "+
+			"Using the current stack '%s' as the 'stack' parameter", input, currentStack))
+	} else {
+		err := fmt.Errorf("invalid number of arguments in the Atmos YAML function: %s", input)
 		u.LogErrorAndExit(cliConfig, err)
 	}
 
-	component := strings.TrimSpace(parts[0])
-	stack := strings.TrimSpace(parts[1])
-	output := strings.TrimSpace(parts[2])
-
 	stackSlug := fmt.Sprintf("%s-%s", stack, component)
 
 	// If the result for the component in the stack already exists in the cache, return it
@@ -49,15 +64,28 @@ func processTagTerraformOutput(cliConfig schema.CliConfiguration, input string)
 		u.LogErrorAndExit(cliConfig, err)
 	}
 
-	outputProcessed, err := execTerraformOutput(cliConfig, component, stack, sections)
+	// Check if the component in the stack is configured with the 'static' remote state backend,
+	// in which case get the `output` from the static remote state instead of executing `terraform output`
+	remoteStateBackendStaticTypeOutputs, err := GetComponentRemoteStateBackendStaticType(sections)
 	if err != nil {
 		u.LogErrorAndExit(cliConfig, err)
 	}
 
-	// Cache the result
-	terraformOutputFuncSyncMap.Store(stackSlug, outputProcessed)
-
-	return getTerraformOutput(cliConfig, input, component, stack, outputProcessed, output)
+	if remoteStateBackendStaticTypeOutputs != nil {
+		// Cache the result
+		terraformOutputFuncSyncMap.Store(stackSlug, remoteStateBackendStaticTypeOutputs)
+		return getStaticRemoteStateOutput(cliConfig, input, component, stack, remoteStateBackendStaticTypeOutputs, output)
+	} else {
+		// Execute `terraform output`
+		terraformOutputs, err := execTerraformOutput(cliConfig, component, stack, sections)
+		if err != nil {
+			u.LogErrorAndExit(cliConfig, err)
+		}
+
+		// Cache the result
+		terraformOutputFuncSyncMap.Store(stackSlug, terraformOutputs)
+		return getTerraformOutput(cliConfig, input, component, stack, terraformOutputs, output)
+	}
 }
 
 func getTerraformOutput(
@@ -81,3 +109,26 @@ func getTerraformOutput(
 
 	return nil
 }
+
+func getStaticRemoteStateOutput(
+	cliConfig schema.CliConfiguration,
+	funcDef string,
+	component string,
+	stack string,
+	remoteStateSection map[string]any,
+	output string,
+) any {
+	if u.MapKeyExists(remoteStateSection, output) {
+		return remoteStateSection[output]
+	}
+
+	u.LogErrorAndExit(cliConfig, fmt.Errorf("invalid Atmos YAML function: %s\nthe component '%s' in the stack '%s' "+
+		"is configured with the 'static' remote state backend, but the remote state backend does not have the output '%s'",
+		funcDef,
+		component,
+		stack,
+		output,
+	))
+
+	return nil
+}