From c4fe76937353e9a3ba36e27b9db83ab62810d610 Mon Sep 17 00:00:00 2001 From: Andriy Knysh Date: Tue, 17 Dec 2024 14:24:23 -0500 Subject: [PATCH] Improve `!terraform.output` Atmos YAML function. Implement `static` remote state backend for `!terraform.output` and `atmos.Component` functions (#863) * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * update docs --- examples/quick-start-advanced/Dockerfile | 2 +- .../template-functions-test2/defaults.yaml | 14 +++ .../template-functions-test3/defaults.yaml | 20 ++++ .../orgs/cp/tenant1/prod/us-east-2.yaml | 5 +- internal/exec/describe_stacks.go | 4 +- internal/exec/stack_utils.go | 27 +++++- internal/exec/template_funcs_component.go | 20 +++- internal/exec/terraform_generate_backends.go | 2 +- internal/exec/terraform_generate_varfiles.go | 2 +- internal/exec/utils.go | 10 +- internal/exec/yaml_func_exec.go | 6 +- internal/exec/yaml_func_template.go | 6 +- internal/exec/yaml_func_terraform_output.go | 83 ++++++++++++---- internal/exec/yaml_func_utils.go | 28 ++++-- .../templates/functions/atmos.Component.mdx | 65 +++++++++++++ .../stacks/yaml-functions/template.mdx | 54 +++++++++++ .../yaml-functions/terraform.output.mdx | 96 +++++++++++++++++-- website/docs/integrations/atlantis.mdx | 2 +- website/package-lock.json | 78 +++++++-------- website/package.json | 16 ++-- 20 files changed, 440 insertions(+), 100 deletions(-) create mode 100644 examples/tests/stacks/catalog/terraform/template-functions-test3/defaults.yaml diff --git a/examples/quick-start-advanced/Dockerfile b/examples/quick-start-advanced/Dockerfile index 895f09c56..3d722f0d3 100644 --- a/examples/quick-start-advanced/Dockerfile +++ b/examples/quick-start-advanced/Dockerfile @@ -6,7 +6,7 @@ ARG GEODESIC_OS=debian # https://atmos.tools/ # https://github.com/cloudposse/atmos # https://github.com/cloudposse/atmos/releases -ARG ATMOS_VERSION=1.127.0 +ARG ATMOS_VERSION=1.129.0 # Terraform: https://github.com/hashicorp/terraform/releases ARG TF_VERSION=1.5.7 diff --git a/examples/tests/stacks/catalog/terraform/template-functions-test2/defaults.yaml b/examples/tests/stacks/catalog/terraform/template-functions-test2/defaults.yaml index 0042836e7..cd2ce4674 100644 --- a/examples/tests/stacks/catalog/terraform/template-functions-test2/defaults.yaml +++ b/examples/tests/stacks/catalog/terraform/template-functions-test2/defaults.yaml @@ -31,3 +31,17 @@ components: test_30: !exec atmos terraform output template-functions-test -s {{ .stack }} --skip-init -- -json test_label_id test_31: !exec atmos terraform output template-functions-test -s {{ .stack }} --skip-init -- -json test_map test_32: !exec atmos terraform output template-functions-test -s {{ .stack }} --skip-init -- -json test_list + # Call the `!terraform.output` function with two parameters + test_40: !terraform.output template-functions-test test_label_id + test_41: !terraform.output template-functions-test test_list + test_42: !terraform.output template-functions-test test_map + # Component `template-functions-test3` is configured with the remote state backend of type `static` + test_50: !terraform.output template-functions-test3 val1 + test_51: !terraform.output template-functions-test3 {{ .stack }} val1 + test_52: !terraform.output template-functions-test3 val2 + test_53: !terraform.output template-functions-test3 val3 + test_54: !terraform.output template-functions-test3 val4 + test_55: !terraform.output template-functions-test3 val5 + test_56: !terraform.output template-functions-test3 val6 + # test_57: !terraform.output does_not_exist val6 + # test_57: !terraform.output template-functions-test3 invalid-val diff --git a/examples/tests/stacks/catalog/terraform/template-functions-test3/defaults.yaml b/examples/tests/stacks/catalog/terraform/template-functions-test3/defaults.yaml new file mode 100644 index 000000000..b0ce720ec --- /dev/null +++ b/examples/tests/stacks/catalog/terraform/template-functions-test3/defaults.yaml @@ -0,0 +1,20 @@ +# yaml-language-server: $schema=https://atmos.tools/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json + +components: + terraform: + template-functions-test3: + remote_state_backend_type: static + remote_state_backend: + static: + val1: true + val2: "2" + val3: 3 + val4: null + val5: + - item1 + - item2 + - item3 + val6: + i1: 1 + i2: 2 + i3: 3 diff --git a/examples/tests/stacks/orgs/cp/tenant1/prod/us-east-2.yaml b/examples/tests/stacks/orgs/cp/tenant1/prod/us-east-2.yaml index bc33175c1..d9723201b 100644 --- a/examples/tests/stacks/orgs/cp/tenant1/prod/us-east-2.yaml +++ b/examples/tests/stacks/orgs/cp/tenant1/prod/us-east-2.yaml @@ -15,8 +15,9 @@ import: - catalog/terraform/spacelift/infrastructure-tenant1 # Configurations to test `atmos.Component` template function - # - catalog/terraform/template-functions-test/defaults - # - catalog/terraform/template-functions-test2/defaults + # - catalog/terraform/template-functions-test/defaults + # - catalog/terraform/template-functions-test2/defaults + # - catalog/terraform/template-functions-test3/defaults components: terraform: diff --git a/internal/exec/describe_stacks.go b/internal/exec/describe_stacks.go index d22b67554..9e4fb7b6a 100644 --- a/internal/exec/describe_stacks.go +++ b/internal/exec/describe_stacks.go @@ -362,7 +362,7 @@ func ExecuteDescribeStacks( u.LogErrorAndExit(cliConfig, err) } - componentSectionFinal, err := ProcessCustomYamlTags(cliConfig, componentSectionConverted) + componentSectionFinal, err := ProcessCustomYamlTags(cliConfig, componentSectionConverted, stackName) if err != nil { return nil, err } @@ -547,7 +547,7 @@ func ExecuteDescribeStacks( u.LogErrorAndExit(cliConfig, err) } - componentSectionFinal, err := ProcessCustomYamlTags(cliConfig, componentSectionConverted) + componentSectionFinal, err := ProcessCustomYamlTags(cliConfig, componentSectionConverted, stackName) if err != nil { return nil, err } diff --git a/internal/exec/stack_utils.go b/internal/exec/stack_utils.go index 779cf79eb..117ba2900 100644 --- a/internal/exec/stack_utils.go +++ b/internal/exec/stack_utils.go @@ -54,7 +54,8 @@ func BuildTerraformWorkspace(cliConfig schema.CliConfiguration, configAndStacksI return strings.Replace(workspace, "/", "-", -1), nil } -// ProcessComponentMetadata processes component metadata and returns a base component (if any) and whether the component is real or abstract and whether the component is disabled or not +// ProcessComponentMetadata processes component metadata and returns a base component (if any) and whether +// the component is real or abstract and whether the component is disabled or not func ProcessComponentMetadata( component string, componentSection map[string]any, @@ -196,3 +197,27 @@ func IsComponentEnabled(varsSection map[string]any) bool { } return true } + +// GetComponentRemoteStateBackendStaticType returns the `remote_state_backend` section for a component in a stack +// if the `remote_state_backend_type` is `static` +func GetComponentRemoteStateBackendStaticType( + sections map[string]any, +) (map[string]any, error) { + var remoteStateBackend map[string]any + var remoteStateBackendType string + var ok bool + + if remoteStateBackendType, ok = sections[cfg.RemoteStateBackendTypeSectionName].(string); !ok { + return nil, nil + } + + if remoteStateBackendType != "static" { + return nil, nil + } + + if remoteStateBackend, ok = sections[cfg.RemoteStateBackendSectionName].(map[string]any); ok { + return remoteStateBackend, nil + } + + return nil, nil +} diff --git a/internal/exec/template_funcs_component.go b/internal/exec/template_funcs_component.go index 5efae2a5f..6e33b5ee7 100644 --- a/internal/exec/template_funcs_component.go +++ b/internal/exec/template_funcs_component.go @@ -44,13 +44,27 @@ func componentFunc(cliConfig schema.CliConfiguration, component string, stack st return nil, err } - outputProcessed, err := execTerraformOutput(cliConfig, component, stack, sections) + var terraformOutputs map[string]any + + // Check if the component in the stack is configured with the 'static' remote state backend, + // in which case get the `output` from the static remote state instead of executing `terraform output` + remoteStateBackendStaticTypeOutputs, err := GetComponentRemoteStateBackendStaticType(sections) if err != nil { return nil, err } + if remoteStateBackendStaticTypeOutputs != nil { + terraformOutputs = remoteStateBackendStaticTypeOutputs + } else { + // Execute `terraform output` + terraformOutputs, err = execTerraformOutput(cliConfig, component, stack, sections) + if err != nil { + return nil, err + } + } + outputs := map[string]any{ - "outputs": outputProcessed, + "outputs": terraformOutputs, } sections = lo.Assign(sections, outputs) @@ -60,7 +74,7 @@ func componentFunc(cliConfig schema.CliConfiguration, component string, stack st if cliConfig.Logs.Level == u.LogLevelTrace { u.LogTrace(cliConfig, fmt.Sprintf("Executed template function 'atmos.Component(%s, %s)'\n\n'outputs' section:", component, stack)) - y, err2 := u.ConvertToYAML(outputProcessed) + y, err2 := u.ConvertToYAML(terraformOutputs) if err2 != nil { u.LogError(cliConfig, err2) } else { diff --git a/internal/exec/terraform_generate_backends.go b/internal/exec/terraform_generate_backends.go index ca1522fa5..1673a731b 100644 --- a/internal/exec/terraform_generate_backends.go +++ b/internal/exec/terraform_generate_backends.go @@ -253,7 +253,7 @@ func ExecuteTerraformGenerateBackends( u.LogErrorAndExit(cliConfig, err) } - componentSectionFinal, err := ProcessCustomYamlTags(cliConfig, componentSectionConverted) + componentSectionFinal, err := ProcessCustomYamlTags(cliConfig, componentSectionConverted, stackName) if err != nil { return err } diff --git a/internal/exec/terraform_generate_varfiles.go b/internal/exec/terraform_generate_varfiles.go index 3917d4032..9c50da888 100644 --- a/internal/exec/terraform_generate_varfiles.go +++ b/internal/exec/terraform_generate_varfiles.go @@ -269,7 +269,7 @@ func ExecuteTerraformGenerateVarfiles( u.LogErrorAndExit(cliConfig, err) } - componentSectionFinal, err := ProcessCustomYamlTags(cliConfig, componentSectionConverted) + componentSectionFinal, err := ProcessCustomYamlTags(cliConfig, componentSectionConverted, stackName) if err != nil { return err } diff --git a/internal/exec/utils.go b/internal/exec/utils.go index 46693a4ba..32bf516af 100644 --- a/internal/exec/utils.go +++ b/internal/exec/utils.go @@ -398,14 +398,12 @@ func ProcessStacks( } } - if foundStackCount == 0 { + if foundStackCount == 0 && !checkStack { // Allow proceeding without error if checkStack is false (e.g., for operations that don't require a stack) - if !checkStack { - return configAndStacksInfo, nil - } + return configAndStacksInfo, nil } - if foundStackCount == 0 && configAndStacksInfo.ComponentIsEnabled { + if foundStackCount == 0 { cliConfigYaml := "" if cliConfig.Logs.Level == u.LogLevelTrace { @@ -521,7 +519,7 @@ func ProcessStacks( u.LogErrorAndExit(cliConfig, err) } - componentSectionFinal, err := ProcessCustomYamlTags(cliConfig, componentSectionConverted) + componentSectionFinal, err := ProcessCustomYamlTags(cliConfig, componentSectionConverted, configAndStacksInfo.Stack) if err != nil { return configAndStacksInfo, err } diff --git a/internal/exec/yaml_func_exec.go b/internal/exec/yaml_func_exec.go index 232809195..c0e7323c3 100644 --- a/internal/exec/yaml_func_exec.go +++ b/internal/exec/yaml_func_exec.go @@ -9,7 +9,11 @@ import ( u "github.com/cloudposse/atmos/pkg/utils" ) -func processTagExec(cliConfig schema.CliConfiguration, input string) any { +func processTagExec( + cliConfig schema.CliConfiguration, + input string, + currentStack string, +) any { u.LogTrace(cliConfig, fmt.Sprintf("Executing Atmos YAML function: %s", input)) str, err := getStringAfterTag(cliConfig, input, config.AtmosYamlFuncExec) diff --git a/internal/exec/yaml_func_template.go b/internal/exec/yaml_func_template.go index c6b622070..5283e5496 100644 --- a/internal/exec/yaml_func_template.go +++ b/internal/exec/yaml_func_template.go @@ -9,7 +9,11 @@ import ( u "github.com/cloudposse/atmos/pkg/utils" ) -func processTagTemplate(cliConfig schema.CliConfiguration, input string) any { +func processTagTemplate( + cliConfig schema.CliConfiguration, + input string, + currentStack string, +) any { u.LogTrace(cliConfig, fmt.Sprintf("Executing Atmos YAML function: %s", input)) str, err := getStringAfterTag(cliConfig, input, config.AtmosYamlFuncTemplate) diff --git a/internal/exec/yaml_func_terraform_output.go b/internal/exec/yaml_func_terraform_output.go index ece3a79f5..8cfdc8633 100644 --- a/internal/exec/yaml_func_terraform_output.go +++ b/internal/exec/yaml_func_terraform_output.go @@ -1,7 +1,6 @@ package exec import ( - "errors" "fmt" "strings" "sync" @@ -15,26 +14,42 @@ var ( terraformOutputFuncSyncMap = sync.Map{} ) -func processTagTerraformOutput(cliConfig schema.CliConfiguration, input string) any { +func processTagTerraformOutput( + cliConfig schema.CliConfiguration, + input string, + currentStack string, +) any { u.LogTrace(cliConfig, fmt.Sprintf("Executing Atmos YAML function: %s", input)) str, err := getStringAfterTag(cliConfig, input, config.AtmosYamlFuncTerraformOutput) - if err != nil { u.LogErrorAndExit(cliConfig, err) } - parts := strings.Split(str, " ") - - if len(parts) != 3 { - err := errors.New(fmt.Sprintf("invalid Atmos YAML function: %s\nthree parameters are required: component, stack, output", input)) + var component string + var stack string + var output string + + // Split the string into slices based on any whitespace (one or more spaces, tabs, or newlines), + // while also ignoring leading and trailing whitespace + parts := strings.Fields(str) + partsLen := len(parts) + + if partsLen == 3 { + component = strings.TrimSpace(parts[0]) + stack = strings.TrimSpace(parts[1]) + output = strings.TrimSpace(parts[2]) + } else if partsLen == 2 { + component = strings.TrimSpace(parts[0]) + stack = currentStack + output = strings.TrimSpace(parts[1]) + u.LogTrace(cliConfig, fmt.Sprintf("Atmos YAML function `%s` is called with two parameters 'component' and 'output'. "+ + "Using the current stack '%s' as the 'stack' parameter", input, currentStack)) + } else { + err := fmt.Errorf("invalid number of arguments in the Atmos YAML function: %s", input) u.LogErrorAndExit(cliConfig, err) } - component := strings.TrimSpace(parts[0]) - stack := strings.TrimSpace(parts[1]) - output := strings.TrimSpace(parts[2]) - stackSlug := fmt.Sprintf("%s-%s", stack, component) // If the result for the component in the stack already exists in the cache, return it @@ -49,15 +64,28 @@ func processTagTerraformOutput(cliConfig schema.CliConfiguration, input string) u.LogErrorAndExit(cliConfig, err) } - outputProcessed, err := execTerraformOutput(cliConfig, component, stack, sections) + // Check if the component in the stack is configured with the 'static' remote state backend, + // in which case get the `output` from the static remote state instead of executing `terraform output` + remoteStateBackendStaticTypeOutputs, err := GetComponentRemoteStateBackendStaticType(sections) if err != nil { u.LogErrorAndExit(cliConfig, err) } - // Cache the result - terraformOutputFuncSyncMap.Store(stackSlug, outputProcessed) - - return getTerraformOutput(cliConfig, input, component, stack, outputProcessed, output) + if remoteStateBackendStaticTypeOutputs != nil { + // Cache the result + terraformOutputFuncSyncMap.Store(stackSlug, remoteStateBackendStaticTypeOutputs) + return getStaticRemoteStateOutput(cliConfig, input, component, stack, remoteStateBackendStaticTypeOutputs, output) + } else { + // Execute `terraform output` + terraformOutputs, err := execTerraformOutput(cliConfig, component, stack, sections) + if err != nil { + u.LogErrorAndExit(cliConfig, err) + } + + // Cache the result + terraformOutputFuncSyncMap.Store(stackSlug, terraformOutputs) + return getTerraformOutput(cliConfig, input, component, stack, terraformOutputs, output) + } } func getTerraformOutput( @@ -81,3 +109,26 @@ func getTerraformOutput( return nil } + +func getStaticRemoteStateOutput( + cliConfig schema.CliConfiguration, + funcDef string, + component string, + stack string, + remoteStateSection map[string]any, + output string, +) any { + if u.MapKeyExists(remoteStateSection, output) { + return remoteStateSection[output] + } + + u.LogErrorAndExit(cliConfig, fmt.Errorf("invalid Atmos YAML function: %s\nthe component '%s' in the stack '%s' "+ + "is configured with the 'static' remote state backend, but the remote state backend does not have the output '%s'", + funcDef, + component, + stack, + output, + )) + + return nil +} diff --git a/internal/exec/yaml_func_utils.go b/internal/exec/yaml_func_utils.go index fbc3c8be0..db1cf5573 100644 --- a/internal/exec/yaml_func_utils.go +++ b/internal/exec/yaml_func_utils.go @@ -8,18 +8,26 @@ import ( "github.com/cloudposse/atmos/pkg/schema" ) -func ProcessCustomYamlTags(cliConfig schema.CliConfiguration, input schema.AtmosSectionMapType) (schema.AtmosSectionMapType, error) { - return processNodes(cliConfig, input), nil +func ProcessCustomYamlTags( + cliConfig schema.CliConfiguration, + input schema.AtmosSectionMapType, + currentStack string, +) (schema.AtmosSectionMapType, error) { + return processNodes(cliConfig, input, currentStack), nil } -func processNodes(cliConfig schema.CliConfiguration, data map[string]any) map[string]any { +func processNodes( + cliConfig schema.CliConfiguration, + data map[string]any, + currentStack string, +) map[string]any { newMap := make(map[string]any) var recurse func(any) any recurse = func(node any) any { switch v := node.(type) { case string: - return processCustomTags(cliConfig, v) + return processCustomTags(cliConfig, v, currentStack) case map[string]any: newNestedMap := make(map[string]any) @@ -47,13 +55,17 @@ func processNodes(cliConfig schema.CliConfiguration, data map[string]any) map[st return newMap } -func processCustomTags(cliConfig schema.CliConfiguration, input string) any { +func processCustomTags( + cliConfig schema.CliConfiguration, + input string, + currentStack string, +) any { if strings.HasPrefix(input, config.AtmosYamlFuncTemplate) { - return processTagTemplate(cliConfig, input) + return processTagTemplate(cliConfig, input, currentStack) } else if strings.HasPrefix(input, config.AtmosYamlFuncExec) { - return processTagExec(cliConfig, input) + return processTagExec(cliConfig, input, currentStack) } else if strings.HasPrefix(input, config.AtmosYamlFuncTerraformOutput) { - return processTagTerraformOutput(cliConfig, input) + return processTagTerraformOutput(cliConfig, input, currentStack) } // If any other YAML explicit type (not currently supported by Atmos) is used, return it w/o processing diff --git a/website/docs/core-concepts/stacks/templates/functions/atmos.Component.mdx b/website/docs/core-concepts/stacks/templates/functions/atmos.Component.mdx index bcb645db6..4e396c874 100644 --- a/website/docs/core-concepts/stacks/templates/functions/atmos.Component.mdx +++ b/website/docs/core-concepts/stacks/templates/functions/atmos.Component.mdx @@ -6,6 +6,7 @@ description: Read the remote state or configuration of any Atmos component --- import File from '@site/src/components/File' import Intro from '@site/src/components/Intro' +import Terminal from '@site/src/components/Terminal' The `atmos.Component` template function allows reading any Atmos section or any attribute from a section for an @@ -281,3 +282,67 @@ The template function `{{ atmos.Component "test" .stack }}` is executed three ti After the first execution, Atmos caches the result in memory (all the component sections, including the `outputs`), and reuses it in the next two calls to the function. The caching makes the stack processing about three times faster in this particular example. In a production environment where many components are used, the speedup can be even more significant. + +## Using `atmos.Component` with `static` remote state backend + +Atmos supports [brownfield configuration by using the remote state of type `static`](/core-concepts/components/terraform/brownfield/#hacking-remote-state-with-static-backends). + +For example: + + +```yaml +components: + terraform: + # Component `static-backend` is configured with the remote state backend of type `static` + static-backend: + remote_state_backend_type: static + remote_state_backend: + static: + region: "us-west-2" + cluster_name: "production-cluster" + vpc_cidr: "10.0.0.0/16" + database: + type: "postgresql" + version: "12.7" + storage_gb: 100 + allowed_ips: + - "192.168.1.0/24" + - "10.1.0.0/16" + tags: + Environment: "production" + Owner: "infra-team" + + eks-cluster: + vars: + region: '{{ (atmos.Component "static-backend" .stack).outputs.region }}' + cluster_name: '{{ (atmos.Component "static-backend" .stack).outputs.cluster_name }}' + vpc_cidr: '{{ (atmos.Component "static-backend" .stack).outputs.vpc_cidr }}' + db_type: '{{ (atmos.Component "static-backend" .stack).outputs.database.type }}' + db_storage: '{{ (atmos.Component "static-backend" .stack).outputs.database.storage_gb }}' + # Use the `!template` YAML function to correctly handle the outputs of types map and list + allowed_ips: !template '{{ (atmos.Component "static-backend" .stack).outputs.allowed_ips }}' + tags: !template '{{ (atmos.Component "static-backend" .stack).outputs.tags }}' +``` + + +When the functions are executed, Atmos detects that the `static-backend` component has the `static` remote state configured, +and instead of executing `terraform output`, it just returns the static values from the `remote_state_backend.static` section. + +Executing the command `atmos describe component eks-cluster -s ` produces the following result: + + +```yaml +vars: + region: us-west-2 + cluster_name: production-cluster + vpc_cidr: 10.0.0.0/16 + db_type: postgresql + db_storage: 100 + allowed_ips: + - 192.168.1.0/24 + - 10.1.0.0/16 + tags: + Environment: production + Owner: infra-team +``` + diff --git a/website/docs/core-concepts/stacks/yaml-functions/template.mdx b/website/docs/core-concepts/stacks/yaml-functions/template.mdx index 7181af9ed..79dfde8ea 100644 --- a/website/docs/core-concepts/stacks/yaml-functions/template.mdx +++ b/website/docs/core-concepts/stacks/yaml-functions/template.mdx @@ -137,3 +137,57 @@ When reading Atmos components outputs (remote state) in Atmos stack manifests, i YAML function. It produces the same results, correctly handles the complex types (lists and maps), and has a much simpler syntax. ::: + +## Advanced Examples + +The `!template` Atmos YAML function can be used to make your stack configuration DRY and reusable. + +For example, suppose we need to restrict the Security Group ingresses on all components provisioned in the infrastructure +(e.g. EKS cluster, RDS Aurora cluster, MemoryDB cluster, Istio Ingress Gateway) to a specific list of IP CIDR blocks. + +We can define the list of allowed CIDR blocks in the global `settings` section (used by all components in all stacks) +in the `allowed_ingress_cidrs` variable: + +```yaml +settings: + allowed_ingress_cidrs: + - "10.20.0.0/20" # VPN 1 + - "10.30.0.0/20" # VPN 2 +``` + +We can then use the `!template` function with the following template in all components that need their Security Group +to be restricted: + +```yaml +# EKS cluster +# Allow ingress only from the allowed CIDR blocks +allowed_cidr_blocks: !template '{{ toJson .settings.allowed_ingress_cidrs }}' +``` + +```yaml +# RDS cluster +# Allow ingress only from the allowed CIDR blocks +cidr_blocks: !template '{{ toJson .settings.allowed_ingress_cidrs }}' +``` + +```yaml +# Istio Ingress Gateway +# Allow ingress only from the allowed CIDR blocks +security_group_ingress_cidrs: !template '{{ toJson .settings.allowed_ingress_cidrs }}' +``` + +
+ +The `!template` function and the `'{{ toJson .settings.allowed_ingress_cidrs }}'` expression allows you to +use the global `allowed_ingress_cidrs` variable and the same template even if the components have different +variable names for the allowed CIDR blocks (which would be difficult to implement using +[Atmos inheritance](/core-concepts/stacks/inheritance) or other [Atmos design patterns](/design-patterns)). + +:::tip +To append additional CIDRs to the template itself, use the `list` and [Sprig](https://masterminds.github.io/sprig/lists.html) +`concat` functions: + +```yaml +allowed_cidr_blocks: !template '{{ toJson (concat .settings.allowed_ingress_cidrs (list "172.20.0.0/16")) }}' +``` +::: diff --git a/website/docs/core-concepts/stacks/yaml-functions/terraform.output.mdx b/website/docs/core-concepts/stacks/yaml-functions/terraform.output.mdx index 3e11ba647..e7bf368a5 100644 --- a/website/docs/core-concepts/stacks/yaml-functions/terraform.output.mdx +++ b/website/docs/core-concepts/stacks/yaml-functions/terraform.output.mdx @@ -6,6 +6,7 @@ description: Read the remote state of any Atmos component --- import File from '@site/src/components/File' import Intro from '@site/src/components/Intro' +import Terminal from '@site/src/components/Terminal' The `!terraform.output` YAML function allows reading the outputs ([remote state](/core-concepts/components/terraform/remote-state)) @@ -14,7 +15,13 @@ of components directly in Atmos stack manifests. ## Usage +The `!template` function can be called with either two or three parameters: + ```yaml + # Get the `output` of the `component` in the current stack + !terraform.output + + # Get the `output` of the `component` in the provided stack !terraform.output ``` @@ -25,7 +32,7 @@ of components directly in Atmos stack manifests.
Atmos component name
`stack`
-
Atmos stack name
+
(Optional) Atmos stack name
`output`
Terraform output
@@ -48,9 +55,11 @@ components: vars: vpc_config: # Output of type string - security_group_id: !terraform.output security-group/lambda {{ .stack }} id + security_group_id: !terraform.output security-group/lambda id + security_group_id2: !terraform.output security-group/lambda2 {{ .stack }} id + security_group_id3: !terraform.output security-group/lambda3 {{ .atmos_stack }} id # Output of type list - subnet_ids: !terraform.output vpc {{ .stack }} private_subnet_ids + subnet_ids: !terraform.output vpc private_subnet_ids # Output of type map config_map: !terraform.output config {{ .stack }} config_map ``` @@ -58,13 +67,12 @@ components: ## Specifying Atmos `stack` -There are multiple ways you can specify the Atmos stack parameter in the `!terraform.output` function. +If you call the `!temolate` function with three parameters, you need to specify the stack as the second argument. -The `stack` argument is the second argument of the `!terraform.output` function, and it can be specified in a few different ways: +There are multiple ways you can specify the Atmos stack parameter in the `!terraform.output` function. ### Hardcoded Stack Name -Hardcoded stack name. Use it if you want to get an output from a component from a different (well-known and static) stack. For example, you have a `tgw` component in a stack `plat-ue2-dev` that requires the `vpc_id` output from the `vpc` component from the stack `plat-ue2-prod`: @@ -96,6 +104,13 @@ For example, you have a `tgw` component that requires the `vpc_id` output from t vpc_id: !terraform.output vpc {{ .stack }} vpc_id ``` +:::note +Using the `.stack` or `.atmos_stack` template identifiers to specify the stack is the same as calling the `!template` +function with two parameters without specifying the current stack, but without using `Go` templates. +If you need to get an output of a component in the current stack, using the `!template` function with two parameters +is preferred because it has a simpler syntax and executes faster. +::: + ### Use a Format Function Use the `printf` template function to construct stack names using static strings and dynamic identifiers. @@ -158,8 +173,8 @@ components: test2: vars: tags: - test: !terraform.output test {{ .stack }} id - test2: !terraform.output test {{ .stack }} id + test: !terraform.output test id + test2: !terraform.output test id test3: !terraform.output test {{ .stack }} id ``` @@ -171,6 +186,69 @@ After the first execution, Atmos caches the result in memory, and reuses it in the next two calls to the function. The caching makes the stack processing much faster. In a production environment where many components are used, the speedup can be significant. +## Using `!terraform.output` with `static` remote state backend + +Atmos supports [brownfield configuration by using the remote state of type `static`](/core-concepts/components/terraform/brownfield/#hacking-remote-state-with-static-backends). + +For example: + + +```yaml +components: + terraform: + # Component `static-backend` is configured with the remote state backend of type `static` + static-backend: + remote_state_backend_type: static + remote_state_backend: + static: + region: "us-west-2" + cluster_name: "production-cluster" + vpc_cidr: "10.0.0.0/16" + database: + type: "postgresql" + version: "12.7" + storage_gb: 100 + allowed_ips: + - "192.168.1.0/24" + - "10.1.0.0/16" + tags: + Environment: "production" + Owner: "infra-team" + + eks-cluster: + vars: + region: !terraform.output static-backend region + cluster_name: !terraform.output static-backend cluster_name + vpc_cidr: !terraform.output static-backend vpc_cidr + db_type: !terraform.output static-backend database.type + db_storage: !terraform.output static-backend database.storage_gb + allowed_ips: !terraform.output static-backend allowed_ips + tags: !terraform.output static-backend tags +``` + + +When the functions are executed, Atmos detects that the `static-backend` component has the `static` remote state configured, +and instead of executing `terraform output`, it just returns the static values from the `remote_state_backend.static` section. + +Executing the command `atmos describe component eks-cluster -s ` produces the following result: + + +```yaml +vars: + region: us-west-2 + cluster_name: production-cluster + vpc_cidr: 10.0.0.0/16 + db_type: postgresql + db_storage: 100 + allowed_ips: + - 192.168.1.0/24 + - 10.1.0.0/16 + tags: + Environment: production + Owner: infra-team +``` + + ## Considerations - Using `!terraform.output` with secrets can expose sensitive data to standard output (stdout) in any commands that describe stacks or components. @@ -182,4 +260,4 @@ In a production environment where many components are used, the speedup can be s - Be mindful of disaster recovery (DR) implications when using it across regions. -- Consider cold-start scenarios: if the dependent component has not yet been provisioned, `terraform output` will fail. +- Consider cold-start scenarios: if the dependent component has not yet been provisioned, `terraform output` will fail. diff --git a/website/docs/integrations/atlantis.mdx b/website/docs/integrations/atlantis.mdx index e3a544497..f38c4c209 100644 --- a/website/docs/integrations/atlantis.mdx +++ b/website/docs/integrations/atlantis.mdx @@ -673,7 +673,7 @@ on: branches: [ main ] env: - ATMOS_VERSION: 1.127.0 + ATMOS_VERSION: 1.129.0 ATMOS_CLI_CONFIG_PATH: ./ jobs: diff --git a/website/package-lock.json b/website/package-lock.json index 699008b83..027a4fa99 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -15,10 +15,10 @@ "@docusaurus/preset-classic": "^3.6.3", "@docusaurus/theme-mermaid": "^3.6.3", "@excalidraw/excalidraw": "^0.17.6", - "@fortawesome/fontawesome-svg-core": "^6.7.1", - "@fortawesome/free-brands-svg-icons": "^6.7.1", - "@fortawesome/free-regular-svg-icons": "^6.7.1", - "@fortawesome/free-solid-svg-icons": "^6.7.1", + "@fortawesome/fontawesome-svg-core": "^6.7.2", + "@fortawesome/free-brands-svg-icons": "^6.7.2", + "@fortawesome/free-regular-svg-icons": "^6.7.2", + "@fortawesome/free-solid-svg-icons": "^6.7.2", "@fortawesome/react-fontawesome": "^0.2.2", "@grnet/docusaurus-terminology": "^2.0.0-rc.1", "@mdx-js/react": "^3.1.0", @@ -27,11 +27,11 @@ "docusaurus-plugin-image-zoom": "^2.0.0", "docusaurus-plugin-sentry": "^2.0.0", "html-loader": "^5.1.0", - "marked": "^15.0.3", + "marked": "^15.0.4", "node-fetch": "^3.3.2", - "posthog-docusaurus": "^2.0.1", - "posthog-js": "^1.194.5", - "prism-react-renderer": "^2.4.0", + "posthog-docusaurus": "^2.0.2", + "posthog-js": "^1.201.0", + "prism-react-renderer": "^2.4.1", "raw-loader": "^4.0.2", "react": "^18.3.1", "react-dom": "^18.3.1", @@ -4080,57 +4080,57 @@ } }, "node_modules/@fortawesome/fontawesome-common-types": { - "version": "6.7.1", - "resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-common-types/-/fontawesome-common-types-6.7.1.tgz", - "integrity": "sha512-gbDz3TwRrIPT3i0cDfujhshnXO9z03IT1UKRIVi/VEjpNHtSBIP2o5XSm+e816FzzCFEzAxPw09Z13n20PaQJQ==", + "version": "6.7.2", + "resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-common-types/-/fontawesome-common-types-6.7.2.tgz", + "integrity": "sha512-Zs+YeHUC5fkt7Mg1l6XTniei3k4bwG/yo3iFUtZWd/pMx9g3fdvkSK9E0FOC+++phXOka78uJcYb8JaFkW52Xg==", "license": "MIT", "engines": { "node": ">=6" } }, "node_modules/@fortawesome/fontawesome-svg-core": { - "version": "6.7.1", - "resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-svg-core/-/fontawesome-svg-core-6.7.1.tgz", - "integrity": "sha512-8dBIHbfsKlCk2jHQ9PoRBg2Z+4TwyE3vZICSnoDlnsHA6SiMlTwfmW6yX0lHsRmWJugkeb92sA0hZdkXJhuz+g==", + "version": "6.7.2", + "resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-svg-core/-/fontawesome-svg-core-6.7.2.tgz", + "integrity": "sha512-yxtOBWDrdi5DD5o1pmVdq3WMCvnobT0LU6R8RyyVXPvFRd2o79/0NCuQoCjNTeZz9EzA9xS3JxNWfv54RIHFEA==", "license": "MIT", "dependencies": { - "@fortawesome/fontawesome-common-types": "6.7.1" + "@fortawesome/fontawesome-common-types": "6.7.2" }, "engines": { "node": ">=6" } }, "node_modules/@fortawesome/free-brands-svg-icons": { - "version": "6.7.1", - "resolved": "https://registry.npmjs.org/@fortawesome/free-brands-svg-icons/-/free-brands-svg-icons-6.7.1.tgz", - "integrity": "sha512-nJR76eqPzCnMyhbiGf6X0aclDirZriTPRcFm1YFvuupyJOGwlNF022w3YBqu+yrHRhnKRpzFX+8wJKqiIjWZkA==", + "version": "6.7.2", + "resolved": "https://registry.npmjs.org/@fortawesome/free-brands-svg-icons/-/free-brands-svg-icons-6.7.2.tgz", + "integrity": "sha512-zu0evbcRTgjKfrr77/2XX+bU+kuGfjm0LbajJHVIgBWNIDzrhpRxiCPNT8DW5AdmSsq7Mcf9D1bH0aSeSUSM+Q==", "license": "(CC-BY-4.0 AND MIT)", "dependencies": { - "@fortawesome/fontawesome-common-types": "6.7.1" + "@fortawesome/fontawesome-common-types": "6.7.2" }, "engines": { "node": ">=6" } }, "node_modules/@fortawesome/free-regular-svg-icons": { - "version": "6.7.1", - "resolved": "https://registry.npmjs.org/@fortawesome/free-regular-svg-icons/-/free-regular-svg-icons-6.7.1.tgz", - "integrity": "sha512-e13cp+bAx716RZOTQ59DhqikAgETA9u1qTBHO3e3jMQQ+4H/N1NC1ZVeFYt1V0m+Th68BrEL1/X6XplISutbXg==", + "version": "6.7.2", + "resolved": "https://registry.npmjs.org/@fortawesome/free-regular-svg-icons/-/free-regular-svg-icons-6.7.2.tgz", + "integrity": "sha512-7Z/ur0gvCMW8G93dXIQOkQqHo2M5HLhYrRVC0//fakJXxcF1VmMPsxnG6Ee8qEylA8b8Q3peQXWMNZ62lYF28g==", "license": "(CC-BY-4.0 AND MIT)", "dependencies": { - "@fortawesome/fontawesome-common-types": "6.7.1" + "@fortawesome/fontawesome-common-types": "6.7.2" }, "engines": { "node": ">=6" } }, "node_modules/@fortawesome/free-solid-svg-icons": { - "version": "6.7.1", - "resolved": "https://registry.npmjs.org/@fortawesome/free-solid-svg-icons/-/free-solid-svg-icons-6.7.1.tgz", - "integrity": "sha512-BTKc0b0mgjWZ2UDKVgmwaE0qt0cZs6ITcDgjrti5f/ki7aF5zs+N91V6hitGo3TItCFtnKg6cUVGdTmBFICFRg==", + "version": "6.7.2", + "resolved": "https://registry.npmjs.org/@fortawesome/free-solid-svg-icons/-/free-solid-svg-icons-6.7.2.tgz", + "integrity": "sha512-GsBrnOzU8uj0LECDfD5zomZJIjrPhIlWU82AHwa2s40FKH+kcxQaBvBo3Z4TxyZHIyX8XTDxsyA33/Vx9eFuQA==", "license": "(CC-BY-4.0 AND MIT)", "dependencies": { - "@fortawesome/fontawesome-common-types": "6.7.1" + "@fortawesome/fontawesome-common-types": "6.7.2" }, "engines": { "node": ">=6" @@ -10924,9 +10924,9 @@ } }, "node_modules/marked": { - "version": "15.0.3", - "resolved": "https://registry.npmjs.org/marked/-/marked-15.0.3.tgz", - "integrity": "sha512-Ai0cepvl2NHnTcO9jYDtcOEtVBNVYR31XnEA3BndO7f5As1wzpcOceSUM8FDkNLJNIODcLpDTWay/qQhqbuMvg==", + "version": "15.0.4", + "resolved": "https://registry.npmjs.org/marked/-/marked-15.0.4.tgz", + "integrity": "sha512-TCHvDqmb3ZJ4PWG7VEGVgtefA5/euFmsIhxtD0XsBxI39gUSKL81mIRFdt0AiNQozUahd4ke98ZdirExd/vSEw==", "license": "MIT", "bin": { "marked": "bin/marked.js" @@ -15949,18 +15949,18 @@ } }, "node_modules/posthog-docusaurus": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/posthog-docusaurus/-/posthog-docusaurus-2.0.1.tgz", - "integrity": "sha512-MnZvqxvaEt48sUHdz2pvZLKBd8KCXGESq98vTpTSm8uVxtAM3ljc2orZdDErSAKp+WozVEDlQnXTCZ1wzswSYw==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/posthog-docusaurus/-/posthog-docusaurus-2.0.2.tgz", + "integrity": "sha512-RUxVzJqZ214JuEr1msngxXgTYlK+q37Vhhvp4yG07K+UFF12HBU1TrOdl/MEmvHQimsFQNEa68eYasJo2kAsJA==", "license": "MIT", "engines": { "node": ">=10.15.1" } }, "node_modules/posthog-js": { - "version": "1.194.5", - "resolved": "https://registry.npmjs.org/posthog-js/-/posthog-js-1.194.5.tgz", - "integrity": "sha512-bYa20TkwzkDsr2y3iCiJNto/bthkYkmHZopIOXzFEw7KeB581Y1WueaOry5MFHEwnpZuomqEmcMQGBAoWvv8VA==", + "version": "1.201.0", + "resolved": "https://registry.npmjs.org/posthog-js/-/posthog-js-1.201.0.tgz", + "integrity": "sha512-5LG4i36eYykIj5FYaQQ1vmSOriVeiswCNPUAPkpnq4ccmavu8y5L+QIxaX98ACmDAW7DkUDS05yUjYWmd84tzQ==", "license": "MIT", "dependencies": { "core-js": "^3.38.1", @@ -16013,9 +16013,9 @@ } }, "node_modules/prism-react-renderer": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/prism-react-renderer/-/prism-react-renderer-2.4.0.tgz", - "integrity": "sha512-327BsVCD/unU4CNLZTWVHyUHKnsqcvj2qbPlQ8MiBE2eq2rgctjigPA1Gp9HLF83kZ20zNN6jgizHJeEsyFYOw==", + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/prism-react-renderer/-/prism-react-renderer-2.4.1.tgz", + "integrity": "sha512-ey8Ls/+Di31eqzUxC46h8MksNuGx/n0AAC8uKpwFau4RPDYLuE3EXTp8N8G2vX2N7UC/+IXeNUnlWBGGcAG+Ig==", "license": "MIT", "dependencies": { "@types/prismjs": "^1.26.0", diff --git a/website/package.json b/website/package.json index 1e3e00c69..c46ac60a1 100644 --- a/website/package.json +++ b/website/package.json @@ -21,10 +21,10 @@ "@docusaurus/preset-classic": "^3.6.3", "@docusaurus/theme-mermaid": "^3.6.3", "@excalidraw/excalidraw": "^0.17.6", - "@fortawesome/fontawesome-svg-core": "^6.7.1", - "@fortawesome/free-brands-svg-icons": "^6.7.1", - "@fortawesome/free-regular-svg-icons": "^6.7.1", - "@fortawesome/free-solid-svg-icons": "^6.7.1", + "@fortawesome/fontawesome-svg-core": "^6.7.2", + "@fortawesome/free-brands-svg-icons": "^6.7.2", + "@fortawesome/free-regular-svg-icons": "^6.7.2", + "@fortawesome/free-solid-svg-icons": "^6.7.2", "@fortawesome/react-fontawesome": "^0.2.2", "@grnet/docusaurus-terminology": "^2.0.0-rc.1", "@mdx-js/react": "^3.1.0", @@ -33,11 +33,11 @@ "docusaurus-plugin-image-zoom": "^2.0.0", "docusaurus-plugin-sentry": "^2.0.0", "html-loader": "^5.1.0", - "marked": "^15.0.3", + "marked": "^15.0.4", "node-fetch": "^3.3.2", - "posthog-docusaurus": "^2.0.1", - "posthog-js": "^1.194.5", - "prism-react-renderer": "^2.4.0", + "posthog-docusaurus": "^2.0.2", + "posthog-js": "^1.201.0", + "prism-react-renderer": "^2.4.1", "raw-loader": "^4.0.2", "react": "^18.3.1", "react-dom": "^18.3.1",