From cf24006a9ffeda9dd91c05a73a5c612192950e2f Mon Sep 17 00:00:00 2001 From: Cerebrovinny Date: Thu, 14 Nov 2024 09:26:42 +0000 Subject: [PATCH 1/9] folder creation vendor fix --- internal/exec/vendor_utils.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index d2d5cd7a6..c51ff6223 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -551,6 +551,11 @@ func ExecuteAtmosVendorInternal( } } + targetDir := filepath.Dir(targetPath) + if err := os.MkdirAll(targetDir, 0755); err != nil { + return fmt.Errorf("failed to create target directory '%s': %w", targetDir, err) + } + if err = cp.Copy(tempDir, targetPath, copyOptions); err != nil { return err } From 1b5f112c0d15eec7b113bbd2aa72f0a71ff0a0e4 Mon Sep 17 00:00:00 2001 From: Cerebrovinny Date: Fri, 15 Nov 2024 07:03:34 +0000 Subject: [PATCH 2/9] display error logic --- internal/exec/vendor_utils.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index c51ff6223..d067e328c 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -128,13 +128,12 @@ func ReadAndProcessVendorConfigFile( vendorConfigFile string, ) (schema.AtmosVendorConfig, bool, string, error) { var vendorConfig schema.AtmosVendorConfig + var foundVendorConfigFile string + var vendorConfigFileExists bool // Initialize empty sources slice vendorConfig.Spec.Sources = []schema.AtmosVendorSource{} - var vendorConfigFileExists bool - var foundVendorConfigFile string - // Check if vendor config is specified in atmos.yaml if cliConfig.Vendor.BasePath != "" { if !filepath.IsAbs(cliConfig.Vendor.BasePath) { @@ -142,6 +141,12 @@ func ReadAndProcessVendorConfigFile( } else { foundVendorConfigFile = cliConfig.Vendor.BasePath } + + // Check if specified path exists + if !u.FileExists(foundVendorConfigFile) { + u.LogWarning(cliConfig, fmt.Sprintf("Vendor config path '%s' specified in atmos.yaml does not exist", foundVendorConfigFile)) + return vendorConfig, false, "", nil + } } else { // Path is not defined in atmos.yaml, proceed with existing logic var fileExists bool @@ -152,8 +157,8 @@ func ReadAndProcessVendorConfigFile( pathToVendorConfig := path.Join(cliConfig.BasePath, vendorConfigFile) if !u.FileExists(pathToVendorConfig) { - vendorConfigFileExists = false - return vendorConfig, vendorConfigFileExists, "", fmt.Errorf("vendor config file or directory '%s' does not exist", pathToVendorConfig) + // Instead of returning error, return false to indicate no vendor config exists + return vendorConfig, false, "", nil } foundVendorConfigFile = pathToVendorConfig @@ -551,11 +556,6 @@ func ExecuteAtmosVendorInternal( } } - targetDir := filepath.Dir(targetPath) - if err := os.MkdirAll(targetDir, 0755); err != nil { - return fmt.Errorf("failed to create target directory '%s': %w", targetDir, err) - } - if err = cp.Copy(tempDir, targetPath, copyOptions); err != nil { return err } From 3159dbdaf94792afe14139c9ebbd6e927578e684 Mon Sep 17 00:00:00 2001 From: Cerebrovinny Date: Fri, 15 Nov 2024 07:37:35 +0000 Subject: [PATCH 3/9] Revert "display error logic" This reverts commit 1b5f112c0d15eec7b113bbd2aa72f0a71ff0a0e4. --- internal/exec/vendor_utils.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index d067e328c..c51ff6223 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -128,12 +128,13 @@ func ReadAndProcessVendorConfigFile( vendorConfigFile string, ) (schema.AtmosVendorConfig, bool, string, error) { var vendorConfig schema.AtmosVendorConfig - var foundVendorConfigFile string - var vendorConfigFileExists bool // Initialize empty sources slice vendorConfig.Spec.Sources = []schema.AtmosVendorSource{} + var vendorConfigFileExists bool + var foundVendorConfigFile string + // Check if vendor config is specified in atmos.yaml if cliConfig.Vendor.BasePath != "" { if !filepath.IsAbs(cliConfig.Vendor.BasePath) { @@ -141,12 +142,6 @@ func ReadAndProcessVendorConfigFile( } else { foundVendorConfigFile = cliConfig.Vendor.BasePath } - - // Check if specified path exists - if !u.FileExists(foundVendorConfigFile) { - u.LogWarning(cliConfig, fmt.Sprintf("Vendor config path '%s' specified in atmos.yaml does not exist", foundVendorConfigFile)) - return vendorConfig, false, "", nil - } } else { // Path is not defined in atmos.yaml, proceed with existing logic var fileExists bool @@ -157,8 +152,8 @@ func ReadAndProcessVendorConfigFile( pathToVendorConfig := path.Join(cliConfig.BasePath, vendorConfigFile) if !u.FileExists(pathToVendorConfig) { - // Instead of returning error, return false to indicate no vendor config exists - return vendorConfig, false, "", nil + vendorConfigFileExists = false + return vendorConfig, vendorConfigFileExists, "", fmt.Errorf("vendor config file or directory '%s' does not exist", pathToVendorConfig) } foundVendorConfigFile = pathToVendorConfig @@ -556,6 +551,11 @@ func ExecuteAtmosVendorInternal( } } + targetDir := filepath.Dir(targetPath) + if err := os.MkdirAll(targetDir, 0755); err != nil { + return fmt.Errorf("failed to create target directory '%s': %w", targetDir, err) + } + if err = cp.Copy(tempDir, targetPath, copyOptions); err != nil { return err } From 59a960220d22691351683e2b076498d6c2eceaae Mon Sep 17 00:00:00 2001 From: Cerebrovinny Date: Fri, 15 Nov 2024 07:48:17 +0000 Subject: [PATCH 4/9] restore condition to continue vendor process if not exists --- internal/exec/vendor_utils.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index c51ff6223..4bac66a06 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -83,7 +83,7 @@ func ExecuteVendorPullCommand(cmd *cobra.Command, args []string) error { // Check `vendor.yaml` vendorConfig, vendorConfigExists, foundVendorConfigFile, err := ReadAndProcessVendorConfigFile(cliConfig, cfg.AtmosVendorConfigFileName) - if err != nil { + if vendorConfigExists && err != nil { return err } @@ -551,11 +551,6 @@ func ExecuteAtmosVendorInternal( } } - targetDir := filepath.Dir(targetPath) - if err := os.MkdirAll(targetDir, 0755); err != nil { - return fmt.Errorf("failed to create target directory '%s': %w", targetDir, err) - } - if err = cp.Copy(tempDir, targetPath, copyOptions); err != nil { return err } From 945e8de0a303658850351b63f6b6260d18fa65e8 Mon Sep 17 00:00:00 2001 From: Haitham Rageh Date: Tue, 19 Nov 2024 07:11:54 +0200 Subject: [PATCH 5/9] handle empty stack yaml file configuration (#791) * fix: handle empty stack YAML configuration and adjust file content retrieval * fix: return existing content from GetFileContent instead of an empty string * fix: remove unnecessary blank lines in ProcessYAMLConfigFile and GetFileContent functions --- examples/demo-localstack/stacks/deploy/footbar.yaml | 0 internal/exec/stack_processor_utils.go | 3 +++ 2 files changed, 3 insertions(+) create mode 100644 examples/demo-localstack/stacks/deploy/footbar.yaml diff --git a/examples/demo-localstack/stacks/deploy/footbar.yaml b/examples/demo-localstack/stacks/deploy/footbar.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/internal/exec/stack_processor_utils.go b/internal/exec/stack_processor_utils.go index b6c6f4d0e..b616e2348 100644 --- a/internal/exec/stack_processor_utils.go +++ b/internal/exec/stack_processor_utils.go @@ -199,6 +199,9 @@ func ProcessYAMLConfigFile( return nil, nil, nil, err } } + if stackYamlConfig == "" { + return map[string]any{}, map[string]map[string]any{}, map[string]any{}, nil + } stackManifestTemplatesProcessed := stackYamlConfig stackManifestTemplatesErrorMessage := "" From 50ff73eb2f202eb112908a6f01f7e16a669df21f Mon Sep 17 00:00:00 2001 From: Haitham Rageh Date: Tue, 19 Nov 2024 16:04:14 +0200 Subject: [PATCH 6/9] Set Default Schema to Remote Schema (#777) * feat: set default Atmos manifest URL if not specified in configuration * feat: log trace message when using default Atmos JSON Schema file * fix: improve error message for missing Atmos JSON Schema file * Update internal/exec/validate_stacks.go * fix: change log level from trace to info for default Atmos JSON Schema message * fix: change log level from info to trace for default Atmos JSON Schema message --------- Co-authored-by: Erik Osterman (CEO @ Cloud Posse) Co-authored-by: Andriy Knysh --- internal/exec/validate_stacks.go | 43 ++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/internal/exec/validate_stacks.go b/internal/exec/validate_stacks.go index 29c50bd15..13830b1a0 100644 --- a/internal/exec/validate_stacks.go +++ b/internal/exec/validate_stacks.go @@ -19,6 +19,8 @@ import ( u "github.com/cloudposse/atmos/pkg/utils" ) +const atmosManifestDefault = "https://atmos.tools/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json" + // ExecuteValidateStacksCmd executes `validate stacks` command func ExecuteValidateStacksCmd(cmd *cobra.Command, args []string) error { info, err := processCommandLineArgs("", cmd, args, nil) @@ -84,27 +86,30 @@ func ValidateStacks(cliConfig schema.CliConfiguration) error { // The path to the Atmos manifest JSON Schema can be absolute path or a path relative to the `base_path` setting in `atmos.yaml` var atmosManifestJsonSchemaFilePath string - if cliConfig.Schemas.Atmos.Manifest != "" { - atmosManifestJsonSchemaFileAbsPath := path.Join(cliConfig.BasePath, cliConfig.Schemas.Atmos.Manifest) - - if u.FileExists(cliConfig.Schemas.Atmos.Manifest) { - atmosManifestJsonSchemaFilePath = cliConfig.Schemas.Atmos.Manifest - } else if u.FileExists(atmosManifestJsonSchemaFileAbsPath) { - atmosManifestJsonSchemaFilePath = atmosManifestJsonSchemaFileAbsPath - } else if u.IsURL(cliConfig.Schemas.Atmos.Manifest) { - atmosManifestJsonSchemaFilePath, err = downloadSchemaFromURL(cliConfig.Schemas.Atmos.Manifest) - if err != nil { - return err - } - } else { - return fmt.Errorf("the Atmos JSON Schema file '%s' does not exist.\n"+ - "It can be configured in the 'schemas.atmos.manifest' section in 'atmos.yaml', or provided using the 'ATMOS_SCHEMAS_ATMOS_MANIFEST' "+ - "ENV variable or '--schemas-atmos-manifest' command line argument.\n"+ - "The path to the schema file should be an absolute path or a path relative to the 'base_path' setting in 'atmos.yaml'. \n"+ - "Alternatively, you can specify a schema file using a URL that will be downloaded automatically.", - cliConfig.Schemas.Atmos.Manifest) + if cliConfig.Schemas.Atmos.Manifest == "" { + cliConfig.Schemas.Atmos.Manifest = atmosManifestDefault + u.LogTrace(cliConfig, fmt.Sprintf("The Atmos JSON Schema file is not configured. Using the default schema '%s'", atmosManifestDefault)) + } + atmosManifestJsonSchemaFileAbsPath := path.Join(cliConfig.BasePath, cliConfig.Schemas.Atmos.Manifest) + + if u.FileExists(cliConfig.Schemas.Atmos.Manifest) { + atmosManifestJsonSchemaFilePath = cliConfig.Schemas.Atmos.Manifest + } else if u.FileExists(atmosManifestJsonSchemaFileAbsPath) { + atmosManifestJsonSchemaFilePath = atmosManifestJsonSchemaFileAbsPath + } else if u.IsURL(cliConfig.Schemas.Atmos.Manifest) { + atmosManifestJsonSchemaFilePath, err = downloadSchemaFromURL(cliConfig.Schemas.Atmos.Manifest) + if err != nil { + return err } + } else { + return fmt.Errorf("Schema file '%s' not found. Configure via:\n"+ + "1. 'schemas.atmos.manifest' in atmos.yaml\n"+ + "2. ATMOS_SCHEMAS_ATMOS_MANIFEST env var\n"+ + "3. --schemas-atmos-manifest flag\n\n"+ + "Accepts: absolute path, paths relative to base_path, or URL", + cliConfig.Schemas.Atmos.Manifest) } + // Include (process and validate) all YAML files in the `stacks` folder in all subfolders includedPaths := []string{"**/*"} // Don't exclude any YAML files for validation From 0f9cead407fd3ab044ff84a0b929c515bcdeb4bf Mon Sep 17 00:00:00 2001 From: Cerebrovinny Date: Tue, 19 Nov 2024 18:00:47 +0000 Subject: [PATCH 7/9] added better error handling --- internal/exec/vendor_utils.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index 4bac66a06..c4cb4f06f 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -83,7 +83,7 @@ func ExecuteVendorPullCommand(cmd *cobra.Command, args []string) error { // Check `vendor.yaml` vendorConfig, vendorConfigExists, foundVendorConfigFile, err := ReadAndProcessVendorConfigFile(cliConfig, cfg.AtmosVendorConfigFileName) - if vendorConfigExists && err != nil { + if err != nil { return err } @@ -153,7 +153,8 @@ func ReadAndProcessVendorConfigFile( if !u.FileExists(pathToVendorConfig) { vendorConfigFileExists = false - return vendorConfig, vendorConfigFileExists, "", fmt.Errorf("vendor config file or directory '%s' does not exist", pathToVendorConfig) + u.LogWarning(cliConfig, fmt.Sprintf("Vendor config file '%s' does not exist. Proceeding without vendor configurations", pathToVendorConfig)) + return vendorConfig, vendorConfigFileExists, "", nil } foundVendorConfigFile = pathToVendorConfig From aa8751916b7a8cba8c5f5124f3487c54ac9a17fc Mon Sep 17 00:00:00 2001 From: Cerebrovinny Date: Thu, 28 Nov 2024 22:38:38 +0000 Subject: [PATCH 8/9] added vendor config integration tests --- pkg/vender/vendor_config_test.go | 142 +++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 pkg/vender/vendor_config_test.go diff --git a/pkg/vender/vendor_config_test.go b/pkg/vender/vendor_config_test.go new file mode 100644 index 000000000..973af686b --- /dev/null +++ b/pkg/vender/vendor_config_test.go @@ -0,0 +1,142 @@ +package vender + +import ( + "os" + "path" + "testing" + + "github.com/stretchr/testify/assert" + + e "github.com/cloudposse/atmos/internal/exec" + "github.com/cloudposse/atmos/pkg/schema" +) + +func TestVendorConfigScenarios(t *testing.T) { + testDir := t.TempDir() + + // Initialize CLI config with required paths + cliConfig := schema.CliConfiguration{ + BasePath: testDir, + Components: schema.Components{ + Terraform: schema.Terraform{ + BasePath: "components/terraform", + }, + }, + } + cliConfig.Logs.Level = "Trace" + + // Setup test component directory + componentPath := path.Join(testDir, "components", "terraform", "myapp") + err := os.MkdirAll(componentPath, 0755) + assert.Nil(t, err) + + // Test Case 1: vendor.yaml exists and component is defined in it + t.Run("vendor.yaml exists with defined component", func(t *testing.T) { + // Create vendor.yaml + vendorYaml := `apiVersion: atmos/v1 +kind: AtmosVendorConfig +metadata: + name: test-vendor-config +spec: + sources: + - component: myapp + source: github.com/cloudposse/terraform-null-label.git//exports?ref={{.Version}} + version: 0.25.0 + included_paths: + - "**/*.tf" +` + vendorYamlPath := path.Join(testDir, "vendor.yaml") + err := os.WriteFile(vendorYamlPath, []byte(vendorYaml), 0644) + assert.Nil(t, err) + + // Test vendoring with component flag + vendorConfig, exists, configFile, err := e.ReadAndProcessVendorConfigFile(cliConfig, vendorYamlPath) + assert.Nil(t, err) + assert.True(t, exists) + assert.NotEmpty(t, configFile) + + // Verify the component exists in vendor config + var found bool + for _, source := range vendorConfig.Spec.Sources { + if source.Component == "myapp" { + found = true + break + } + } + assert.True(t, found, "Component 'myapp' should be defined in vendor.yaml") + + // Clean up + err = os.Remove(vendorYamlPath) + assert.Nil(t, err) + }) + + // Test Case 2: No vendor.yaml but component.yaml exists + t.Run("component.yaml exists without vendor.yaml", func(t *testing.T) { + // Create component.yaml + componentYaml := `apiVersion: atmos/v1 +kind: ComponentVendorConfig +metadata: + name: myapp-vendor-config +spec: + source: + uri: github.com/cloudposse/terraform-null-label.git//exports?ref={{.Version}} + version: 0.25.0 +` + componentYamlPath := path.Join(componentPath, "component.yaml") + err := os.WriteFile(componentYamlPath, []byte(componentYaml), 0644) + assert.Nil(t, err) + + // Test component vendoring + componentConfig, compPath, err := e.ReadAndProcessComponentVendorConfigFile(cliConfig, "myapp", "terraform") + assert.Nil(t, err) + assert.NotNil(t, componentConfig) + assert.Equal(t, componentPath, compPath) + + // Clean up + err = os.Remove(componentYamlPath) + assert.Nil(t, err) + }) + + // Test Case 3: Neither vendor.yaml nor component.yaml exists + t.Run("no vendor.yaml or component.yaml", func(t *testing.T) { + // Test vendoring with component flag + vendorYamlPath := path.Join(testDir, "vendor.yaml") + _, exists, _, err := e.ReadAndProcessVendorConfigFile(cliConfig, vendorYamlPath) + assert.Nil(t, err) + assert.False(t, exists) + + // Test component vendoring + _, _, err = e.ReadAndProcessComponentVendorConfigFile(cliConfig, "myapp", "terraform") + assert.Error(t, err) + assert.Contains(t, err.Error(), "does not exist") + }) + + // Test Case 4: No component specified with vendor.yaml + t.Run("no component specified with vendor.yaml", func(t *testing.T) { + // Create vendor.yaml + vendorYaml := `apiVersion: atmos/v1 +kind: AtmosVendorConfig +metadata: + name: test-vendor-config +spec: + sources: + - component: myapp + source: github.com/cloudposse/terraform-null-label.git//exports?ref={{.Version}} + version: 0.25.0 +` + vendorYamlPath := path.Join(testDir, "vendor.yaml") + err := os.WriteFile(vendorYamlPath, []byte(vendorYaml), 0644) + assert.Nil(t, err) + + // Test vendoring without component flag + vendorConfig, exists, configFile, err := e.ReadAndProcessVendorConfigFile(cliConfig, vendorYamlPath) + assert.Nil(t, err) + assert.True(t, exists) + assert.NotEmpty(t, configFile) + assert.NotNil(t, vendorConfig.Spec.Sources) + + // Clean up + err = os.Remove(vendorYamlPath) + assert.Nil(t, err) + }) +} From d71a49a4757e4104617f443dd2bcb39ff27a2368 Mon Sep 17 00:00:00 2001 From: "Vinicius C." Date: Tue, 3 Dec 2024 21:53:55 +0000 Subject: [PATCH 9/9] explain package name choose Co-authored-by: Erik Osterman (CEO @ Cloud Posse) --- pkg/vender/vendor_config_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/vender/vendor_config_test.go b/pkg/vender/vendor_config_test.go index 973af686b..a37bf2696 100644 --- a/pkg/vender/vendor_config_test.go +++ b/pkg/vender/vendor_config_test.go @@ -1,3 +1,5 @@ +// The package name `vendor` is reserved in Go for dependency management. +// To avoid conflicts, the name `vender` was chosen as an alternative. package vender import (