From 6a274426a0c9a63db226a41305f6952f225fd678 Mon Sep 17 00:00:00 2001 From: "Erik Osterman (CEO @ Cloud Posse)" Date: Mon, 23 Dec 2024 15:47:17 -0600 Subject: [PATCH] Fix Mocks for Windows (#877) --- .github/workflows/test.yml | 106 +++++++++++++--- tests/cli_test.go | 250 +++++++++++++++++++++++++++++++++++++ tests/test_cases.yaml | 170 +++++++++++++++++++++++++ 3 files changed, 509 insertions(+), 17 deletions(-) create mode 100644 tests/cli_test.go create mode 100644 tests/test_cases.yaml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 516b80097..3c589351e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -40,6 +40,10 @@ jobs: - name: Build run: echo "Building on ${{ matrix.os }}" + - name: Add GNU tar to PATH (significantly faster than windows tar) + if: matrix.target == 'windows' + run: echo "C:\Program Files\Git\usr\bin" >> $Env:GITHUB_PATH + - name: Check out code into the Go module directory uses: actions/checkout@v4 @@ -75,19 +79,57 @@ jobs: strategy: fail-fast: false matrix: - include: - - os: ubuntu-latest - target: linux - - os: windows-latest - target: windows - - os: macos-latest - target: macos + flavor: + - { os: ubuntu-latest, target: linux } + - { os: windows-latest, target: windows } + - { os: macos-latest, target: macos } timeout-minutes: 15 - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.flavor.os }} steps: - name: Check out code into the Go module directory uses: actions/checkout@v4 + - name: Add GNU tar to PATH (significantly faster than windows tar) + if: matrix.flavor.target == 'windows' + run: echo "C:\Program Files\Git\usr\bin" >> $Env:GITHUB_PATH + + - name: Download build artifacts for ${{ matrix.flavor.target }} + uses: actions/download-artifact@v4 + with: + name: build-artifacts-${{ matrix.flavor.target }} + path: ${{ github.workspace }} + + - name: Add build artifacts directory to PATH for linux or macos + if: matrix.flavor.target == 'linux' || matrix.flavor.target == 'macos' + run: | + echo "${{ github.workspace }}" >> $GITHUB_PATH + chmod +x "${{ github.workspace }}/atmos" + + - name: Add build artifacts directory to PATH for windows + if: matrix.flavor.target == 'windows' + shell: pwsh + run: | + $atmosPath = Join-Path ${{ github.workspace }} "atmos.exe" + if (-not (Test-Path $atmosPath)) { + throw "atmos.exe not found at: $atmosPath" + } + echo "${{ github.workspace }}" >> $Env:GITHUB_PATH + + - uses: hashicorp/setup-terraform@v3 + with: + terraform_version: ${{ env.TERRAFORM_VERSION }} + terraform_wrapper: false + + - name: Check atmos.exe integrity + if: matrix.flavor.target == 'windows' + shell: pwsh + run: | + Write-Output "PATH=$Env:PATH" + Write-Output "PATHEXT=$Env:PATHEXT" + Get-ChildItem "${{ github.workspace }}" + Get-Command "${{ github.workspace }}\atmos.exe" + atmos version + - name: Set up Go uses: actions/setup-go@v5 with: @@ -279,26 +321,56 @@ jobs: timeout-minutes: 20 steps: - - name: Download build artifacts + - name: Check out code into the Go module directory + uses: actions/checkout@v4 + + - name: Add GNU tar to PATH (significantly faster than windows tar) + if: matrix.flavor.target == 'windows' + run: echo "C:\Program Files\Git\usr\bin" >> $Env:GITHUB_PATH + + - name: Download build artifacts for ${{ matrix.flavor.target }} uses: actions/download-artifact@v4 with: name: build-artifacts-${{ matrix.flavor.target }} - path: /usr/local/bin + path: ${{ github.workspace }} - - name: Set execute permissions on atmos - run: chmod +x /usr/local/bin/atmos + - name: Add build artifacts directory to PATH for linux or macos + if: matrix.flavor.target == 'linux' || matrix.flavor.target == 'macos' + run: | + echo "${{ github.workspace }}" >> $GITHUB_PATH + chmod +x "${{ github.workspace }}/atmos" - - name: Check out code into the Go module directory - uses: actions/checkout@v4 + - name: Add build artifacts directory to PATH for windows + if: matrix.flavor.target == 'windows' + run: | + echo "${{ github.workspace }}" >> $Env:GITHUB_PATH - uses: hashicorp/setup-terraform@v3 with: terraform_version: ${{ env.TERRAFORM_VERSION }} terraform_wrapper: false - - name: Run tests for ${{ matrix.demo-folder }} + - name: Run tests in ${{ matrix.demo-folder }} for ${{ matrix.flavor.target }} + working-directory: examples/${{ matrix.demo-folder }} + if: matrix.flavor.target == 'linux' || matrix.flavor.target == 'macos' + run: | + atmos test + + - name: Check atmos.exe integrity + if: matrix.flavor.target == 'windows' + shell: pwsh + run: | + Write-Output "PATH=$Env:PATH" + Write-Output "PATHEXT=$Env:PATHEXT" + Get-ChildItem "${{ github.workspace }}" + Get-Command "${{ github.workspace }}\atmos.exe" + atmos version + + - name: Run tests in ${{ matrix.demo-folder }} for ${{ matrix.flavor.target }} + working-directory: examples/${{ matrix.demo-folder }} + if: matrix.flavor.target == 'windows' + shell: pwsh run: | - cd examples/${{ matrix.demo-folder }} atmos test # run other demo tests @@ -352,7 +424,7 @@ jobs: --minimum-failure-severity=warning --recursive --config=${{ github.workspace }}/examples/.tflint.hcl - fail_on_error: true + fail_level: error # run other demo tests validate: diff --git a/tests/cli_test.go b/tests/cli_test.go new file mode 100644 index 000000000..255503cc8 --- /dev/null +++ b/tests/cli_test.go @@ -0,0 +1,250 @@ +package tests + +import ( + "bytes" + "errors" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" // For resolving absolute paths + "regexp" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +type Expectation struct { + Stdout []string `yaml:"stdout"` + Stderr []string `yaml:"stderr"` + ExitCode int `yaml:"exit_code"` + FileExists []string `yaml:"file_exists"` + FileContains map[string][]string `yaml:"file_contains"` +} + +type TestCase struct { + Name string `yaml:"name"` + Description string `yaml:"description"` + Enabled bool `yaml:"enabled"` + Workdir string `yaml:"workdir"` + Command string `yaml:"command"` + Args []string `yaml:"args"` + Env map[string]string `yaml:"env"` + Expect Expectation `yaml:"expect"` +} + +type TestSuite struct { + Tests []TestCase `yaml:"tests"` +} + +func loadTestSuite(filePath string) (*TestSuite, error) { + data, err := ioutil.ReadFile(filePath) + if err != nil { + return nil, err + } + + var suite TestSuite + err = yaml.Unmarshal(data, &suite) + if err != nil { + return nil, err + } + + return &suite, nil +} + +type PathManager struct { + OriginalPath string + Prepended []string +} + +// NewPathManager initializes a PathManager with the current PATH. +func NewPathManager() *PathManager { + return &PathManager{ + OriginalPath: os.Getenv("PATH"), + Prepended: []string{}, + } +} + +// Prepend adds directories to the PATH with precedence. +func (pm *PathManager) Prepend(dirs ...string) { + for _, dir := range dirs { + absPath, err := filepath.Abs(dir) + if err != nil { + fmt.Printf("Failed to resolve absolute path for %q: %v\n", dir, err) + continue + } + pm.Prepended = append(pm.Prepended, absPath) + } +} + +// GetPath returns the updated PATH. +func (pm *PathManager) GetPath() string { + return fmt.Sprintf("%s%c%s", + strings.Join(pm.Prepended, string(os.PathListSeparator)), + os.PathListSeparator, + pm.OriginalPath, + ) +} + +// Apply updates the PATH environment variable globally. +func (pm *PathManager) Apply() error { + return os.Setenv("PATH", pm.GetPath()) +} + +func TestCLICommands(t *testing.T) { + // Capture the starting working directory + startingDir, err := os.Getwd() + if err != nil { + t.Fatalf("Failed to get the current working directory: %v", err) + } + + // Initialize PathManager and update PATH + pathManager := NewPathManager() + pathManager.Prepend("../build", "..") + err = pathManager.Apply() + if err != nil { + t.Fatalf("Failed to apply updated PATH: %v", err) + } + fmt.Printf("Updated PATH: %s\n", pathManager.GetPath()) + + testSuite, err := loadTestSuite("test_cases.yaml") + if err != nil { + t.Fatalf("Failed to load test suite: %v", err) + } + + for _, tc := range testSuite.Tests { + + if !tc.Enabled { + t.Logf("Skipping disabled test: %s", tc.Name) + continue + } + + t.Run(tc.Name, func(t *testing.T) { + defer func() { + // Change back to the original working directory after the test + if err := os.Chdir(startingDir); err != nil { + t.Fatalf("Failed to change back to the starting directory: %v", err) + } + }() + + // Change to the specified working directory + if tc.Workdir != "" { + err := os.Chdir(tc.Workdir) + if err != nil { + t.Fatalf("Failed to change directory to %q: %v", tc.Workdir, err) + } + } + + // Check if the binary exists + binaryPath, err := exec.LookPath(tc.Command) + if err != nil { + t.Fatalf("Binary not found: %s. Current PATH: %s", tc.Command, pathManager.GetPath()) + } + + // Prepare the command + cmd := exec.Command(binaryPath, tc.Args...) + + // Set environment variables + envVars := os.Environ() + for key, value := range tc.Env { + envVars = append(envVars, fmt.Sprintf("%s=%s", key, value)) + } + cmd.Env = envVars + + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + // Run the command + err = cmd.Run() + + // Validate exit code + exitCode := 0 + if err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + exitCode = exitErr.ExitCode() + } + } + if exitCode != tc.Expect.ExitCode { + t.Errorf("Description: %s", tc.Description) + t.Errorf("Reason: Expected exit code %d, got %d", tc.Expect.ExitCode, exitCode) + } + + // Validate stdout + if !verifyOutput(t, "stdout", stdout.String(), tc.Expect.Stdout) { + t.Errorf("Description: %s", tc.Description) + } + + // Validate stderr + if !verifyOutput(t, "stderr", stderr.String(), tc.Expect.Stderr) { + t.Errorf("Description: %s", tc.Description) + } + + // Validate file existence + if !verifyFileExists(t, tc.Expect.FileExists) { + t.Errorf("Description: %s", tc.Description) + } + + // Validate file contents + if !verifyFileContains(t, tc.Expect.FileContains) { + t.Errorf("Description: %s", tc.Description) + } + }) + } +} + +func verifyOutput(t *testing.T, outputType, output string, patterns []string) bool { + success := true + for _, pattern := range patterns { + re, err := regexp.Compile(pattern) + if err != nil { + t.Errorf("Invalid %s regex: %q, error: %v", outputType, pattern, err) + success = false + continue + } + if !re.MatchString(output) { + t.Errorf("Reason: %s did not match pattern %q.", outputType, pattern) + t.Errorf("Output: %q", output) + success = false + } + } + return success +} + +func verifyFileExists(t *testing.T, files []string) bool { + success := true + for _, file := range files { + if _, err := os.Stat(file); errors.Is(err, os.ErrNotExist) { + t.Errorf("Reason: Expected file does not exist: %q", file) + success = false + } + } + return success +} + +func verifyFileContains(t *testing.T, filePatterns map[string][]string) bool { + success := true + for file, patterns := range filePatterns { + content, err := ioutil.ReadFile(file) + if err != nil { + t.Errorf("Reason: Failed to read file %q: %v", file, err) + success = false + continue + } + for _, pattern := range patterns { + re, err := regexp.Compile(pattern) + if err != nil { + t.Errorf("Invalid regex for file %q: %q, error: %v", file, pattern, err) + success = false + continue + } + if !re.Match(content) { + t.Errorf("Reason: File %q did not match pattern %q.", file, pattern) + t.Errorf("Content: %q", string(content)) + success = false + } + } + } + return success +} diff --git a/tests/test_cases.yaml b/tests/test_cases.yaml new file mode 100644 index 000000000..9ebb6641f --- /dev/null +++ b/tests/test_cases.yaml @@ -0,0 +1,170 @@ +tests: + - name: "which atmos" + enabled: true + description: "Ensure atmos CLI is installed and we're using the one that was built." + workdir: "../" + command: "which" + args: + - "atmos" + expect: + stdout: + - '(build[/\\]atmos|atmos[/\\]atmos)' + stderr: + - "^$" + exit_code: 0 + + - name: "atmos" + enabled: true + description: "Verify atmos CLI reports missing stacks directory." + workdir: "../" + command: "atmos" + expect: + stdout: + - "atmos.yaml CLI config file specifies the directory for Atmos stacks as stacks," + - "but the directory does not exist." + stderr: + - "^$" + exit_code: 0 + + - name: atmos --help + enabled: true + description: "Ensure atmos CLI help command lists available commands." + workdir: "../examples/demo-stacks" + command: "atmos" + args: + - "--help" + expect: + stdout: + - "Available Commands:" + - "\\batlantis\\b" + - "\\baws\\b" + - "\\bcompletion\\b" + - "\\bdescribe\\b" + - "\\bdocs\\b" + - "\\bhelmfile\\b" + - "\\bhelp\\b" + - "\\blist\\b" + - "\\bpro\\b" + - "\\bterraform\\b" + - "\\bvalidate\\b" + - "\\bvendor\\b" + - "\\bversion\\b" + - "\\bworkflow\\b" + - "Flags:" + - "for more information about a command" + stderr: + - "^$" + exit_code: 0 + + - name: atmos version + enabled: true + description: "Check that atmos version command outputs version details." + workdir: "../examples/demo-stacks" + command: "atmos" + args: + - "version" + expect: + stdout: + - '👽 Atmos (\d+\.\d+\.\d+|test) on [a-z]+/[a-z0-9]+' + stderr: + - "^$" + exit_code: 0 + + - name: atmos version --check + enabled: true + description: "Verify atmos version --check command functions correctly." + workdir: "../examples/demo-stacks" + command: "atmos" + args: + - "version" + - "--check" + expect: + stdout: + - '👽 Atmos (\d+\.\d+\.\d+|test) on [a-z]+/[a-z0-9]+' + stderr: + - "^$" + exit_code: 0 + + - name: atmos docs + enabled: false + description: "Ensure atmos docs command executes without errors." + workdir: "../" + command: "atmos" + args: + - "docs" + expect: + exit_code: 0 + stderr: + - "^$" + + - name: atmos docs myapp + enabled: true + description: "Validate atmos docs command outputs documentation for a specific component." + workdir: "../examples/demo-stacks/" + command: "atmos" + args: + - "docs" + - "myapp" + expect: + stdout: + - "Example Terraform Weather Component" + stderr: + - "^$" + exit_code: 0 + + - name: atmos non-existent + enabled: true + description: "Ensure atmos CLI returns an error for a non-existent command." + workdir: "../" + command: "atmos" + args: + - "non-existent" + expect: + stderr: + - 'unknown command "non-existent" for "atmos"' + exit_code: 1 + + - name: atmos terraform non-existent + enabled: false + description: "Ensure atmos CLI returns an error for a non-existent command." + workdir: "../" + command: "atmos" + args: + - "terraform" + - "non-existent" + expect: + stderr: + - 'unknown command "non-existent" for "atmos"' + exit_code: 1 + + - name: atmos describe config -f yaml + enabled: true + description: "Ensure atmos CLI outputs the Atmos configuration in YAML." + workdir: "../examples/demo-stacks/" + command: "atmos" + args: + - "describe" + - "config" + - "-f" + - "yaml" + expect: + stdout: + - 'append_user_agent: Atmos/(\d+\.\d+\.\d+|test) \(Cloud Posse; \+https:\/\/atmos\.tools\)' + stderr: + - "^$" + exit_code: 0 + + - name: atmos describe config + enabled: true + description: "Ensure atmos CLI outputs the Atmos configuration in JSON." + workdir: "../examples/demo-stacks/" + command: "atmos" + args: + - "describe" + - "config" + expect: + stdout: + - '"append_user_agent": "Atmos/(\d+\.\d+\.\d+|test) \(Cloud Posse; \+https:\/\/atmos\.tools\)"' + stderr: + - "^$" + exit_code: 0