Merge pull request #58 from davidkl97/fix/external-credentials

refactor(external-git): retrieve credentials from credentials helper

.github/workflows/golang-build-test-coverage.yml

@@ -4,7 +4,7 @@ on:
      branches-ignore:
       - main
   pull_request:
-    branches:    
+    branches:
       - main
   release:
 
@@ -93,6 +93,10 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
+      - name: Setup credentials
+        uses: fusion-engineering/setup-git-credentials@v2
+        with:
+          credentials: https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com
       - name: Setup go
         uses: actions/setup-go@v3
         with:

README.md

@@ -80,7 +80,7 @@ in order to activate this feature all you have to do is to set `fetch-remote` to
 ``` bash
 terra-crust terraform-all  --main-template-path=./terraform/main.tmpl  --destination-path="." --source-path=".../modules" --fetch-remote=true
 ```  
-or with an external git client with the flag `--ext-git` to true like so:  
+or with an external git credential with the flag `--ext-git` to true like so:  
 ``` bash
 terra-crust terraform-all  --main-template-path=./terraform/main.tmpl  --destination-path="." --source-path=".../modules" --ext-git=true
 ```

internal/services/drivers/version_control/git.go

@@ -1,29 +1,36 @@
 package version_control
 
 import (
+	"bytes"
+	"context"
 	"fmt"
-	"github.com/go-git/go-git/v5/plumbing"
 	"os"
 	"os/exec"
 	"strings"
+	"time"
 
 	log "github.com/AppsFlyer/go-logger"
 	"github.com/go-git/go-git/v5" /// with go modules disabled
+	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/go-git/go-git/v5/plumbing/transport/http"
 	cp "github.com/otiai10/copy"
 	"github.com/pkg/errors"
 )
 
 const (
-	FolderPathFormat = "%s/%s"
-	TempFolderPath   = "%s/temp_clone_path/%s"
-	remoteName       = "origin"
-	GitlabTokenENV   = "GITLAB_TOKEN"
-	GitRefTag        = "refs/tags/%s"
-	GitRefBranch     = "refs/remotes/%s/%s"
-	GitlabUserENV    = "GITLAB_USER"
-	GithubTokenENV   = "GITHUB_TOKEN"
-	GithubUserENV    = "GITHUB_USER"
+	FolderPathFormat            = "%s/%s"
+	TempFolderPath              = "%s/temp_clone_path/%s"
+	remoteName                  = "origin"
+	GitlabTokenENV              = "GITLAB_TOKEN"
+	GitRefTag                   = "refs/tags/%s"
+	GitRefBranch                = "refs/remotes/%s/%s"
+	GitCredentialUrl            = "url=%s"
+	GitCredentialDeadLineMs     = 500
+	GitCredentialUserNamePrefix = "username="
+	GitCredentialPasswordPrefix = "password="
+	GitlabUserENV               = "GITLAB_USER"
+	GithubTokenENV              = "GITHUB_TOKEN"
+	GithubUserENV               = "GITHUB_USER"
 )
 
 type RemoteModule struct {
@@ -78,25 +85,20 @@ func (g *Git) CloneModules(modules map[string]*RemoteModule, modulesSource strin
 
 func (g *Git) clone(moduleData *RemoteModule, directoryPath string, externalGit bool) error {
 
-	if externalGit {
-		args := []string{"clone", moduleData.Url, directoryPath, "--no-tags", "--single-branch", "--depth", "1", "-o", remoteName}
-		if moduleData.Version != "" {
-			args = append(args, "--branch", moduleData.Version)
-		}
-		err := exec.Command("git", args...).Run()
+	userName, token, err := g.getGitCredentials(moduleData.Url, externalGit)
+	if err != nil {
 		return err
 	}
-	userName, token := g.getGitUserNameAndToken(moduleData.Url)
 
-	cloneOpts := git.CloneOptions{
+	repo, err := git.PlainClone(directoryPath, false, &git.CloneOptions{
 		URL:        moduleData.Url,
 		Auth:       &http.BasicAuth{Password: token, Username: userName},
 		RemoteName: remoteName,
 		Depth:      1,
+	})
+	if err != nil {
+		return err
 	}
-
-	repo, err := git.PlainClone(directoryPath, false, &cloneOpts)
-
 	if moduleData.Version != "" {
 		workTree, err := repo.Worktree()
 		if err != nil {
@@ -116,7 +118,7 @@ func (g *Git) clone(moduleData *RemoteModule, directoryPath string, externalGit
 			return bErr
 		}
 	}
-	return err
+	return nil
 }
 
 func (g *Git) CleanModulesFolders(modules map[string]*RemoteModule, modulesSource string) error {
@@ -153,6 +155,33 @@ func (g *Git) cleanTemp(modulesSourcePath string) error {
 	return nil
 }
 
+func (g *Git) getGitCredentials(url string, externalGit bool) (userName string, password string, err error) {
+	if !externalGit {
+		userName, password = g.getGitUserNameAndToken(url)
+		return userName, password, nil
+	}
+	// Required until https://github.com/go-git/go-git/issues/490 addressed
+	ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(GitCredentialDeadLineMs*time.Millisecond))
+	cmd := exec.CommandContext(ctx, "git", "credential", "fill")
+	defer cancel()
+	cmd.Stdin = strings.NewReader(fmt.Sprintf(GitCredentialUrl, url))
+	var out bytes.Buffer
+	cmd.Stdout = &out
+	err = cmd.Run()
+	if err != nil {
+		return userName, password, err
+	}
+	lines := strings.Split(out.String(), "\n")
+	for _, line := range lines {
+		if strings.HasPrefix(line, GitCredentialUserNamePrefix) {
+			userName = strings.TrimPrefix(line, GitCredentialUserNamePrefix)
+		}
+		if strings.HasPrefix(line, GitCredentialPasswordPrefix) {
+			password = strings.TrimPrefix(line, GitCredentialPasswordPrefix)
+		}
+	}
+	return userName, password, nil
+}
 func (g *Git) getGitUserNameAndToken(url string) (string, string) {
 	if strings.Contains(url, "gitlab") {
 		return os.Getenv(GitlabUserENV), os.Getenv(GitlabTokenENV)

internal/services/drivers/version_control/git_test.go

@@ -16,10 +16,7 @@ const (
 	TerraCrustURL        = "https://github.com/AppsFlyer/terra-crust.git"
 )
 
-var ModulesTestPath = map[bool]string{
-	true:  "./ext-temp-git-test",
-	false: "./int-temp-git-test",
-}
+var ModulesTestPath = "./temp-git-test"
 
 var mockBadUrl = map[string]*version_control.RemoteModule{
 	TerraCrustModuleName: {
@@ -34,6 +31,13 @@ var mockBadVersion = map[string]*version_control.RemoteModule{
 	},
 }
 
+var mockBadRemote = map[string]*version_control.RemoteModule{
+	NamingModuleName: {
+		Url:     "https://gitbad.com/fajrinazis/terraform-aws-resource-naming.git",
+		Version: "bad-tag",
+	},
+}
+
 func TestCloneAndCleanupInternalGit(t *testing.T) {
 	CloneAndCleanup(t, false)
 }
@@ -47,8 +51,8 @@ func CloneAndCleanupModules(modules map[string]*version_control.RemoteModule, ex
 
 	gitDriver := version_control.InitGitProvider(log)
 
-	err := gitDriver.CloneModules(modules, ModulesTestPath[externalGit], externalGit)
-	cErr := gitDriver.CleanModulesFolders(modules, ModulesTestPath[externalGit])
+	err := gitDriver.CloneModules(modules, ModulesTestPath, externalGit)
+	cErr := gitDriver.CleanModulesFolders(modules, ModulesTestPath)
 	if err != nil {
 		if cErr != nil {
 			return fmt.Errorf("failed to clone and cleanup module %v %v", err, cErr)
@@ -75,12 +79,12 @@ func CloneAndCleanup(t *testing.T, externalGit bool) {
 
 	gitDriver := version_control.InitGitProvider(log)
 
-	err := gitDriver.CloneModules(mockModules, ModulesTestPath[externalGit], externalGit)
+	err := gitDriver.CloneModules(mockModules, ModulesTestPath, externalGit)
 	if err != nil {
 		t.Errorf(err.Error())
 	}
 
-	folders, err := getFolderAsMap(ModulesTestPath[externalGit])
+	folders, err := getFolderAsMap(ModulesTestPath)
 	if err != nil {
 		t.Errorf(err.Error())
 	}
@@ -101,11 +105,11 @@ func CloneAndCleanup(t *testing.T, externalGit bool) {
 		t.Errorf("Expected 3 folder count received %d", len(folders))
 	}
 
-	if err = gitDriver.CleanModulesFolders(mockModules, ModulesTestPath[externalGit]); err != nil {
+	if err = gitDriver.CleanModulesFolders(mockModules, ModulesTestPath); err != nil {
 		t.Errorf("failed to clean up the downloaded modules,  %s", err.Error())
 	}
 
-	folders, err = getFolderAsMap(ModulesTestPath[externalGit])
+	folders, err = getFolderAsMap(ModulesTestPath)
 	if err != nil {
 		t.Errorf(err.Error())
 	}
@@ -143,6 +147,14 @@ func TestFailBadVersionExternalGit(t *testing.T) {
 	FailBadVersion(t, true)
 }
 
+func TestFailBadRemoteInternalGit(t *testing.T) {
+	FailGetGitCredentials(t, false)
+}
+
+func TestFailBadRemoteExternalGit(t *testing.T) {
+	FailGetGitCredentials(t, true)
+}
+
 func FailBadUrl(t *testing.T, externalGit bool) {
 	t.Parallel()
 
@@ -161,6 +173,15 @@ func FailBadVersion(t *testing.T, externalGit bool) {
 	}
 }
 
+func FailGetGitCredentials(t *testing.T, externalGit bool) {
+	log := logger.NewSimple()
+	gitDriver := version_control.InitGitProvider(log)
+	err := gitDriver.CloneModules(mockBadRemote, ModulesTestPath, externalGit)
+	if err == nil {
+		t.Errorf("expected error received error nil")
+	}
+}
+
 func getFolderAsMap(path string) (map[string]struct{}, error) {
 	folders := make(map[string]struct{})