Implementing git-clone target

.github/workflows/test.yml

@@ -16,6 +16,11 @@ jobs:
   test:
     name: Test
     runs-on: ubuntu-latest
+    services:
+      dind:
+        image: docker:23.0-rc-dind-rootless
+        ports:
+          - 2375:2375
     concurrency:
       group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-test
       cancel-in-progress: true

.gitignore

@@ -9,3 +9,4 @@ tests/output
 deploy/smithy/chart/charts
 .idea/
 components/consumers/pdf/report.html
+new-components/targets/git-clone/pkg/git/testdata/gitea

Makefile

@@ -9,6 +9,9 @@ go_protos=$(protos:.proto=.pb.go)
 latest_tag=$(shell git tag --list --sort="-version:refname" | head -n 1)
 commits_since_latest_tag=$(shell git log --oneline $(latest_tag)..HEAD | wc -l)
 GO_TEST_PACKAGES=$(shell go list ./... | grep -v /vendor/)
+VENDOR_DIRS=$(shell find . -type d -name "vendor")
+NOT_VENDOR_PATHS := $(shell echo $(VENDOR_DIRS) | awk '{for (i=1; i<=NF; i++) print "-not -path \""$$i"/*\""}' | tr '\n' ' ')
+EXCLUDE_VENDOR_PATHS := $(shell echo $(VENDOR_DIRS) | awk '{for (i=1; i<=NF; i++) print "--exclude-path \""$$i"\""}' | tr '\n' ' ')
 
 # Deployment vars
 # The following variables are used to define the deployment environment
@@ -98,7 +101,7 @@ publish-component-containers: $(component_containers_publish)
 publish-containers: publish-component-containers smithyctl-image-publish
 
 clean-protos:
-	@find . -not -path './vendor/*' -name '*.pb.go' -delete
+	@find . $(NOT_VENDOR_PATHS) -name '*.pb.go' -delete
 
 clean-migrations-compose:
 	cd tests/migrations/ && docker compose rm --force
@@ -145,9 +148,9 @@ install-go-fmt-tools:
 	@go install golang.org/x/tools/cmd/goimports@latest
 
 fmt-go:
-	@echo "Tidying up Go files"
-	@gofmt -l -w $$(find . -name *.go -not -path "**/vendor/*" | xargs -n 1 dirname | uniq)
-	@goimports -local $$(cat go.mod | grep -E "^module" | sed 's/module //') -w $$(find . -name *.go -not -path "**/vendor/*" | xargs -n 1 dirname | uniq)
+	echo "Tidying up Go files"
+	$(shell find . -type f -name "*.go" -not -name "*.pb.*" $(NOT_VENDOR_PATHS) | xargs gofmt -w | uniq)
+	@goimports -local $$(cat go.mod | grep -E "^module" | sed 's/module //') -w $$(find . -type f -name *.go -not -name "*.pb.*" $(NOT_VENDOR_PATHS) | xargs -n 1 dirname | uniq)
 
 install-md-fmt-tools:
 	@npm ci
@@ -327,11 +330,11 @@ run-buf: build-buf-container
 
 fmt-proto: build-buf-container
 	@echo "Tidying up Proto files"
-	$(MAKE) run-buf ARGS="format -w --exclude-path vendor/"
+	$(MAKE) run-buf ARGS="format -w $(EXCLUDE_VENDOR_PATHS)"
 
 lint-proto: build-buf-container
 	@echo "Linting Proto files"
-	$(MAKE) run-buf ARGS="lint --exclude-path vendor/"
+	$(MAKE) run-buf ARGS="lint $(EXCLUDE_VENDOR_PATHS)"
 
 # Buf doesn't have a way to configure where to put the generated code from remote repositories;
 # that's why we make sure to correctly move it around. In this case it's just COM for OCSF types.

containers/Dockerfile.buf

@@ -8,7 +8,7 @@ RUN go install google.golang.org/protobuf/cmd/[email protected]
 
 # Install git and its dependencies to enable cloning as we generate protos from remote for OCSF.
 FROM alpine AS git
-RUN apk --no-cache add git ca-certificates openssl
+RUN apk --no-cache add git ca-certificates openssl diffutils
 RUN mkdir -p /git-deps/bin /git-deps/lib /git-deps/etc/ssl/certs
 RUN cp /usr/bin/git /git-deps/bin
 RUN cp /usr/libexec/git-core/git-remote-https /git-deps/bin/
@@ -25,6 +25,7 @@ COPY --from=git /git-deps/bin/git /usr/bin/
 COPY --from=git /git-deps/bin/git-remote-https /usr/bin/
 COPY --from=git /git-deps/lib /lib/
 COPY --from=git /git-deps/etc/ssl/certs /etc/ssl/certs/
+COPY --from=git /usr/bin/diff /usr/bin/diff
 ENV GIT_SSL_CAINFO=/etc/ssl/certs/ca-certificates.crt
 ENV PATH="/go/bin:/usr/bin:${PATH}"
 ENTRYPOINT ["/go/bin/buf"]

new-components/targets/git-clone/.env.git-clone

@@ -0,0 +1,6 @@
+SMITHY_COMPONENT_NAME=git-clone
+SMITHY_INSTANCE_ID=8d719c1c-c569-4078-87b3-4951bd4012ee
+SMITHY_LOG_LEVEL=debug
+SMITHY_BACKEND_STORE_TYPE=local
+GIT_CLONE_REPO_URL=http://gitea:3000/root/testrepo.git
+GIT_CLONE_REFERENCE=main

new-components/targets/git-clone/.env.gitea

@@ -0,0 +1,8 @@
+USER_UID=1000
+USER_GID=1000
+GITEA__database__DB_TYPE=sqlite3
+GITEA__database__PATH=/data/gitea/gitea.db
+GITEA__server__ROOT_URL=http://localhost:3000
+GITEA__admin__username=root
+GITEA__admin__password=root
+[email protected]

new-components/targets/git-clone/Makefile

@@ -0,0 +1,24 @@
+.PHONY: run run-with-seeder shutdown build-target
+
+run:
+	docker-compose up --build --force-recreate --remove-orphans gitea target
+
+run-with-seeder:
+	docker-compose up --build --force-recreate --remove-orphans
+
+shutdown:
+	docker-compose down --rmi all
+
+build-target:
+	# Security hardening and building flags for minimal binaries.
+    #
+    # These CGO_CPPFLAGS help preventing overflows.
+    # Add a small overhead at compile time.
+	CGO_CPPFLAGS="-D_FORTIFY_SOURCE=2 -fstack-protector-all" \
+	# Makes memory exploitation harder.
+    # Add a small overhead at compile time.
+	GOFLAGS="-buildmode=pie" \
+	CGO_ENABLED=0 \
+	GOOS="linux" \
+	GOARCH="amd64" \
+		go build -ldflags "-s -w" -trimpath -o target ./cmd/git-clone/main.go

new-components/targets/git-clone/README.md

@@ -0,0 +1,51 @@
+# git-clone
+
+This is component implements a [target](https://github.com/smithy-security/smithy/blob/main/sdk/component/component.go#L55)
+that clones repositories to the filesystem.
+
+## Environment variables
+
+The component used environment variables for configuration.
+
+It requires the component
+environment variables defined [here](https://github.com/smithy-security/smithy/blob/main/sdk/README.md#component) as well
+as the following:
+
+| Environment Variable      | Type   | Required | Default           | Description                                        |
+|---------------------------|--------|----------|-------------------|----------------------------------------------------|
+| GIT\_CLONE\_PATH            | string | no       | Current directory | The path where to clone repositories to            |
+| GIT\_CLONE\_REPO\_URL        | string | yes      | -                 | Valid URL of the repository to clone               |
+| GIT\_CLONE\_BRANCH\_NAME     | string | yes      | -                 | Valid branch name of the repository to clone       |
+| GIT\_CLONE\_AUTH\_ENABLED    | bool   | no       | false             | Whether authentication should be used for VCS      |
+| GIT\_CLONE\_ACCESS\_TOKEN    | string | no       | -                 | Access token to be leveraged for authentication    |
+| GIT\_CLONE\_ACCESS\_USERNAME | string | no       | -                 | Access username to be leveraged for authentication |
+
+## build git-clone
+
+```shell
+make build-target
+```
+
+## How to run
+
+## git-clone and gitea
+
+Spins up [gitea](https://about.gitea.com/) locally and the `git-clone` component.
+
+```shell
+make run
+```
+
+## git-clone, gitea and seeder
+
+Like above but also seeding `gitea` with a sample repository that `git-clone` can clone out of the box.
+
+```shell
+make run-with-seeder
+```
+
+## shutdown
+
+```shell
+make shutdown
+```

new-components/targets/git-clone/cmd/git-clone/main.go

@@ -0,0 +1,44 @@
+package main
+
+import (
+	"context"
+	"log"
+	"time"
+
+	"github.com/go-errors/errors"
+
+	"github.com/smithy-security/smithy/sdk/component"
+
+	"github.com/smithy-security/smithy/new-components/targets/git-clone/pkg/git"
+)
+
+func main() {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	defer cancel()
+
+	if err := Main(ctx); err != nil {
+		log.Fatalf("unexpected error: %v", err)
+	}
+}
+
+func Main(ctx context.Context) error {
+	conf, err := git.NewConf(nil)
+	if err != nil {
+		return errors.Errorf("could not create new configuration: %w", err)
+	}
+
+	gitCloneTarget, err := git.NewTarget(conf)
+	if err != nil {
+		return errors.Errorf("could not create git clone target: %w", err)
+	}
+
+	if err := component.RunTarget(
+		ctx,
+		gitCloneTarget,
+		component.RunnerWithComponentName("git-clone"),
+	); err != nil {
+		return errors.Errorf("could not run target: %w", err)
+	}
+
+	return nil
+}

new-components/targets/git-clone/cmd/seeder/main.go

@@ -0,0 +1,175 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/smithy-security/pkg/env"
+)
+
+func main() {
+	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
+	defer cancel()
+
+	if err := Main(ctx); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func Main(ctx context.Context) error {
+	giteaURL, err := env.GetOrDefault("GITEA_URL", "http://localhost:3000")
+	if err != nil {
+		return err
+	}
+	user, err := env.GetOrDefault("GITEA_ADMIN_USER", "gitcloner")
+	if err != nil {
+		return err
+	}
+	pass, err := env.GetOrDefault("GITEA_ADMIN_PASSWORD", "smithy1234")
+	if err != nil {
+		return err
+	}
+	repo, err := env.GetOrDefault("GIT_REPO_NAME", "testrepo")
+	if err != nil {
+		return err
+	}
+
+	if err := createUser(ctx, giteaURL, user, pass); err != nil {
+		return err
+	}
+
+	token, err := createToken(ctx, giteaURL, user, pass)
+	if err != nil {
+		return err
+	}
+
+	if err := createRepo(ctx, giteaURL, repo, token); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func createUser(ctx context.Context, giteaURL string, user, pass string) error {
+	createUserURL, err := url.Parse(fmt.Sprintf("%s/user/sign_up", giteaURL))
+	if err != nil {
+		return fmt.Errorf("failed to parse get token URL: %w", err)
+	}
+
+	payload := strings.NewReader(
+		fmt.Sprintf(
+			`user_name=%s&email=%[email protected]&password=%s&retype=%s`,
+			user,
+			user,
+			pass,
+			pass,
+		),
+	)
+
+	req, err := http.NewRequestWithContext(
+		ctx,
+		http.MethodPost,
+		createUserURL.String(),
+		payload,
+	)
+	if err != nil {
+		return fmt.Errorf("failed to create new request: %w", err)
+	}
+
+	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to do request: %w", err)
+	}
+
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("failed to do create user request: status %d", resp.StatusCode)
+	}
+
+	return nil
+}
+
+func createToken(ctx context.Context, giteaURL, user, pass string) (string, error) {
+	getTokenURL, err := url.Parse(fmt.Sprintf("%s/api/v1/users/%s/tokens", giteaURL, user))
+	if err != nil {
+		return "", fmt.Errorf("failed to parse get token URL: %w", err)
+	}
+
+	req, err := http.NewRequestWithContext(
+		ctx,
+		http.MethodPost,
+		getTokenURL.String(),
+		bytes.NewBuffer([]byte(`{"name": "all-scopes-token", "scopes": ["all"]}`)),
+	)
+	if err != nil {
+		return "", fmt.Errorf("failed to create new request: %w", err)
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+	req.SetBasicAuth(user, pass)
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("failed to do request: %w", err)
+	}
+
+	defer resp.Body.Close()
+
+	var tokenRes map[string]any
+	if err := json.NewDecoder(resp.Body).Decode(&tokenRes); err != nil {
+		return "", fmt.Errorf("failed to decode response body: %w", err)
+	}
+
+	if resp.StatusCode != http.StatusCreated {
+		return "", fmt.Errorf("failed to do create token request: status %d", resp.StatusCode)
+	}
+
+	token, ok := tokenRes["sha1"]
+	if !ok {
+		return "", fmt.Errorf("failed to find token sha1 in response: %v", tokenRes)
+	}
+
+	return token.(string), nil
+}
+
+func createRepo(ctx context.Context, giteaURL, repoName, token string) error {
+	createRepoURL, err := url.Parse(fmt.Sprintf("%s/api/v1/user/repos", giteaURL))
+	if err != nil {
+		return fmt.Errorf("failed to parse get token URL: %w", err)
+	}
+
+	req, err := http.NewRequestWithContext(
+		ctx,
+		http.MethodPost,
+		createRepoURL.String(),
+		bytes.NewBuffer([]byte(fmt.Sprintf(`{"name": "%s", "private": false}`, repoName))),
+	)
+	if err != nil {
+		return fmt.Errorf("failed to create new request: %w", err)
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Authorization", fmt.Sprintf("token %s", token))
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to do create repo request: %w", err)
+	}
+
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusCreated {
+		return fmt.Errorf("failed to do request: status %d", resp.StatusCode)
+	}
+
+	return nil
+}