From d66c92000f4508e628b8de13fdbaf15bca733859 Mon Sep 17 00:00:00 2001 From: dlaw4608 Date: Wed, 8 May 2024 17:13:55 +0100 Subject: [PATCH] Includes quay_overflow.go file and quay_overflow_test.go Signed-off-by: dlaw4608 --- .github/workflows/build-images-base.yaml | 116 ++++++------- Dockerfile | 7 +- go.mod | 4 + go.sum | 4 + quay/quay_overflow.go | 168 ++++++++++++++++++ quay/quay_overflow_test.go | 208 +++++++++++++++++++++++ 6 files changed, 439 insertions(+), 68 deletions(-) create mode 100644 quay/quay_overflow.go create mode 100644 quay/quay_overflow_test.go diff --git a/.github/workflows/build-images-base.yaml b/.github/workflows/build-images-base.yaml index 38bad8572..57ee556d6 100644 --- a/.github/workflows/build-images-base.yaml +++ b/.github/workflows/build-images-base.yaml @@ -85,31 +85,26 @@ jobs: steps: - name: Check out code uses: actions/checkout@v3 - - name: Install qemu dependency - run: | - sudo apt-get update - sudo apt-get install -y qemu-user-static - - name: Build Image - id: build-image - uses: redhat-actions/buildah-build@v2 - with: - image: ${{ env.OPERATOR_NAME }} - tags: ${{ env.IMG_TAGS }} - platforms: linux/amd64,linux/arm64 - dockerfiles: | - ./Dockerfile - - name: Push Image - if: ${{ !env.ACT }} - id: push-to-quay - uses: redhat-actions/push-to-registry@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to container registry + uses: docker/login-action@v2 with: - image: ${{ steps.build-image.outputs.image }} - tags: ${{ steps.build-image.outputs.tags }} - registry: ${{ env.IMG_REGISTRY_HOST }}/${{ env.IMG_REGISTRY_ORG }} username: ${{ secrets.IMG_REGISTRY_USERNAME }} password: ${{ secrets.IMG_REGISTRY_TOKEN }} + registry: ${{ env.IMG_REGISTRY_HOST }} + - name: Build and Push Image + id: build-image + uses: docker/build-push-action@v5 + with: + context: . + file: ./Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: | + ${{ env.IMG_REGISTRY_HOST }}/${{ env.IMG_REGISTRY_ORG }}/${{ env.OPERATOR_NAME }}:${{ env.IMG_TAGS }} - name: Print Image URL - run: echo "Image pushed to ${{ steps.push-to-quay.outputs.registry-paths }}" + run: echo "Image pushed to ${{ env.IMG_REGISTRY_HOST }}/${{ env.IMG_REGISTRY_ORG }}/${{ env.OPERATOR_NAME }}:${{ env.IMG_TAGS }}" build-bundle: needs: build @@ -123,10 +118,6 @@ jobs: id: go - name: Check out code uses: actions/checkout@v3 - - name: Install qemu dependency - run: | - sudo apt-get update - sudo apt-get install -y qemu-user-static - name: Run make bundle id: make-bundle run: | @@ -138,27 +129,26 @@ jobs: WASM_SHIM_VERSION=${{ inputs.wasmShimVersion }} \ REPLACES_VERSION=${{ inputs.replacesVersion }} \ CHANNELS=${{ inputs.channels }} - - name: Build Image - id: build-image - uses: redhat-actions/buildah-build@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to container registry + uses: docker/login-action@v2 with: - image: ${{ env.OPERATOR_NAME }}-bundle - tags: ${{ env.IMG_TAGS }} - platforms: linux/amd64,linux/arm64 - dockerfiles: | - ./bundle.Dockerfile - - name: Push Image - if: ${{ !env.ACT }} - id: push-to-quay - uses: redhat-actions/push-to-registry@v2 - with: - image: ${{ steps.build-image.outputs.image }} - tags: ${{ steps.build-image.outputs.tags }} - registry: ${{ env.IMG_REGISTRY_HOST }}/${{ env.IMG_REGISTRY_ORG }} username: ${{ secrets.IMG_REGISTRY_USERNAME }} password: ${{ secrets.IMG_REGISTRY_TOKEN }} - - name: Print Image URL - run: echo "Image pushed to ${{ steps.push-to-quay.outputs.registry-paths }}" + registry: ${{ env.IMG_REGISTRY_HOST }} + - name: Build and Push Bundle Image + id: build-bundle-image + uses: docker/build-push-action@v5 + with: + context: . + file: ./bundle.Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: | + ${{ env.IMG_REGISTRY_HOST }}/${{ env.IMG_REGISTRY_ORG }}/${{ env.OPERATOR_NAME }}-bundle:${{ env.IMG_TAGS }} + - name: Print Bundle Image URL + run: echo "Bundle image pushed to ${{ env.IMG_REGISTRY_HOST }}/${{ env.IMG_REGISTRY_ORG }}/${{ env.OPERATOR_NAME }}-bundle:${{ env.IMG_TAGS }}" build-catalog: name: Build Catalog @@ -182,29 +172,23 @@ jobs: WASM_SHIM_VERSION=${{ inputs.wasmShimVersion }} \ REPLACES_VERSION=${{ inputs.replacesVersion }} \ CHANNELS=${{ inputs.channels }} - - name: Install qemu dependency - run: | - sudo apt-get update - sudo apt-get install -y qemu-user-static - - name: Build Image - id: build-image - uses: redhat-actions/buildah-build@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to container registry + uses: docker/login-action@v2 with: - image: ${{ env.OPERATOR_NAME }}-catalog - tags: ${{ env.IMG_TAGS }} - platforms: linux/amd64,linux/arm64 - context: ./catalog - dockerfiles: | - ./catalog/kuadrant-operator-catalog.Dockerfile - - name: Push Image - if: ${{ !env.ACT }} - id: push-to-quay - uses: redhat-actions/push-to-registry@v2 - with: - image: ${{ steps.build-image.outputs.image }} - tags: ${{ steps.build-image.outputs.tags }} - registry: ${{ env.IMG_REGISTRY_HOST }}/${{ env.IMG_REGISTRY_ORG }} username: ${{ secrets.IMG_REGISTRY_USERNAME }} password: ${{ secrets.IMG_REGISTRY_TOKEN }} - - name: Print Image URL - run: echo "Image pushed to ${{ steps.push-to-quay.outputs.registry-paths }}" + registry: ${{ env.IMG_REGISTRY_HOST }} + - name: Build and Push Catalog Image + id: build-catalog-image + uses: docker/build-push-action@v5 + with: + context: ./catalog + file: ./catalog/kuadrant-operator-catalog.Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: | + ${{ env.IMG_REGISTRY_HOST }}/${{ env.IMG_REGISTRY_ORG }}/${{ env.OPERATOR_NAME }}-catalog:${{ env.IMG_TAGS }} + - name: Print Catalog Image URL + run: echo "Catalog image pushed to ${{ env.IMG_REGISTRY_HOST }}/${{ env.IMG_REGISTRY_ORG }}/${{ env.OPERATOR_NAME }}-catalog:${{ env.IMG_TAGS }}" \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 00d477eb6..b2e105887 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM golang:1.22 as builder +FROM --platform=$BUILDPLATFORM golang:1.22 as builder WORKDIR /workspace # Copy the Go Modules manifests @@ -15,8 +15,11 @@ COPY api/ api/ COPY controllers/ controllers/ COPY pkg/ pkg/ +# Set environment variables for cross-compilation +ARG TARGETARCH + # Build -RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go +RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -a -o manager main.go # Use distroless as minimal base image to package the manager binary # Refer to https://github.com/GoogleContainerTools/distroless for more details diff --git a/go.mod b/go.mod index c818f9c7d..b3bf09cbe 100644 --- a/go.mod +++ b/go.mod @@ -123,6 +123,7 @@ require ( github.com/opencontainers/image-spec v1.1.0-rc5 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.48.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect @@ -134,9 +135,12 @@ require ( github.com/spf13/cast v1.6.0 // indirect github.com/spf13/cobra v1.8.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/stretchr/objx v0.5.2 // indirect + github.com/stretchr/testify v1.9.0 // indirect github.com/tidwall/gjson v1.14.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect + github.com/vitorsalgado/mocha/v3 v3.0.2 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect diff --git a/go.sum b/go.sum index 8aa8204b0..f49668829 100644 --- a/go.sum +++ b/go.sum @@ -400,6 +400,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.5.1 h1:4VhoImhV/Bm0ToFkXFi8hXNXwpDRZ/ynw3amt82mzq0= github.com/stretchr/objx v0.5.1/go.mod h1:/iHQpkQwBD6DLUmQ4pE+s1TXdob1mORJ4/UFdrifcy0= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= @@ -413,6 +415,8 @@ github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/vitorsalgado/mocha/v3 v3.0.2 h1:uTx/+7kZvTWddXzoF34vUQTa3OL9OE+f5fPjD2XCMoY= +github.com/vitorsalgado/mocha/v3 v3.0.2/go.mod h1:ZMpyjuNfWPqLP2v7ztaaLJwOcyl4jmmHVQCEoDsFD0Q= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= diff --git a/quay/quay_overflow.go b/quay/quay_overflow.go new file mode 100644 index 000000000..8a2d9d8ab --- /dev/null +++ b/quay/quay_overflow.go @@ -0,0 +1,168 @@ +package main + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "strings" + "time" + + "k8s.io/client-go/rest" +) + +const ( + repo = "kuadrant/kuadrant-operator" + baseURL = "https://quay.io/api/v1/repository/" +) + +var ( + robotPass = os.Getenv("ROBOT_PASS") + robotUser = os.Getenv("ROBOT_USER") + accessToken = os.Getenv("ACCESS_TOKEN") + preserveSubstring = "latest" // Example Tag name that wont be deleted i.e relevant tags +) + +// Tag represents a tag in the repository. +type Tag struct { + Name string `json:"name"` + LastModified string `json:"last_modified"` +} + +// TagsResponse represents the structure of the API response that contains tags. +type TagsResponse struct { + Tags []Tag `json:"tags"` +} + +func main() { + client := &http.Client{} + + // Fetch tags from the API + tags, err := fetchTags(client) + if err != nil { + fmt.Println("Error fetching tags:", err) + return + } + + // Use filterTags to get tags to delete and remaining tags + tagsToDelete, remainingTags := filterTags(tags, preserveSubstring) + + // Delete tags and update remainingTags + for tagName := range tagsToDelete { + if deleteTag(client, accessToken, tagName) { + delete(remainingTags, tagName) // Remove deleted tag from remainingTags + } + } + + // Print remaining tags + fmt.Println("Remaining tags:") + for tag := range remainingTags { + fmt.Println(tag) + } +} + +// fetchTags retrieves the tags from the repository using the Quay.io API. +func fetchTags(client rest.HTTPClient) ([]Tag, error) { + // TODO - DO you want to seperate out builidng the request to a function to unit test? + // TODO - Is adding the headers even needed to fetch tags for a public repo? + req, err := http.NewRequest("GET", baseURL+repo+"/tag", nil) + if err != nil { + return nil, fmt.Errorf("error creating request: %w", err) + } + + // Prioritize Bearer token for authorization + if accessToken != "" { + req.Header.Add("Authorization", "Bearer "+accessToken) + } else { + // Fallback to Basic Authentication if no access token + auth := base64.StdEncoding.EncodeToString([]byte(robotUser + ":" + robotPass)) + req.Header.Add("Authorization", "Basic "+auth) + } + + // Execute the request + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("error making request: %w", err) + } + defer resp.Body.Close() + + // Handle possible non-200 status codes + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("error: received status code %d\nBody: %s", resp.StatusCode, string(body)) + } + + // Read the response body + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("error reading response body: %w", err) + } + + // Parse the JSON response + var tagsResp TagsResponse + if err := json.Unmarshal(body, &tagsResp); err != nil { + return nil, fmt.Errorf("error unmarshalling response: %w", err) + } + + return tagsResp.Tags, nil +} + +// filterTags takes a slice of tags and returns two maps: one for tags to delete and one for remaining tags. +func filterTags(tags []Tag, preserveSubstring string) (map[string]struct{}, map[string]struct{}) { + // Calculate the cutoff time + cutOffTime := time.Now().AddDate(0, 0, 0).Add(0 * time.Hour).Add(-1 * time.Minute) + + tagsToDelete := make(map[string]struct{}) + remainingTags := make(map[string]struct{}) + + for _, tag := range tags { + // Parse the LastModified timestamp + lastModified, err := time.Parse(time.RFC1123, tag.LastModified) + if err != nil { + fmt.Println("Error parsing time:", err) + continue + } + + // Check if tag should be deleted + if lastModified.Before(cutOffTime) && !containsSubstring(tag.Name, preserveSubstring) { + tagsToDelete[tag.Name] = struct{}{} + } else { + remainingTags[tag.Name] = struct{}{} + } + } + + return tagsToDelete, remainingTags +} + +func containsSubstring(tagName, substring string) bool { + return strings.Contains(tagName, substring) +} + +// deleteTag sends a DELETE request to remove the specified tag from the repository +// Returns true if successful, false otherwise +func deleteTag(client rest.HTTPClient, accessToken, tagName string) bool { + req, err := http.NewRequest("DELETE", baseURL+repo+"/tag/"+tagName, nil) + if err != nil { + fmt.Println("Error creating DELETE request:", err) + return false + } + req.Header.Add("Authorization", "Bearer "+accessToken) + + resp, err := client.Do(req) + if err != nil { + fmt.Println("Error deleting tag:", err) + return false + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusNoContent { + fmt.Printf("Successfully deleted tag: %s\n", tagName) + return true + } else { + body, _ := io.ReadAll(resp.Body) + fmt.Printf("Failed to delete tag %s: Status code %d\nBody: %s\n", tagName, resp.StatusCode, string(body)) + return false + } +} \ No newline at end of file diff --git a/quay/quay_overflow_test.go b/quay/quay_overflow_test.go new file mode 100644 index 000000000..0def55de1 --- /dev/null +++ b/quay/quay_overflow_test.go @@ -0,0 +1,208 @@ +package main + +import ( + "bytes" + "errors" + "io" + "net/http" + "strings" + "testing" + "time" + + "k8s.io/client-go/rest" +) + +type MockHTTPClient struct { + wantErr bool + mutateFn func(res *http.Response) +} + +func (m MockHTTPClient) Do(request *http.Request) (*http.Response, error) { + if m.wantErr { + return nil, errors.New("oops") + } + + resp := &http.Response{} + if m.mutateFn != nil { + m.mutateFn(resp) + } + + return resp, nil +} + +var _ rest.HTTPClient = &MockHTTPClient{} + +func Test_fetchTags(t *testing.T) { + t.Run("test error making request", func(t *testing.T) { + tags, err := fetchTags(&MockHTTPClient{wantErr: true}) + + if err == nil { + t.Error("error expected") + } + + if err.Error() != "error making request: oops" { + t.Errorf("error expected, got %s", err.Error()) + } + + if tags != nil { + t.Error("expected nil tags") + } + }) + + t.Run("test error for non-200 status codes", func(t *testing.T) { + tags, err := fetchTags(&MockHTTPClient{mutateFn: func(res *http.Response) { + res.Status = string(rune(400)) + res.Body = io.NopCloser(bytes.NewReader(nil)) + }}) + + if err == nil { + t.Error("error expected") + } + + if strings.Contains(err.Error(), "tags, error: received status code 400") { + t.Errorf("error expected, got %s", err.Error()) + } + + if tags != nil { + t.Error("expected nil tags") + } + }) + + t.Run("test error parsing json", func(t *testing.T) { + tags, err := fetchTags(&MockHTTPClient{mutateFn: func(res *http.Response) { + res.Status = string(rune(200)) + res.Body = io.NopCloser(bytes.NewReader([]byte("{notTags: error}"))) + }}) + + if err == nil { + t.Error("error expected") + } + + if strings.Contains(err.Error(), "error unmarshalling response:") { + t.Errorf("error expected, got %s", err.Error()) + } + + if tags != nil { + t.Error("expected nil tags") + } + }) + + t.Run("test successful response with tags", func(t *testing.T) { + mockJSONResponse := `{ + "tags": [ + {"name": "v1.0.0", "last_modified": "Mon, 02 Jan 2006 15:04:05 MST"}, + {"name": "v1.1.0", "last_modified": "Tue, 03 Jan 2006 15:04:05 MST"}, + {"name": "latest", "last_modified": "Wed, 04 Jan 2006 15:04:05 MST"} + ] + }` + + tags, err := fetchTags(&MockHTTPClient{mutateFn: func(res *http.Response) { + res.StatusCode = http.StatusOK + res.Body = io.NopCloser(bytes.NewReader([]byte(mockJSONResponse))) + }}) + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // Validate the returned tags + if len(tags) != 3 { + t.Fatalf("expected 3 tags, got %d", len(tags)) + } + + expectedTags := map[string]string{ + "v1.0.0": "Mon, 02 Jan 2006 15:04:05 MST", + "v1.1.0": "Tue, 03 Jan 2006 15:04:05 MST", + "latest": "Wed, 04 Jan 2006 15:04:05 MST", + } + + for _, tag := range tags { + if expectedDate, ok := expectedTags[tag.Name]; !ok || expectedDate != tag.LastModified { + t.Errorf("unexpected tag: got %v, expected %v", tag, expectedTags[tag.Name]) + } + } + }) +} + +func Test_deleteTag(t *testing.T) { + t.Run("test successful delete", func(t *testing.T) { + client := &MockHTTPClient{mutateFn: func(res *http.Response) { + res.StatusCode = http.StatusNoContent + res.Body = io.NopCloser(bytes.NewReader(nil)) + }} + + success := deleteTag(client, "fake_access_token", "v1.0.0") + + if !success { + t.Error("expected successful delete, got failure") + } + }) + + t.Run("test delete with error response", func(t *testing.T) { + client := &MockHTTPClient{mutateFn: func(res *http.Response) { + res.StatusCode = http.StatusInternalServerError + res.Body = io.NopCloser(bytes.NewReader([]byte("internal server error"))) + }} + + success := deleteTag(client, "fake_access_token", "v1.0.0") + + if success { + t.Error("expected failure, got success") + } + }) + + t.Run("test error making delete request", func(t *testing.T) { + client := &MockHTTPClient{wantErr: true} + + success := deleteTag(client, "fake_access_token", "v1.0.0") + + if success { + t.Error("expected failure, got success") + } + }) +} + +func Test_filterTags(t *testing.T) { + t.Run("test filter tags correctly", func(t *testing.T) { + tags := []Tag{ + {"nightly-build", time.Now().Add(-24 * time.Hour).Format(time.RFC1123)}, // Old tag, should be deleted + {"v1.1.0", time.Now().Format(time.RFC1123)}, // Recent tag, should be kept + {"latest", time.Now().Add(-24 * time.Hour).Format(time.RFC1123)}, // Old tag, but name contains preserveSubstring + } + + preserveSubstring := "latest" + + tagsToDelete, remainingTags := filterTags(tags, preserveSubstring) + + if len(tagsToDelete) != 1 || len(remainingTags) != 2 { + t.Fatalf("expected 1 tag to delete and 2 remaining, got %d to delete and %d remaining", len(tagsToDelete), len(remainingTags)) + } + + if _, ok := tagsToDelete["nightly-build"]; !ok { + t.Error("expected nightly-build to be deleted") + } + + if _, ok := remainingTags["v1.1.0"]; !ok { + t.Error("expected v1.1.0 to be kept") + } + + if _, ok := remainingTags["latest"]; !ok { + t.Error("expected latest to be kept") + } + }) + + t.Run("test filter tags with no deletions", func(t *testing.T) { + tags := []Tag{ + {"v1.1.0", time.Now().Format(time.RFC1123)}, // Recent tag, should be kept + {"latest", time.Now().Format(time.RFC1123)}, // Recent tag, should be kept + } + + preserveSubstring := "latest" + + tagsToDelete, remainingTags := filterTags(tags, preserveSubstring) + + if len(tagsToDelete) != 0 || len(remainingTags) != 2 { + t.Fatalf("expected 0 tags to delete and 2 remaining, got %d to delete and %d remaining", len(tagsToDelete), len(remainingTags)) + } + }) +}