From 125b3f0194921f5ea39122f699121a1662eba9f4 Mon Sep 17 00:00:00 2001 From: KevFan Date: Tue, 10 Sep 2024 17:11:50 +0100 Subject: [PATCH] refactor: remove cut off date logic, obselete if tags would use expiry label Signed-off-by: KevFan --- go.mod | 7 +-- go.sum | 6 --- quay/quay_overflow.go | 49 ++++++++------------ quay/quay_overflow_test.go | 92 +++++++++++++++++--------------------- 4 files changed, 61 insertions(+), 93 deletions(-) diff --git a/go.mod b/go.mod index d26995d0f..6f8714093 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/prometheus/client_golang v1.19.1 github.com/samber/lo v1.39.0 go.uber.org/zap v1.26.0 + golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f google.golang.org/protobuf v1.33.0 gotest.tools v2.2.0+incompatible istio.io/api v1.20.0 @@ -30,6 +31,7 @@ require ( k8s.io/klog/v2 v2.120.1 k8s.io/utils v0.0.0-20240423183400-0849a56e8f22 maistra.io/istio-operator v0.0.0-20240217080932-98753cb28cd7 + oras.land/oras-go v1.2.4 sigs.k8s.io/controller-runtime v0.18.0 sigs.k8s.io/external-dns v0.14.0 sigs.k8s.io/gateway-api v1.1.0 @@ -122,7 +124,6 @@ require ( github.com/opencontainers/image-spec v1.1.0-rc5 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.48.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect @@ -135,11 +136,9 @@ require ( github.com/spf13/cobra v1.8.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/objx v0.5.2 // indirect - github.com/stretchr/testify v1.9.0 // indirect github.com/tidwall/gjson v1.14.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect - github.com/vitorsalgado/mocha/v3 v3.0.2 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect @@ -151,7 +150,6 @@ require ( go.starlark.net v0.0.0-20231121155337-90ade8b19d09 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.22.0 // indirect - golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect golang.org/x/net v0.24.0 // indirect golang.org/x/oauth2 v0.19.0 // indirect golang.org/x/sync v0.7.0 // indirect @@ -175,7 +173,6 @@ require ( k8s.io/component-base v0.30.0 // indirect k8s.io/kube-openapi v0.0.0-20240423202451-8948a665c108 // indirect k8s.io/kubectl v0.29.1 // indirect - oras.land/oras-go v1.2.4 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect sigs.k8s.io/kustomize/kyaml v0.16.0 // indirect diff --git a/go.sum b/go.sum index ac966ed6f..6c6383f8d 100644 --- a/go.sum +++ b/go.sum @@ -240,8 +240,6 @@ github.com/kuadrant/authorino v0.17.2 h1:UgWH4NY/n36IhoaU+ELUkoujaly1/9sx5mHY5vU github.com/kuadrant/authorino v0.17.2/go.mod h1:al71fN0FX6c9Orrhk9GR4CtjtC+CD/lUHJCs7drlRNM= github.com/kuadrant/authorino-operator v0.11.1 h1:jndTZhiHMU+2Dk0NU+KP2+MUSfvclrn+YtTCQDJj+1s= github.com/kuadrant/authorino-operator v0.11.1/go.mod h1:TeFFdX477vUTMushCojaHpvwPLga4DpErGI2oQbqFIs= -github.com/kuadrant/dns-operator v0.0.0-20240731163454-777df870df90 h1:T08iFChpKyulZ/umDEuYBLvYgJBuv/9nli3W0wjr8OA= -github.com/kuadrant/dns-operator v0.0.0-20240731163454-777df870df90/go.mod h1:Aq4LYFwhBzQYUew71KjtWPKr+e0jzgraX10Ki8wIKCY= github.com/kuadrant/dns-operator v0.0.0-20240809151102-e79ebbca8f70 h1:Jiq7dZWaepPZAVrG3QsDfVAIyR3qdgTdqN5v2lTvO8k= github.com/kuadrant/dns-operator v0.0.0-20240809151102-e79ebbca8f70/go.mod h1:Aq4LYFwhBzQYUew71KjtWPKr+e0jzgraX10Ki8wIKCY= github.com/kuadrant/limitador-operator v0.9.0 h1:hTQ6CFPayf/sL7cIzwWjCoU8uTn6fzWdsJgKbDlnFts= @@ -400,8 +398,6 @@ github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AV github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.5.1 h1:4VhoImhV/Bm0ToFkXFi8hXNXwpDRZ/ynw3amt82mzq0= -github.com/stretchr/objx v0.5.1/go.mod h1:/iHQpkQwBD6DLUmQ4pE+s1TXdob1mORJ4/UFdrifcy0= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -417,8 +413,6 @@ github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/vitorsalgado/mocha/v3 v3.0.2 h1:uTx/+7kZvTWddXzoF34vUQTa3OL9OE+f5fPjD2XCMoY= -github.com/vitorsalgado/mocha/v3 v3.0.2/go.mod h1:ZMpyjuNfWPqLP2v7ztaaLJwOcyl4jmmHVQCEoDsFD0Q= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= diff --git a/quay/quay_overflow.go b/quay/quay_overflow.go index c6495e82a..110bb246b 100644 --- a/quay/quay_overflow.go +++ b/quay/quay_overflow.go @@ -9,7 +9,6 @@ import ( "net/http" "os" "regexp" - "time" "golang.org/x/exp/maps" "oras.land/oras-go/pkg/registry/remote" @@ -21,24 +20,19 @@ const ( ) var ( - repo *string - baseURL *string - dryRun *bool accessToken = os.Getenv("ACCESS_TOKEN") preserveSubstrings = []string{ "latest", - // Preserve release branch images - "release-v*", - // Semver regex - vX.Y.Z(-rc1) - "^v(?P0|[1-9]\\d*)\\.(?P0|[1-9]\\d*)\\.(?P0|[1-9]\\d*)(?:-(?P(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+(?P[0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$", + // Preserve semver release branch or semver tag regex - release-vX.Y.Z(-rc1) or vX.Y.Z(-rc1) + // Based on https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string + "^(v|release-v)(?P0|[1-9]\\d*)\\.(?P0|[1-9]\\d*)\\.(?P0|[1-9]\\d*)(?:-(?P(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+(?P[0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$", } ) // Tag represents a tag in the repository. type Tag struct { - Name string `json:"name"` - LastModified string `json:"last_modified"` - Expiration string `json:"expiration"` + Name string `json:"name"` + Expiration string `json:"expiration"` } // TagsResponse represents the structure of the API response that contains tags. @@ -49,9 +43,9 @@ type TagsResponse struct { } func main() { - repo = flag.String("repo", "kuadrant/kuadrant-operator", "Repository name") - baseURL = flag.String("base-url", "https://quay.io/api/v1/repository/", "Base API URL") - dryRun = flag.Bool("dry-run", true, "Dry run") + repo := flag.String("repo", "kuadrant/kuadrant-operator", "Repository name") + baseURL := flag.String("base-url", "https://quay.io/api/v1/repository/", "Base API URL") + dryRun := flag.Bool("dry-run", true, "Dry run") flag.Parse() client := &http.Client{} @@ -63,7 +57,7 @@ func main() { } // Fetch tags from the API - tags, err := fetchTags(client) + tags, err := fetchTags(client, baseURL, repo) if err != nil { logger.Fatalln("Error fetching tags:", err) } @@ -81,7 +75,7 @@ func main() { if dryRun != nil && *dryRun { logger.Printf("DRY RUN - Successfully deleted tag: %s\n", tagName) } else { - if err := deleteTag(client, accessToken, tagName); err != nil { + if err := deleteTag(client, baseURL, repo, accessToken, tagName); err != nil { logger.Println(err) continue } @@ -99,13 +93,14 @@ func main() { // fetchTags retrieves the tags from the repository using the Quay.io API. // https://docs.quay.io/api/swagger/#!/tag/listRepoTags -func fetchTags(client remote.Client) ([]Tag, error) { +func fetchTags(client remote.Client, baseURL, repo *string) ([]Tag, error) { allTags := make([]Tag, 0) i := 1 for { - req, err := http.NewRequest("GET", fmt.Sprintf("%s%s/tag/?page=%d&limit=%d", *baseURL, *repo, i, pageLimit), nil) + url := fmt.Sprintf("%s%s/tag/?page=%d&limit=%d", *baseURL, *repo, i, pageLimit) + req, err := http.NewRequest("GET", url, nil) if err != nil { return nil, fmt.Errorf("error creating request: %w", err) } @@ -155,7 +150,7 @@ func fetchTags(client remote.Client) ([]Tag, error) { // deleteTag sends a DELETE request to remove the specified tag from the repository // Returns nil if successful, error otherwise // https://docs.quay.io/api/swagger/#!/tag/deleteFullTag -func deleteTag(client remote.Client, accessToken, tagName string) error { +func deleteTag(client remote.Client, baseURL, repo *string, accessToken, tagName string) error { req, err := http.NewRequest("DELETE", fmt.Sprintf("%s%s/tag/%s", *baseURL, *repo, tagName), nil) if err != nil { return fmt.Errorf("error creating DELETE request: %s", err) @@ -178,11 +173,8 @@ func deleteTag(client remote.Client, accessToken, tagName string) error { // filterTags takes a slice of tags and preserves string regex and returns two maps: one for tags to delete and one for remaining tags. func filterTags(tags []Tag, preserveSubstrings []string) (map[string]struct{}, map[string]struct{}, error) { - // Calculate the cutoff time - cutOffTime := time.Now().AddDate(0, 0, 0).Add(0 * time.Hour).Add(-1 * time.Minute) - tagsToDelete := make(map[string]struct{}) - perservedTags := make(map[string]struct{}) + preservedTags := make(map[string]struct{}) // Compile the regexes for each preserve substring var preserveRegexes []*regexp.Regexp @@ -211,17 +203,12 @@ func filterTags(tags []Tag, preserveSubstrings []string) (map[string]struct{}, m } } - lastModified, err := time.Parse(time.RFC1123, tag.LastModified) - if err != nil { - return nil, nil, err - } - - if lastModified.Before(cutOffTime) && !preserve { + if !preserve { tagsToDelete[tag.Name] = struct{}{} } else { - perservedTags[tag.Name] = struct{}{} + preservedTags[tag.Name] = struct{}{} } } - return tagsToDelete, perservedTags, nil + return tagsToDelete, preservedTags, nil } diff --git a/quay/quay_overflow_test.go b/quay/quay_overflow_test.go index 8973b67c8..7cbc5f52f 100644 --- a/quay/quay_overflow_test.go +++ b/quay/quay_overflow_test.go @@ -5,13 +5,16 @@ import ( "errors" "io" "net/http" + "slices" "strings" "testing" "time" - "k8s.io/client-go/rest" + "oras.land/oras-go/pkg/registry/remote" ) +var _ remote.Client = &MockHTTPClient{} + type MockHTTPClient struct { wantErr bool mutateFn func(res *http.Response) @@ -30,11 +33,14 @@ func (m MockHTTPClient) Do(_ *http.Request) (*http.Response, error) { return resp, nil } -var _ rest.HTTPClient = &MockHTTPClient{} +var ( + testBaseUrl = "https://quay.io/api/v1/" + testRepo = "testOrg/kuadrant-operator" +) func Test_fetchTags(t *testing.T) { t.Run("test error making request", func(t *testing.T) { - tags, err := fetchTags(&MockHTTPClient{wantErr: true}) + tags, err := fetchTags(&MockHTTPClient{wantErr: true}, &testBaseUrl, &testRepo) if err == nil { t.Error("error expected") @@ -53,7 +59,7 @@ func Test_fetchTags(t *testing.T) { tags, err := fetchTags(&MockHTTPClient{mutateFn: func(res *http.Response) { res.Status = string(rune(400)) res.Body = io.NopCloser(bytes.NewReader(nil)) - }}) + }}, &testBaseUrl, &testRepo) if err == nil { t.Error("error expected") @@ -72,7 +78,7 @@ func Test_fetchTags(t *testing.T) { tags, err := fetchTags(&MockHTTPClient{mutateFn: func(res *http.Response) { res.Status = string(rune(200)) res.Body = io.NopCloser(bytes.NewReader([]byte("{notTags: error}"))) - }}) + }}, &testBaseUrl, &testRepo) if err == nil { t.Error("error expected") @@ -90,16 +96,16 @@ func Test_fetchTags(t *testing.T) { t.Run("test successful response with tags", func(t *testing.T) { mockJSONResponse := `{ "tags": [ - {"name": "v1.0.0", "last_modified": "Mon, 02 Jan 2006 15:04:05 MST"}, - {"name": "v1.1.0", "last_modified": "Tue, 03 Jan 2006 15:04:05 MST"}, - {"name": "latest", "last_modified": "Wed, 04 Jan 2006 15:04:05 MST"} + {"name": "v1.0.0"}, + {"name": "v1.1.0"}, + {"name": "latest"} ] }` tags, err := fetchTags(&MockHTTPClient{mutateFn: func(res *http.Response) { res.StatusCode = http.StatusOK res.Body = io.NopCloser(bytes.NewReader([]byte(mockJSONResponse))) - }}) + }}, &testBaseUrl, &testRepo) if err != nil { t.Fatalf("unexpected error: %v", err) @@ -110,15 +116,15 @@ func Test_fetchTags(t *testing.T) { t.Fatalf("expected 3 tags, got %d", len(tags)) } - expectedTags := map[string]string{ - "v1.0.0": "Mon, 02 Jan 2006 15:04:05 MST", - "v1.1.0": "Tue, 03 Jan 2006 15:04:05 MST", - "latest": "Wed, 04 Jan 2006 15:04:05 MST", + expectedTags := []string{ + "v1.0.0", + "v1.1.0", + "latest", } for _, tag := range tags { - if expectedDate, ok := expectedTags[tag.Name]; !ok || expectedDate != tag.LastModified { - t.Errorf("unexpected tag: got %v, expected %v", tag, expectedTags[tag.Name]) + if !slices.Contains(expectedTags, tag.Name) { + t.Errorf("unexpected tag: %v, does not exist in expected tags %v", tag, expectedTags) } } }) @@ -131,7 +137,7 @@ func Test_deleteTag(t *testing.T) { res.Body = io.NopCloser(bytes.NewReader(nil)) }} - err := deleteTag(client, "fake_access_token", "v1.0.0") + err := deleteTag(client, &testBaseUrl, &testRepo, "fake_access_token", "v1.0.0") if err != nil { t.Error("expected successful delete, got error") @@ -144,7 +150,7 @@ func Test_deleteTag(t *testing.T) { res.Body = io.NopCloser(bytes.NewReader([]byte("internal server error"))) }} - err := deleteTag(client, "fake_access_token", "v1.0.0") + err := deleteTag(client, &testBaseUrl, &testRepo, "fake_access_token", "v1.0.0") if err == nil { t.Error("expected failure, got success") @@ -154,7 +160,7 @@ func Test_deleteTag(t *testing.T) { t.Run("test error making delete request", func(t *testing.T) { client := &MockHTTPClient{wantErr: true} - err := deleteTag(client, "fake_access_token", "v1.0.0") + err := deleteTag(client, &testBaseUrl, &testRepo, "fake_access_token", "v1.0.0") if err == nil { t.Error("expected failure, got success") @@ -165,13 +171,13 @@ func Test_deleteTag(t *testing.T) { func Test_filterTags(t *testing.T) { t.Run("test filter tags correctly", func(t *testing.T) { tags := []Tag{ - {Name: "nightly-build", LastModified: time.Now().Add(-24 * time.Hour).Format(time.RFC1123)}, // Old tag, should be deleted - {Name: "v1.1.0", LastModified: time.Now().Format(time.RFC1123)}, // Recent tag, should be kept - {Name: "latest", LastModified: time.Now().Add(-24 * time.Hour).Format(time.RFC1123)}, // Old tag, but name contains preserveSubstring latest - {Name: "release-v1.2.3", LastModified: time.Now().Add(-24 * time.Hour).Format(time.RFC1123)}, // Old tag, but name contains preserveSubstring release-v* - {Name: "v1.0.0", LastModified: time.Now().Add(-24 * time.Hour).Format(time.RFC1123)}, // Old tag, but name contains preserveSubstring semver release - {Name: "v1.2.0-rc1", LastModified: time.Now().Add(-24 * time.Hour).Format(time.RFC1123)}, // Old tag, but name contains preserveSubstring semver release-candidate - {Name: "expiry_set", LastModified: time.Now().Add(-24 * time.Hour).Format(time.RFC1123), Expiration: time.Now().Format(time.RFC1123)}, // Old tag, but already has an expiry set + {Name: "nightly-build"}, // Not a preserved tag, should be deleted + {Name: "latest"}, // Preserved tag, name is latest + {Name: "release-v1.0.0"}, // Preserved tag, name contains preserveSubstring branch release semver, release-v* + {Name: "v1.0.0"}, // Preserved tag, but name contains preserveSubstring tag semver release + {Name: "v1.1.0-rc1"}, // Preserved tag, but name contains preserveSubstring tag semver release-candidate + {Name: "expiry_set", Expiration: time.Now().Format(time.RFC1123)}, // Skipped tag, already has an expiry set + {Name: "release-not-semver"}, // Not a preserved tag, should be deleted } tagsToDelete, remainingTags, err := filterTags(tags, preserveSubstrings) @@ -180,43 +186,39 @@ func Test_filterTags(t *testing.T) { t.Errorf("unexpected error: %v", err) } - if len(tagsToDelete) != 1 || len(remainingTags) != 6 { - t.Fatalf("expected 1 tag to delete and 6 remaining, got %d to delete and %d remaining", len(tagsToDelete), len(remainingTags)) + if len(tagsToDelete) != 2 || len(remainingTags) != 4 { + t.Fatalf("expected 2 tag to delete and 4 remaining, got %d to delete and %d remaining", len(tagsToDelete), len(remainingTags)) } if _, ok := tagsToDelete["nightly-build"]; !ok { t.Error("expected nightly-build to be deleted") } - if _, ok := remainingTags["v1.1.0"]; !ok { - t.Error("expected v1.1.0 to be kept") + if _, ok := tagsToDelete["release-not-semver"]; !ok { + t.Error("expected release-not-semver to be deleted") } if _, ok := remainingTags["latest"]; !ok { t.Error("expected latest to be kept") } - if _, ok := remainingTags["release-v1.2.3"]; !ok { - t.Error("expected release-v1.2.3 to be kept") + if _, ok := remainingTags["release-v1.0.0"]; !ok { + t.Error("expected release-v1.0.0 to be kept") } if _, ok := remainingTags["v1.0.0"]; !ok { t.Error("expected v1.0.0 to be kept") } - if _, ok := remainingTags["v1.2.0-rc1"]; !ok { - t.Error("expected v1.2.0-rc1 to be kept") - } - - if _, ok := remainingTags["expiry_set"]; !ok { - t.Error("expected expiry_set to be kept") + if _, ok := remainingTags["v1.1.0-rc1"]; !ok { + t.Error("expected v1.1.0-rc1 to be kept") } }) t.Run("test filter tags with no deletions", func(t *testing.T) { tags := []Tag{ - {Name: "v1.1.0", LastModified: time.Now().Format(time.RFC1123)}, // Preserved tag, should be kept - {Name: "recent", LastModified: time.Now().Format(time.RFC1123)}, // Recent tag, should be kept + {Name: "v1.1.0"}, // Preserved tag, should be kept + {Name: "latest"}, // Preserved tag, should be kept } tagsToDelete, remainingTags, err := filterTags(tags, preserveSubstrings) @@ -229,16 +231,4 @@ func Test_filterTags(t *testing.T) { t.Fatalf("expected 0 tags to delete and 2 remaining, got %d to delete and %d remaining", len(tagsToDelete), len(remainingTags)) } }) - - t.Run("test error unexpected time format", func(t *testing.T) { - tags := []Tag{ - {Name: "v1.1.0", LastModified: time.Now().Format(time.ANSIC)}, - } - - _, _, err := filterTags(tags, preserveSubstrings) - - if err == nil { - t.Fatal("expected error, got success") - } - }) }