From aaeb561a3d3d6ea1758e264025fa1ecc4424b5d9 Mon Sep 17 00:00:00 2001 From: Petu Eusebiu Date: Fri, 22 Sep 2023 15:29:01 +0300 Subject: [PATCH] feat(retention): added image retention policies feat(metaDB): add more image statistics info Signed-off-by: Petu Eusebiu --- errors/errors.go | 1 + examples/config-gc-periodic.json | 4 +- examples/config-gc.json | 42 +++- examples/config-sync-localhost.json | 37 ---- examples/config-sync.json | 6 +- pkg/api/config/config.go | 47 +++- pkg/api/controller.go | 18 +- pkg/api/controller_test.go | 44 +++- pkg/api/routes.go | 4 +- pkg/api/zot/cache.db | Bin 0 -> 32768 bytes pkg/api/zot/repo.db | Bin 0 -> 32768 bytes pkg/cli/server/extensions_test.go | 89 ++++---- pkg/cli/server/root.go | 35 ++- pkg/cli/server/stress_test.go | 2 +- .../search/convert/convert_internal_test.go | 2 +- pkg/extensions/search/cve/cve_test.go | 22 +- pkg/extensions/search/cve/pagination_test.go | 5 +- pkg/extensions/search/cve/scan_test.go | 32 ++- .../search/cve/trivy/scanner_internal_test.go | 15 +- pkg/extensions/search/resolver_test.go | 2 +- pkg/extensions/search/search_test.go | 6 +- pkg/extensions/sync/local.go | 5 +- pkg/extensions/sync/references/cosign.go | 3 +- pkg/extensions/sync/references/oci.go | 3 +- pkg/extensions/sync/references/oras.go | 3 +- pkg/extensions/sync/sync_internal_test.go | 8 +- pkg/extensions/sync/sync_test.go | 2 +- pkg/meta/boltdb/boltdb.go | 22 +- pkg/meta/boltdb/boltdb_test.go | 28 ++- pkg/meta/dynamodb/dynamodb.go | 26 ++- pkg/meta/dynamodb/dynamodb_test.go | 65 +++--- pkg/meta/hooks.go | 8 +- pkg/meta/hooks_test.go | 24 ++- pkg/meta/meta_test.go | 200 +++++++++++------- pkg/meta/parse.go | 11 +- pkg/meta/parse_test.go | 14 +- pkg/meta/types/types.go | 12 +- pkg/storage/gc/gc.go | 167 +++++++++++---- pkg/storage/gc/gc_internal_test.go | 122 ++++++----- pkg/storage/gc/policy.go | 198 +++++++++++++++++ .../gc/retention/candidate/candidate.go | 38 ++++ .../gc/retention/rules/dayspull/dayspull.go | 39 ++++ .../gc/retention/rules/dayspush/dayspush.go | 38 ++++ .../retention/rules/latestpull/latestpull.go | 35 +++ .../retention/rules/latestpush/latestpush.go | 41 ++++ pkg/storage/gc/retention/rules/name/name.go | 47 ++++ .../gc/retention/rules/retainall/retainall.go | 27 +++ pkg/storage/gc/retention/rules/rule.go | 8 + pkg/storage/imagestore/imagestore.go | 5 +- pkg/storage/local/local_test.go | 34 +-- pkg/storage/storage_test.go | 44 ++-- pkg/test/image-utils/upload_test.go | 17 +- pkg/test/mocks/repo_db_mock.go | 17 +- test/blackbox/garbage_collect.bats | 12 +- test/cluster/config-minio.json | 2 +- test/gc-stress/config-gc-bench-local.json | 1 - .../config-gc-bench-s3-localstack.json | 9 +- test/gc-stress/config-gc-bench-s3-minio.json | 9 +- .../config-gc-referrers-bench-local.json | 11 +- ...nfig-gc-referrers-bench-s3-localstack.json | 17 +- .../config-gc-referrers-bench-s3-minio.json | 17 +- 61 files changed, 1342 insertions(+), 460 deletions(-) delete mode 100644 examples/config-sync-localhost.json create mode 100644 pkg/api/zot/cache.db create mode 100644 pkg/api/zot/repo.db create mode 100644 pkg/storage/gc/policy.go create mode 100644 pkg/storage/gc/retention/candidate/candidate.go create mode 100644 pkg/storage/gc/retention/rules/dayspull/dayspull.go create mode 100644 pkg/storage/gc/retention/rules/dayspush/dayspush.go create mode 100644 pkg/storage/gc/retention/rules/latestpull/latestpull.go create mode 100644 pkg/storage/gc/retention/rules/latestpush/latestpush.go create mode 100644 pkg/storage/gc/retention/rules/name/name.go create mode 100644 pkg/storage/gc/retention/rules/retainall/retainall.go create mode 100644 pkg/storage/gc/retention/rules/rule.go diff --git a/errors/errors.go b/errors/errors.go index ed2dcf42cf..d240a970ad 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -163,4 +163,5 @@ var ( ErrInvalidOutputFormat = errors.New("cli: invalid output format") ErrFlagValueUnsupported = errors.New("supported values ") ErrUnknownSubcommand = errors.New("cli: unknown subcommand") + ErrGCPolicyNotFound = errors.New("gc: repo/tag policy not found") ) diff --git a/examples/config-gc-periodic.json b/examples/config-gc-periodic.json index 88e38cffa8..8e44166c4d 100644 --- a/examples/config-gc-periodic.json +++ b/examples/config-gc-periodic.json @@ -4,13 +4,13 @@ "rootDirectory": "/tmp/zot", "gc": true, "gcDelay": "1h", - "gcInterval": "24h", + "gcInterval": "1h", "subPaths": { "/a": { "rootDirectory": "/tmp/zot1", "gc": true, "gcDelay": "1h", - "gcInterval": "24h" + "gcInterval": "1h" } } }, diff --git a/examples/config-gc.json b/examples/config-gc.json index cad60105fd..e25b912b2f 100644 --- a/examples/config-gc.json +++ b/examples/config-gc.json @@ -3,10 +3,48 @@ "storage": { "rootDirectory": "/tmp/zot", "gc": true, - "gcReferrers": true, "gcDelay": "2h", "untaggedImageRetentionDelay": "4h", - "gcInterval": "1h" + "gcInterval": "1h", + "retention": { + "dryRun": false, + "policies": [ + { + "repoNames": ["infra/*", "prod/*"], + "deleteReferrers": false, + "deleteUntagged": true, + "tagsRetention": [{ + "names": ["v2.*", "*-prod"], + "retainAlways": true + }, + { + "names": ["v3.*", "*-prod"], + "pulledWithinLastNrDays": 7 + }] + }, + { + "repoNames": ["tmp/**"], + "deleteReferrers": true, + "deleteUntagged": true, + "tagsRetention": [{ + "names": ["v1.*"], + "pulledWithinLastNrDays": 7, + "pushedWithinLastNrDays": 7 + }] + }, + { + "repoNames": ["**"], + "deleteReferrers": true, + "deleteUntagged": true, + "tagsRetention": [{ + "mostRecentlyPushedCount": 10, + "mostRecentlyPulledCount": 10, + "pulledWithinLastNrDays": 30, + "pushedWithinLastNrDays": 30 + }] + } + ] + } }, "http": { "address": "127.0.0.1", diff --git a/examples/config-sync-localhost.json b/examples/config-sync-localhost.json deleted file mode 100644 index fc545f7b98..0000000000 --- a/examples/config-sync-localhost.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "distspecversion":"1.1.0-dev", - "storage": { - "rootDirectory": "/tmp/zot_to_sync", - "dedupe": false, - "gc": false - }, - "http": { - "address": "127.0.0.1", - "port": "8081" - }, - "log": { - "level": "debug" - }, - "extensions": { - "sync": { - "registries": [ - { - "urls": [ - "http://localhost:8080" - ], - "onDemand": true, - "tlsVerify": false, - "PollInterval": "30s", - "content": [ - { - "prefix": "**" - } - ] - } - ] - }, - "scrub": { - "interval": "24h" - } - } -} \ No newline at end of file diff --git a/examples/config-sync.json b/examples/config-sync.json index 092e4b1e85..c993119711 100644 --- a/examples/config-sync.json +++ b/examples/config-sync.json @@ -35,12 +35,12 @@ } }, { - "prefix": "/repo1/repo", + "prefix": "/repo2/repo", "destination": "/repo", "stripPrefix": true }, { - "prefix": "/repo2/repo" + "prefix": "/repo3/**" } ] }, @@ -54,7 +54,7 @@ "onDemand": false, "content": [ { - "prefix": "/repo2", + "prefix": "**", "tags": { "semver": true } diff --git a/pkg/api/config/config.go b/pkg/api/config/config.go index 8ab156e328..797ae5bd04 100644 --- a/pkg/api/config/config.go +++ b/pkg/api/config/config.go @@ -27,15 +27,36 @@ type StorageConfig struct { Dedupe bool RemoteCache bool GC bool - Commit bool GCDelay time.Duration GCInterval time.Duration - GCReferrers bool UntaggedImageRetentionDelay time.Duration + Retention ImageRetention + Commit bool StorageDriver map[string]interface{} `mapstructure:",omitempty"` CacheDriver map[string]interface{} `mapstructure:",omitempty"` } +type ImageRetention struct { + DryRun bool + Policies []GCPolicy +} + +type GCPolicy struct { + RepoNames []string + DeleteReferrers bool + DeleteUntagged bool + TagsRetention []TagsRetentionPolicy +} + +type TagsRetentionPolicy struct { + Names []string + RetainAlways bool // default is true + PulledWithinLastNrDays *int + PushedWithinLastNrDays *int + MostRecentlyPushedCount *int + MostRecentlyPulledCount *int +} + type TLSConfig struct { Cert string Key string @@ -190,9 +211,12 @@ func New() *Config { BinaryType: BinaryType, Storage: GlobalStorageConfig{ StorageConfig: StorageConfig{ - GC: true, GCReferrers: true, GCDelay: storageConstants.DefaultGCDelay, + Dedupe: true, + GC: true, + GCDelay: storageConstants.DefaultGCDelay, UntaggedImageRetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, - GCInterval: storageConstants.DefaultGCInterval, Dedupe: true, + GCInterval: storageConstants.DefaultGCInterval, + Retention: ImageRetention{}, }, }, HTTP: HTTPConfig{Address: "127.0.0.1", Port: "8080", Auth: &AuthConfig{FailDelay: 0}}, @@ -202,7 +226,8 @@ func New() *Config { func (expConfig StorageConfig) ParamsEqual(actConfig StorageConfig) bool { return expConfig.GC == actConfig.GC && expConfig.Dedupe == actConfig.Dedupe && - expConfig.GCDelay == actConfig.GCDelay && expConfig.GCInterval == actConfig.GCInterval + expConfig.GCDelay == actConfig.GCDelay && expConfig.GCInterval == actConfig.GCInterval && + expConfig.UntaggedImageRetentionDelay == actConfig.UntaggedImageRetentionDelay } // SameFile compare two files. @@ -368,6 +393,18 @@ func (c *Config) IsImageTrustEnabled() bool { return c.Extensions != nil && c.Extensions.Trust != nil && *c.Extensions.Trust.Enable } +func (c *Config) IsGarbageCollectEnabled() bool { + gcEnabled := c.Storage.GC + + for _, subpath := range c.Storage.SubPaths { + if subpath.GC { + gcEnabled = true + } + } + + return gcEnabled +} + func (c *Config) IsCosignEnabled() bool { return c.IsImageTrustEnabled() && c.Extensions.Trust.Cosign } diff --git a/pkg/api/controller.go b/pkg/api/controller.go index 9df5054784..66b9ac37b8 100644 --- a/pkg/api/controller.go +++ b/pkg/api/controller.go @@ -261,7 +261,8 @@ func (c *Controller) InitImageStore() error { func (c *Controller) InitMetaDB(reloadCtx context.Context) error { // init metaDB if search is enabled or we need to store user profiles, api keys or signatures - if c.Config.IsSearchEnabled() || c.Config.IsBasicAuthnEnabled() || c.Config.IsImageTrustEnabled() { + if c.Config.IsSearchEnabled() || c.Config.IsBasicAuthnEnabled() || c.Config.IsImageTrustEnabled() || + c.Config.IsGarbageCollectEnabled() { driver, err := meta.New(c.Config.Storage.StorageConfig, c.Log) //nolint:contextcheck if err != nil { return err @@ -277,7 +278,7 @@ func (c *Controller) InitMetaDB(reloadCtx context.Context) error { return err } - err = meta.ParseStorage(driver, c.StoreController, c.Log) + err = meta.ParseStorage(driver, c.StoreController, c.Log) //nolint: contextcheck if err != nil { return err } @@ -293,10 +294,7 @@ func (c *Controller) LoadNewConfig(reloadCtx context.Context, newConfig *config. c.Config.HTTP.AccessControl = newConfig.HTTP.AccessControl // reload periodical gc config - c.Config.Storage.GCInterval = newConfig.Storage.GCInterval c.Config.Storage.GC = newConfig.Storage.GC - c.Config.Storage.GCDelay = newConfig.Storage.GCDelay - c.Config.Storage.GCReferrers = newConfig.Storage.GCReferrers // reload background tasks if newConfig.Extensions != nil { @@ -340,9 +338,10 @@ func (c *Controller) StartBackgroundTasks(reloadCtx context.Context) { // Enable running garbage-collect periodically for DefaultStore if c.Config.Storage.GC { gc := gc.NewGarbageCollect(c.StoreController.DefaultStore, c.MetaDB, gc.Options{ - Referrers: c.Config.Storage.GCReferrers, + DryRun: c.Config.Storage.Retention.DryRun, Delay: c.Config.Storage.GCDelay, - RetentionDelay: c.Config.Storage.UntaggedImageRetentionDelay, + UntaggedDelay: c.Config.Storage.UntaggedImageRetentionDelay, + ImageRetention: c.Config.Storage.Retention, }, c.Log) gc.CleanImageStorePeriodically(c.Config.Storage.GCInterval, taskScheduler) @@ -363,9 +362,10 @@ func (c *Controller) StartBackgroundTasks(reloadCtx context.Context) { if storageConfig.GC { gc := gc.NewGarbageCollect(c.StoreController.SubStore[route], c.MetaDB, gc.Options{ - Referrers: storageConfig.GCReferrers, + DryRun: storageConfig.Retention.DryRun, Delay: storageConfig.GCDelay, - RetentionDelay: storageConfig.UntaggedImageRetentionDelay, + UntaggedDelay: storageConfig.UntaggedImageRetentionDelay, + ImageRetention: storageConfig.Retention, }, c.Log) gc.CleanImageStorePeriodically(storageConfig.GCInterval, taskScheduler) diff --git a/pkg/api/controller_test.go b/pkg/api/controller_test.go index 41f395dbfa..4bd90e39f0 100644 --- a/pkg/api/controller_test.go +++ b/pkg/api/controller_test.go @@ -314,9 +314,6 @@ func TestRunAlreadyRunningServer(t *testing.T) { defer cm.StopServer() err := ctlr.Init(context.Background()) - So(err, ShouldBeNil) - - err = ctlr.Run(context.Background()) So(err, ShouldNotBeNil) }) } @@ -395,6 +392,7 @@ func TestObjectStorageController(t *testing.T) { port := test.GetFreePort() conf := config.New() conf.HTTP.Port = port + conf.Storage.GC = false endpoint := os.Getenv("S3MOCK_ENDPOINT") @@ -501,6 +499,7 @@ func TestObjectStorageControllerSubPaths(t *testing.T) { port := test.GetFreePort() conf := config.New() conf.HTTP.Port = port + conf.Storage.GC = false endpoint := os.Getenv("S3MOCK_ENDPOINT") @@ -521,6 +520,7 @@ func TestObjectStorageControllerSubPaths(t *testing.T) { subPathMap["/a"] = config.StorageConfig{ RootDirectory: "/a", StorageDriver: storageDriverParams, + GC: false, } ctlr.Config.Storage.SubPaths = subPathMap @@ -7695,6 +7695,15 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) { ctlr.Config.Storage.GC = true ctlr.Config.Storage.GCDelay = 1 * time.Millisecond ctlr.Config.Storage.UntaggedImageRetentionDelay = 1 * time.Millisecond + ctlr.Config.Storage.Retention = config.ImageRetention{ + Policies: []config.GCPolicy{ + { + RepoNames: []string{"**"}, + DeleteReferrers: true, + DeleteUntagged: true, + }, + }, + } ctlr.Config.Storage.Dedupe = false @@ -7710,11 +7719,10 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) { gc := gc.NewGarbageCollect(ctlr.StoreController.DefaultStore, ctlr.MetaDB, gc.Options{ - Referrers: ctlr.Config.Storage.GCReferrers, Delay: ctlr.Config.Storage.GCDelay, - RetentionDelay: ctlr.Config.Storage.UntaggedImageRetentionDelay, - }, - ctlr.Log) + UntaggedDelay: ctlr.Config.Storage.UntaggedImageRetentionDelay, + ImageRetention: ctlr.Config.Storage.Retention, + }, ctlr.Log) resp, err := resty.R().Get(baseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, tag)) So(err, ShouldBeNil) @@ -7940,6 +7948,15 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) { ctlr.Config.Storage.GC = true ctlr.Config.Storage.GCDelay = 1 * time.Second ctlr.Config.Storage.UntaggedImageRetentionDelay = 1 * time.Second + ctlr.Config.Storage.Retention = config.ImageRetention{ + Policies: []config.GCPolicy{ + { + RepoNames: []string{"**"}, + DeleteReferrers: true, + DeleteUntagged: true, + }, + }, + } err := WriteImageToFileSystem(CreateDefaultImage(), repoName, tag, ociutils.GetDefaultStoreController(dir, ctlr.Log)) @@ -7951,9 +7968,9 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) { gc := gc.NewGarbageCollect(ctlr.StoreController.DefaultStore, ctlr.MetaDB, gc.Options{ - Referrers: ctlr.Config.Storage.GCReferrers, Delay: ctlr.Config.Storage.GCDelay, - RetentionDelay: ctlr.Config.Storage.UntaggedImageRetentionDelay, + UntaggedDelay: ctlr.Config.Storage.UntaggedImageRetentionDelay, + ImageRetention: ctlr.Config.Storage.Retention, }, ctlr.Log) resp, err := resty.R().Get(baseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, tag)) @@ -8084,8 +8101,13 @@ func TestPeriodicGC(t *testing.T) { subPaths := make(map[string]config.StorageConfig) subPaths["/a"] = config.StorageConfig{ - RootDirectory: subDir, GC: true, GCDelay: 1 * time.Second, - UntaggedImageRetentionDelay: 1 * time.Second, GCInterval: 24 * time.Hour, RemoteCache: false, Dedupe: false, + RootDirectory: subDir, + GC: true, + GCDelay: 1 * time.Second, + UntaggedImageRetentionDelay: 1 * time.Second, + GCInterval: 24 * time.Hour, + RemoteCache: false, + Dedupe: false, } //nolint:lll // gofumpt conflicts with lll ctlr.Config.Storage.Dedupe = false ctlr.Config.Storage.SubPaths = subPaths diff --git a/pkg/api/routes.go b/pkg/api/routes.go index 97cbe7ade7..046c1eb904 100644 --- a/pkg/api/routes.go +++ b/pkg/api/routes.go @@ -725,8 +725,8 @@ func (rh *RouteHandler) UpdateManifest(response http.ResponseWriter, request *ht } if rh.c.MetaDB != nil { - err := meta.OnUpdateManifest(name, reference, mediaType, digest, body, rh.c.StoreController, rh.c.MetaDB, - rh.c.Log) + err := meta.OnUpdateManifest(request.Context(), name, reference, mediaType, + digest, body, rh.c.StoreController, rh.c.MetaDB, rh.c.Log) if err != nil { response.WriteHeader(http.StatusInternalServerError) diff --git a/pkg/api/zot/cache.db b/pkg/api/zot/cache.db new file mode 100644 index 0000000000000000000000000000000000000000..79af015b9e543ef877f552ecd8fcbf8e5b1deb48 GIT binary patch literal 32768 zcmeI(zYf7r6bInz?_@BU#cN1xUc@X&OkT?SU^H7u%v!x?AWf`od`&nvZEtdXrr*77 zO4ZTN^>!ZJy5)HMc|KoGy*wD^8uK>u!|wWAF5*6w009C72oNAZfB*pk1PBmljzA~3 z5;^`?{(r9x6muf~5AyuuP>+1yL|P(%>UEkvs!Og`=Zt!u~<-h(MbXX2oNAZfB*pk1PBlyK%glCvCsd# z{~ylss=T*2Z*F)7sh(EIQ{yaH=mEfqVz zG5eu{LK;QoS$H!u*~!lA(@ka?jovj>m$%oGw}YiV{r3C%c4N4%jMU|G->A12X6DZR zKi*|OB>@5i2oNAZfB*pk1PBly@O=aZ>ZyFR{?GM&&iGGV$@PD>r*CNdiN8(D4dRVOR0ZY%2VUOA!lR{LPT>U8_*yICC_u2Qu)| z8>?&8Y3JSXDtfm+tqRB;IRGi;|fCWDyJ$6Z5d)^x&GnyeD9`D0t5&UAV7cs z0RjXF5Fqeb0=a)bU9Woc96+A)|EzI6mjD3*1PBlyK!5-N0t5&U_&R}H@7w=>F}MH! zVxdNW009C72oNAZfB*pkze6DR`E&n2*S>i!;CH;Gw;(`(009C72oNAZfB*pk1PBly nK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pkKU?4l5;9Fn literal 0 HcmV?d00001 diff --git a/pkg/cli/server/extensions_test.go b/pkg/cli/server/extensions_test.go index bf2c6a61d6..5804888185 100644 --- a/pkg/cli/server/extensions_test.go +++ b/pkg/cli/server/extensions_test.go @@ -30,9 +30,9 @@ func TestVerifyExtensionsConfig(t *testing.T) { So(err, ShouldBeNil) defer os.Remove(tmpfile.Name()) // clean up - content := []byte(`{ + content := fmt.Sprintf(`{ "storage":{ - "rootDirectory":"/tmp/zot", + "rootDirectory":"%s", "dedupe":true, "remoteCache":false, "storageDriver":{ @@ -56,21 +56,22 @@ func TestVerifyExtensionsConfig(t *testing.T) { } } } - }`) - err = os.WriteFile(tmpfile.Name(), content, 0o0600) + }`, t.TempDir()) + + err = os.WriteFile(tmpfile.Name(), []byte(content), 0o0600) So(err, ShouldBeNil) os.Args = []string{"cli_test", "verify", tmpfile.Name()} So(func() { _ = cli.NewServerRootCmd().Execute() }, ShouldPanic) - content = []byte(`{ + content = fmt.Sprintf(`{ "storage":{ - "rootDirectory":"/tmp/zot", + "rootDirectory":"%s", "dedupe":true, "remoteCache":false, "subPaths":{ "/a": { - "rootDirectory": "/tmp/zot1", + "rootDirectory": "%s", "dedupe": false, "storageDriver":{ "name":"s3", @@ -95,8 +96,8 @@ func TestVerifyExtensionsConfig(t *testing.T) { } } } - }`) - err = os.WriteFile(tmpfile.Name(), content, 0o0600) + }`, t.TempDir(), t.TempDir()) + err = os.WriteFile(tmpfile.Name(), []byte(content), 0o0600) So(err, ShouldBeNil) os.Args = []string{"cli_test", "verify", tmpfile.Name()} @@ -107,12 +108,12 @@ func TestVerifyExtensionsConfig(t *testing.T) { tmpfile, err := os.CreateTemp("", "zot-test*.json") So(err, ShouldBeNil) defer os.Remove(tmpfile.Name()) // clean up - content := []byte(`{"storage":{"rootDirectory":"/tmp/zot", "storageDriver": {"name": "s3"}}, + content := fmt.Sprintf(`{"storage":{"rootDirectory":"%s", "storageDriver": {"name": "s3"}}, "http":{"address":"127.0.0.1","port":"8080","realm":"zot", "auth":{"htpasswd":{"path":"test/data/htpasswd"},"failDelay":1}}, "extensions":{"sync": {"registries": [{"urls":["localhost:9999"], - "maxRetries": 1, "retryDelay": "10s"}]}}}`) - _, err = tmpfile.Write(content) + "maxRetries": 1, "retryDelay": "10s"}]}}}`, t.TempDir()) + _, err = tmpfile.Write([]byte(content)) So(err, ShouldBeNil) err = tmpfile.Close() So(err, ShouldBeNil) @@ -124,12 +125,12 @@ func TestVerifyExtensionsConfig(t *testing.T) { tmpfile, err := os.CreateTemp("", "zot-test*.json") So(err, ShouldBeNil) defer os.Remove(tmpfile.Name()) // clean up - content := []byte(`{"storage":{"rootDirectory":"/tmp/zot"}, + content := fmt.Sprintf(`{"storage":{"rootDirectory":"%s"}, "http":{"address":"127.0.0.1","port":"8080","realm":"zot", "auth":{"htpasswd":{"path":"test/data/htpasswd"},"failDelay":1}}, "extensions":{"sync": {"registries": [{"urls":["localhost:9999"], - "maxRetries": 1, "retryDelay": "10s"}]}}}`) - _, err = tmpfile.Write(content) + "maxRetries": 1, "retryDelay": "10s"}]}}}`, t.TempDir()) + _, err = tmpfile.Write([]byte(content)) So(err, ShouldBeNil) err = tmpfile.Close() So(err, ShouldBeNil) @@ -141,13 +142,13 @@ func TestVerifyExtensionsConfig(t *testing.T) { tmpfile, err := os.CreateTemp("", "zot-test*.json") So(err, ShouldBeNil) defer os.Remove(tmpfile.Name()) // clean up - content := []byte(`{"storage":{"rootDirectory":"/tmp/zot"}, + content := fmt.Sprintf(`{"storage":{"rootDirectory":"%s"}, "http":{"address":"127.0.0.1","port":"8080","realm":"zot", "auth":{"htpasswd":{"path":"test/data/htpasswd"},"failDelay":1}}, "extensions":{"sync": {"registries": [{"urls":["localhost:9999"], "maxRetries": 1, "retryDelay": "10s", - "content": [{"prefix":"[repo%^&"}]}]}}}`) - _, err = tmpfile.Write(content) + "content": [{"prefix":"[repo^&["}]}]}}}`, t.TempDir()) + _, err = tmpfile.Write([]byte(content)) So(err, ShouldBeNil) err = tmpfile.Close() So(err, ShouldBeNil) @@ -159,13 +160,13 @@ func TestVerifyExtensionsConfig(t *testing.T) { tmpfile, err := os.CreateTemp("", "zot-test*.json") So(err, ShouldBeNil) defer os.Remove(tmpfile.Name()) // clean up - content := []byte(`{"storage":{"rootDirectory":"/tmp/zot"}, + content := fmt.Sprintf(`{"storage":{"rootDirectory":"%s"}, "http":{"address":"127.0.0.1","port":"8080","realm":"zot", "auth":{"htpasswd":{"path":"test/data/htpasswd"},"failDelay":1}}, "extensions":{"sync": {"registries": [{"urls":["localhost:9999"], "maxRetries": 1, "retryDelay": "10s", - "content": [{"prefix":"zot-repo","stripPrefix":true,"destination":"/"}]}]}}}`) - _, err = tmpfile.Write(content) + "content": [{"prefix":"zot-repo","stripPrefix":true,"destination":"/"}]}]}}}`, t.TempDir()) + _, err = tmpfile.Write([]byte(content)) So(err, ShouldBeNil) err = tmpfile.Close() So(err, ShouldBeNil) @@ -177,13 +178,13 @@ func TestVerifyExtensionsConfig(t *testing.T) { tmpfile, err := os.CreateTemp("", "zot-test*.json") So(err, ShouldBeNil) defer os.Remove(tmpfile.Name()) // clean up - content := []byte(`{"storage":{"rootDirectory":"/tmp/zot"}, + content := fmt.Sprintf(`{"storage":{"rootDirectory":"%s"}, "http":{"address":"127.0.0.1","port":"8080","realm":"zot", "auth":{"htpasswd":{"path":"test/data/htpasswd"},"failDelay":1}}, "extensions":{"sync": {"registries": [{"urls":["localhost:9999"], "maxRetries": 1, "retryDelay": "10s", - "content": [{"prefix":"zot-repo/*","stripPrefix":true,"destination":"/"}]}]}}}`) - _, err = tmpfile.Write(content) + "content": [{"prefix":"zot-repo/*","stripPrefix":true,"destination":"/"}]}]}}}`, t.TempDir()) + _, err = tmpfile.Write([]byte(content)) So(err, ShouldBeNil) err = tmpfile.Close() So(err, ShouldBeNil) @@ -196,13 +197,13 @@ func TestVerifyExtensionsConfig(t *testing.T) { tmpfile, err := os.CreateTemp("", "zot-test*.json") So(err, ShouldBeNil) defer os.Remove(tmpfile.Name()) // clean up - content := []byte(`{"storage":{"rootDirectory":"/tmp/zot"}, + content := fmt.Sprintf(`{"storage":{"rootDirectory":"%s"}, "http":{"address":"127.0.0.1","port":"8080","realm":"zot", "auth":{"htpasswd":{"path":"test/data/htpasswd"},"failDelay":1}}, "extensions":{"sync": {"registries": [{"urls":["localhost:9999"], "maxRetries": 1, "retryDelay": "10s", - "content": [{"prefix":"repo**"}]}]}}}`) - _, err = tmpfile.Write(content) + "content": [{"prefix":"repo**"}]}]}}}`, t.TempDir()) + _, err = tmpfile.Write([]byte(content)) So(err, ShouldBeNil) err = tmpfile.Close() So(err, ShouldBeNil) @@ -215,12 +216,12 @@ func TestVerifyExtensionsConfig(t *testing.T) { tmpfile, err := os.CreateTemp("", "zot-test*.json") So(err, ShouldBeNil) defer os.Remove(tmpfile.Name()) // clean up - content := []byte(`{"storage":{"rootDirectory":"/tmp/zot"}, + content := fmt.Sprintf(`{"storage":{"rootDirectory":"%s"}, "http":{"address":"127.0.0.1","port":"8080","realm":"zot", "auth":{"htpasswd":{"path":"test/data/htpasswd"},"failDelay":1}}, "extensions":{"sync": {"registries": [{"urls":["localhost:9999"], - "maxRetries": 10, "content": [{"prefix":"repo**"}]}]}}}`) - _, err = tmpfile.Write(content) + "maxRetries": 10, "content": [{"prefix":"repo**"}]}]}}}`, t.TempDir()) + _, err = tmpfile.Write([]byte(content)) So(err, ShouldBeNil) err = tmpfile.Close() So(err, ShouldBeNil) @@ -377,7 +378,7 @@ func TestServeExtensions(t *testing.T) { content := fmt.Sprintf(`{ "storage": { - "rootDirectory": "/tmp/zot" + "rootDirectory": "%s" }, "http": { "address": "127.0.0.1", @@ -387,7 +388,7 @@ func TestServeExtensions(t *testing.T) { "level": "debug", "output": "%s" } - }`, port, logFile.Name()) + }`, t.TempDir(), port, logFile.Name()) cfgfile, err := os.CreateTemp("", "zot-test*.json") So(err, ShouldBeNil) @@ -418,7 +419,7 @@ func TestServeExtensions(t *testing.T) { content := fmt.Sprintf(`{ "storage": { - "rootDirectory": "/tmp/zot" + "rootDirectory": "%s" }, "http": { "address": "127.0.0.1", @@ -430,7 +431,7 @@ func TestServeExtensions(t *testing.T) { }, "extensions": { } - }`, port, logFile.Name()) + }`, t.TempDir(), port, logFile.Name()) cfgfile, err := os.CreateTemp("", "zot-test*.json") So(err, ShouldBeNil) @@ -454,7 +455,7 @@ func TestServeExtensions(t *testing.T) { }) } -func testWithMetricsEnabled(cfgContentFormat string) { +func testWithMetricsEnabled(rootDir string, cfgContentFormat string) { port := GetFreePort() baseURL := GetBaseURL(port) logFile, err := os.CreateTemp("", "zot-log*.txt") @@ -462,7 +463,7 @@ func testWithMetricsEnabled(cfgContentFormat string) { defer os.Remove(logFile.Name()) // clean up - content := fmt.Sprintf(cfgContentFormat, port, logFile.Name()) + content := fmt.Sprintf(cfgContentFormat, rootDir, port, logFile.Name()) cfgfile, err := os.CreateTemp("", "zot-test*.json") So(err, ShouldBeNil) @@ -502,7 +503,7 @@ func TestServeMetricsExtension(t *testing.T) { Convey("no explicit enable", t, func(c C) { content := `{ "storage": { - "rootDirectory": "/tmp/zot" + "rootDirectory": "%s" }, "http": { "address": "127.0.0.1", @@ -517,13 +518,13 @@ func TestServeMetricsExtension(t *testing.T) { } } }` - testWithMetricsEnabled(content) + testWithMetricsEnabled(t.TempDir(), content) }) Convey("no explicit enable but with prometheus parameter", t, func(c C) { content := `{ "storage": { - "rootDirectory": "/tmp/zot" + "rootDirectory": "%s" }, "http": { "address": "127.0.0.1", @@ -541,13 +542,13 @@ func TestServeMetricsExtension(t *testing.T) { } } }` - testWithMetricsEnabled(content) + testWithMetricsEnabled(t.TempDir(), content) }) Convey("with explicit enable, but without prometheus parameter", t, func(c C) { content := `{ "storage": { - "rootDirectory": "/tmp/zot" + "rootDirectory": "%s" }, "http": { "address": "127.0.0.1", @@ -563,7 +564,7 @@ func TestServeMetricsExtension(t *testing.T) { } } }` - testWithMetricsEnabled(content) + testWithMetricsEnabled(t.TempDir(), content) }) Convey("with explicit disable", t, func(c C) { @@ -575,7 +576,7 @@ func TestServeMetricsExtension(t *testing.T) { content := fmt.Sprintf(`{ "storage": { - "rootDirectory": "/tmp/zot" + "rootDirectory": "%s" }, "http": { "address": "127.0.0.1", @@ -590,7 +591,7 @@ func TestServeMetricsExtension(t *testing.T) { "enable": false } } - }`, port, logFile.Name()) + }`, t.TempDir(), port, logFile.Name()) cfgfile, err := os.CreateTemp("", "zot-test*.json") So(err, ShouldBeNil) diff --git a/pkg/cli/server/root.go b/pkg/cli/server/root.go index c4088fa806..d6d6c83444 100644 --- a/pkg/cli/server/root.go +++ b/pkg/cli/server/root.go @@ -615,7 +615,7 @@ func applyDefaultValues(config *config.Config, viperInstance *viper.Viper, log z config.Storage.RemoteCache = true } - // s3 dedup=false, check for previous dedup usage and set to true if cachedb found + // s3 dedup=false, check for previous dedupe usage and set to true if cachedb found if !config.Storage.Dedupe && config.Storage.StorageDriver != nil { cacheDir, _ := config.Storage.StorageDriver["rootdirectory"].(string) cachePath := path.Join(cacheDir, storageConstants.BoltdbName+storageConstants.DBExtensionName) @@ -651,11 +651,6 @@ func applyDefaultValues(config *config.Config, viperInstance *viper.Viper, log z // if gc is enabled if storageConfig.GC { - // and gcReferrers is not set, it is set to default value - if !viperInstance.IsSet("storage::subpaths::" + name + "::gcreferrers") { - storageConfig.GCReferrers = true - } - // and gcDelay is not set, it is set to default value if !viperInstance.IsSet("storage::subpaths::" + name + "::gcdelay") { storageConfig.GCDelay = storageConstants.DefaultGCDelay @@ -823,6 +818,24 @@ func validateHTTP(config *config.Config, log zlog.Logger) error { return nil } +func validateGCRules(gc config.ImageRetention, log zlog.Logger) error { + for _, policy := range gc.Policies { + for _, tagRule := range policy.TagsRetention { + if tagRule.RetainAlways && (tagRule.MostRecentlyPulledCount != nil || + tagRule.MostRecentlyPushedCount != nil || + tagRule.PulledWithinLastNrDays != nil || + tagRule.PushedWithinLastNrDays != nil) { + log.Error().Err(zerr.ErrBadConfig). + Msg("retainAlways can only be used alone or with name rule, the other are mutual exclusive") + + return zerr.ErrBadConfig + } + } + } + + return nil +} + func validateGC(config *config.Config, log zlog.Logger) error { // enforce GC params if config.Storage.GCDelay < 0 { @@ -833,7 +846,7 @@ func validateGC(config *config.Config, log zlog.Logger) error { } if config.Storage.GCInterval < 0 { - log.Error().Err(zerr.ErrBadConfig).Dur("interval", config.Storage.GCInterval). + log.Error().Err(zerr.ErrBadConfig).Dur("interval", config.Storage.GCDelay). Msg("invalid garbage-collect interval specified") return zerr.ErrBadConfig @@ -851,6 +864,10 @@ func validateGC(config *config.Config, log zlog.Logger) error { } } + if err := validateGCRules(config.Storage.Retention, log); err != nil { + return err + } + // subpaths for name, subPath := range config.Storage.SubPaths { if subPath.GC && subPath.GCDelay <= 0 { @@ -861,6 +878,10 @@ func validateGC(config *config.Config, log zlog.Logger) error { return zerr.ErrBadConfig } + + if err := validateGCRules(subPath.Retention, log); err != nil { + return err + } } return nil diff --git a/pkg/cli/server/stress_test.go b/pkg/cli/server/stress_test.go index a12113ca9f..8e25d60f79 100644 --- a/pkg/cli/server/stress_test.go +++ b/pkg/cli/server/stress_test.go @@ -80,7 +80,7 @@ func TestSressTooManyOpenFiles(t *testing.T) { "storage": { "rootDirectory": "%s", "dedupe": %t, - "gc": %t + "gc": %v }, "http": { "address": "127.0.0.1", diff --git a/pkg/extensions/search/convert/convert_internal_test.go b/pkg/extensions/search/convert/convert_internal_test.go index fe99ae68c7..85948f85d4 100644 --- a/pkg/extensions/search/convert/convert_internal_test.go +++ b/pkg/extensions/search/convert/convert_internal_test.go @@ -56,7 +56,7 @@ func TestCVEConvert(t *testing.T) { digest11 := godigest.FromString("abc1") err = metaDB.SetManifestMeta("repo1", digest11, repoMeta11) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo1", "0.1.0", digest11, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo1", "0.1.0", digest11, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) reposMeta, manifestMetaMap, _, err := metaDB.SearchRepos(context.Background(), "") diff --git a/pkg/extensions/search/cve/cve_test.go b/pkg/extensions/search/cve/cve_test.go index 32d9059150..ace30a86ef 100644 --- a/pkg/extensions/search/cve/cve_test.go +++ b/pkg/extensions/search/cve/cve_test.go @@ -5,6 +5,7 @@ package cveinfo_test import ( + "context" "encoding/json" "fmt" "io" @@ -773,7 +774,7 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo err = metaDB.SetManifestMeta(repo1, image11.ManifestDescriptor.Digest, repoMeta11) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, "0.1.0", image11.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, "0.1.0", image11.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) image12 := CreateImageWith().DefaultLayers(). @@ -788,7 +789,7 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo err = metaDB.SetManifestMeta(repo1, image12.ManifestDescriptor.Digest, repoMeta12) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, "1.0.0", image12.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, "1.0.0", image12.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) image13 := CreateImageWith().DefaultLayers(). @@ -803,7 +804,7 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo err = metaDB.SetManifestMeta(repo1, image13.ManifestDescriptor.Digest, repoMeta13) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, "1.1.0", image13.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, "1.1.0", image13.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) image14 := CreateImageWith().DefaultLayers(). @@ -816,7 +817,7 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo err = metaDB.SetManifestMeta(repo1, image14.ManifestDescriptor.Digest, repoMeta14) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, "1.0.1", image14.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, "1.0.1", image14.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) // Create metadb data for scannable image with no vulnerabilities @@ -830,7 +831,7 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo err = metaDB.SetManifestMeta(repo6, image61.ManifestDescriptor.Digest, repoMeta61) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo6, "1.0.0", image61.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo6, "1.0.0", image61.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) // Create metadb data for image not supporting scanning @@ -847,7 +848,7 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo err = metaDB.SetManifestMeta(repo2, image21.ManifestDescriptor.Digest, repoMeta21) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, "1.0.0", image21.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo2, "1.0.0", image21.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) // Create metadb data for invalid images/negative tests @@ -861,7 +862,7 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo digest31 := godigest.FromBytes(manifestBlob31) err = metaDB.SetManifestMeta(repo3, digest31, repoMeta31) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo3, "invalid-manifest", digest31, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo3, "invalid-manifest", digest31, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) image41 := CreateImageWith().DefaultLayers(). @@ -874,12 +875,12 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo err = metaDB.SetManifestMeta(repo4, image41.ManifestDescriptor.Digest, repoMeta41) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo4, "invalid-config", image41.ManifestDescriptor.Digest, + err = metaDB.SetRepoReference(context.Background(), repo4, "invalid-config", image41.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) digest51 := godigest.FromString("abc8") - err = metaDB.SetRepoReference(repo5, "nonexitent-manifest", digest51, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo5, "nonexitent-manifest", digest51, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) // Create metadb data for scannable image which errors during scan @@ -893,7 +894,7 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo err = metaDB.SetManifestMeta(repo7, image71.ManifestDescriptor.Digest, repoMeta71) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo7, "1.0.0", image71.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo7, "1.0.0", image71.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) // create multiarch image with vulnerabilities @@ -933,6 +934,7 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo So(err, ShouldBeNil) err = metaDB.SetRepoReference( + context.Background(), repoMultiarch, "tagIndex", multiarchImage.IndexDescriptor.Digest, diff --git a/pkg/extensions/search/cve/pagination_test.go b/pkg/extensions/search/cve/pagination_test.go index ab518c2cd0..fad034bf5a 100644 --- a/pkg/extensions/search/cve/pagination_test.go +++ b/pkg/extensions/search/cve/pagination_test.go @@ -4,6 +4,7 @@ package cveinfo_test import ( + "context" "encoding/json" "fmt" "sort" @@ -65,7 +66,7 @@ func TestCVEPagination(t *testing.T) { digest11 := godigest.FromBytes(manifestBlob11) err = metaDB.SetManifestMeta("repo1", digest11, repoMeta11) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo1", "0.1.0", digest11, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo1", "0.1.0", digest11, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) timeStamp12 := time.Date(2009, 1, 1, 12, 0, 0, 0, time.UTC) @@ -99,7 +100,7 @@ func TestCVEPagination(t *testing.T) { digest12 := godigest.FromBytes(manifestBlob12) err = metaDB.SetManifestMeta("repo1", digest12, repoMeta12) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo1", "1.0.0", digest12, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo1", "1.0.0", digest12, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) // MetaDB loaded with initial data, mock the scanner diff --git a/pkg/extensions/search/cve/scan_test.go b/pkg/extensions/search/cve/scan_test.go index ccb45dfc8b..a9ded0fcc5 100644 --- a/pkg/extensions/search/cve/scan_test.go +++ b/pkg/extensions/search/cve/scan_test.go @@ -82,7 +82,8 @@ func TestScanGeneratorWithMockedData(t *testing.T) { //nolint: gocyclo err = metaDB.SetManifestMeta("repo1", image11.ManifestDescriptor.Digest, repoMeta11) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo1", "0.1.0", image11.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo1", + "0.1.0", image11.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) image12 := CreateImageWith().DefaultLayers(). @@ -97,7 +98,8 @@ func TestScanGeneratorWithMockedData(t *testing.T) { //nolint: gocyclo err = metaDB.SetManifestMeta("repo1", image12.ManifestDescriptor.Digest, repoMeta12) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo1", "1.0.0", image12.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo1", + "1.0.0", image12.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) image13 := CreateImageWith().DefaultLayers(). @@ -112,7 +114,8 @@ func TestScanGeneratorWithMockedData(t *testing.T) { //nolint: gocyclo err = metaDB.SetManifestMeta("repo1", image13.ManifestDescriptor.Digest, repoMeta13) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo1", "1.1.0", image13.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo1", + "1.1.0", image13.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) image14 := CreateImageWith().DefaultLayers(). @@ -125,7 +128,8 @@ func TestScanGeneratorWithMockedData(t *testing.T) { //nolint: gocyclo err = metaDB.SetManifestMeta("repo1", image14.ManifestDescriptor.Digest, repoMeta14) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo1", "1.0.1", image14.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo1", + "1.0.1", image14.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) // Create metadb data for scannable image with no vulnerabilities @@ -139,7 +143,8 @@ func TestScanGeneratorWithMockedData(t *testing.T) { //nolint: gocyclo err = metaDB.SetManifestMeta("repo6", image61.ManifestDescriptor.Digest, repoMeta61) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo6", "1.0.0", image61.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo6", + "1.0.0", image61.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) // Create metadb data for image not supporting scanning @@ -156,7 +161,8 @@ func TestScanGeneratorWithMockedData(t *testing.T) { //nolint: gocyclo err = metaDB.SetManifestMeta("repo2", image21.ManifestDescriptor.Digest, repoMeta21) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo2", "1.0.0", image21.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo2", + "1.0.0", image21.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) // Create metadb data for invalid images/negative tests @@ -170,7 +176,8 @@ func TestScanGeneratorWithMockedData(t *testing.T) { //nolint: gocyclo digest31 := godigest.FromBytes(manifestBlob31) err = metaDB.SetManifestMeta("repo3", digest31, repoMeta31) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo3", "invalid-manifest", digest31, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo3", + "invalid-manifest", digest31, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) image41 := CreateImageWith().DefaultLayers(). @@ -183,12 +190,13 @@ func TestScanGeneratorWithMockedData(t *testing.T) { //nolint: gocyclo err = metaDB.SetManifestMeta("repo4", image41.ManifestDescriptor.Digest, repoMeta41) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo4", "invalid-config", image41.ManifestDescriptor.Digest, - ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo4", + "invalid-config", image41.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) digest51 := godigest.FromString("abc8") - err = metaDB.SetRepoReference("repo5", "nonexitent-manifest", digest51, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo5", + "nonexitent-manifest", digest51, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) // Create metadb data for scannable image which errors during scan @@ -202,7 +210,8 @@ func TestScanGeneratorWithMockedData(t *testing.T) { //nolint: gocyclo err = metaDB.SetManifestMeta("repo7", image71.ManifestDescriptor.Digest, repoMeta71) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo7", "1.0.0", image71.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo7", + "1.0.0", image71.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) // Create multiarch image with vulnerabilities @@ -242,6 +251,7 @@ func TestScanGeneratorWithMockedData(t *testing.T) { //nolint: gocyclo So(err, ShouldBeNil) err = metaDB.SetRepoReference( + context.Background(), repoIndex, "tagIndex", multiarchImage.IndexDescriptor.Digest, diff --git a/pkg/extensions/search/cve/trivy/scanner_internal_test.go b/pkg/extensions/search/cve/trivy/scanner_internal_test.go index 4d81ef9778..8842b810c2 100644 --- a/pkg/extensions/search/cve/trivy/scanner_internal_test.go +++ b/pkg/extensions/search/cve/trivy/scanner_internal_test.go @@ -5,6 +5,7 @@ package trivy import ( "bytes" + "context" "encoding/json" "os" "path" @@ -325,7 +326,8 @@ func TestImageScannable(t *testing.T) { panic(err) } - err = metaDB.SetRepoReference("repo1", "valid", digestValidManifest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo1", + "valid", digestValidManifest, ispec.MediaTypeImageManifest) if err != nil { panic(err) } @@ -361,7 +363,7 @@ func TestImageScannable(t *testing.T) { panic(err) } - err = metaDB.SetRepoReference("repo1", "unscannable-layer", digestManifestUnscannableLayer, + err = metaDB.SetRepoReference(context.Background(), "repo1", "unscannable-layer", digestManifestUnscannableLayer, ispec.MediaTypeImageManifest) if err != nil { panic(err) @@ -381,7 +383,8 @@ func TestImageScannable(t *testing.T) { panic(err) } - err = metaDB.SetRepoReference("repo1", "unmarshable", digestUnmarshableManifest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo1", + "unmarshable", digestUnmarshableManifest, ispec.MediaTypeImageManifest) if err != nil { panic(err) } @@ -389,13 +392,15 @@ func TestImageScannable(t *testing.T) { // Manifest meta cannot be found digestMissingManifest := godigest.FromBytes([]byte("Some other string")) - err = metaDB.SetRepoReference("repo1", "missing", digestMissingManifest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo1", + "missing", digestMissingManifest, ispec.MediaTypeImageManifest) if err != nil { panic(err) } // RepoMeta contains invalid digest - err = metaDB.SetRepoReference("repo1", "invalid-digest", "invalid", ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo1", + "invalid-digest", "invalid", ispec.MediaTypeImageManifest) if err != nil { panic(err) } diff --git a/pkg/extensions/search/resolver_test.go b/pkg/extensions/search/resolver_test.go index eedcce38b3..dae30468f2 100644 --- a/pkg/extensions/search/resolver_test.go +++ b/pkg/extensions/search/resolver_test.go @@ -2072,7 +2072,7 @@ func TestCVEResolvers(t *testing.T) { //nolint:gocyclo for image, digest := range tagsMap { repo, tag := common.GetImageDirAndTag(image) - err := metaDB.SetRepoReference(repo, tag, digest, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo, tag, digest, ispec.MediaTypeImageManifest) if err != nil { panic(err) } diff --git a/pkg/extensions/search/search_test.go b/pkg/extensions/search/search_test.go index 1ed410162f..58d3831a60 100644 --- a/pkg/extensions/search/search_test.go +++ b/pkg/extensions/search/search_test.go @@ -4530,7 +4530,9 @@ func TestMetaDBWhenPushingImages(t *testing.T) { Convey("SetManifestMeta succeeds but SetRepoReference fails", func() { ctlr.MetaDB = mocks.MetaDBMock{ - SetRepoReferenceFn: func(repo, reference string, manifestDigest godigest.Digest, mediaType string) error { + SetRepoReferenceFn: func(ctx context.Context, repo, + reference string, manifestDigest godigest.Digest, mediaType string, + ) error { return ErrTestError }, } @@ -5258,7 +5260,7 @@ func TestMetaDBWhenReadingImages(t *testing.T) { Convey("Error when incrementing", func() { ctlr.MetaDB = mocks.MetaDBMock{ - IncrementImageDownloadsFn: func(repo string, tag string) error { + UpdateStatsOnDownloadFn: func(repo string, tag string) error { return ErrTestError }, } diff --git a/pkg/extensions/sync/local.go b/pkg/extensions/sync/local.go index 9bf1717901..86ccf357c5 100644 --- a/pkg/extensions/sync/local.go +++ b/pkg/extensions/sync/local.go @@ -4,6 +4,7 @@ package sync import ( + "context" "encoding/json" "errors" "fmt" @@ -164,7 +165,7 @@ func (registry *LocalRegistry) CommitImage(imageReference types.ImageReference, } if registry.metaDB != nil { - err = meta.SetImageMetaFromInput(repo, reference, mediaType, + err = meta.SetImageMetaFromInput(context.Background(), repo, reference, mediaType, manifestDigest, manifestBlob, imageStore, registry.metaDB, registry.log) if err != nil { return fmt.Errorf("metaDB: failed to set metadata for image '%s %s': %w", repo, reference, err) @@ -222,7 +223,7 @@ func (registry *LocalRegistry) copyManifest(repo string, manifestContent []byte, } if registry.metaDB != nil { - err = meta.SetImageMetaFromInput(repo, reference, ispec.MediaTypeImageManifest, + err = meta.SetImageMetaFromInput(context.Background(), repo, reference, ispec.MediaTypeImageManifest, digest, manifestContent, imageStore, registry.metaDB, registry.log) if err != nil { registry.log.Error().Str("errorType", common.TypeOf(err)). diff --git a/pkg/extensions/sync/references/cosign.go b/pkg/extensions/sync/references/cosign.go index 74d399f99d..6867849e06 100644 --- a/pkg/extensions/sync/references/cosign.go +++ b/pkg/extensions/sync/references/cosign.go @@ -166,7 +166,8 @@ func (ref CosignReference) SyncReferences(ctx context.Context, localRepo, remote SignatureDigest: referenceDigest.String(), }) } else { - err = meta.SetImageMetaFromInput(localRepo, cosignTag, ispec.MediaTypeImageManifest, + err = meta.SetImageMetaFromInput(context.Background(), localRepo, //nolint:contextcheck + cosignTag, ispec.MediaTypeImageManifest, referenceDigest, manifestBuf, ref.storeController.GetImageStore(localRepo), ref.metaDB, ref.log) } diff --git a/pkg/extensions/sync/references/oci.go b/pkg/extensions/sync/references/oci.go index 4a98a33e6b..7985214d9d 100644 --- a/pkg/extensions/sync/references/oci.go +++ b/pkg/extensions/sync/references/oci.go @@ -150,7 +150,8 @@ func (ref OciReferences) SyncReferences(ctx context.Context, localRepo, remoteRe SignatureDigest: referenceDigest.String(), }) } else { - err = meta.SetImageMetaFromInput(localRepo, referenceDigest.String(), referrer.MediaType, + err = meta.SetImageMetaFromInput(context.Background(), localRepo, //nolint:contextcheck + referenceDigest.String(), referrer.MediaType, referenceDigest, referenceBuf, ref.storeController.GetImageStore(localRepo), ref.metaDB, ref.log) } diff --git a/pkg/extensions/sync/references/oras.go b/pkg/extensions/sync/references/oras.go index c5a208d613..12ba3a8e68 100644 --- a/pkg/extensions/sync/references/oras.go +++ b/pkg/extensions/sync/references/oras.go @@ -154,7 +154,8 @@ func (ref ORASReferences) SyncReferences(ctx context.Context, localRepo, remoteR ref.log.Debug().Str("repository", localRepo).Str("subject", subjectDigestStr). Msg("metaDB: trying to sync oras artifact for image") - err := meta.SetImageMetaFromInput(localRepo, referenceDigest.String(), referrer.MediaType, + err := meta.SetImageMetaFromInput(context.Background(), localRepo, //nolint:contextcheck + referenceDigest.String(), referrer.MediaType, referenceDigest, orasBuf, ref.storeController.GetImageStore(localRepo), ref.metaDB, ref.log) if err != nil { diff --git a/pkg/extensions/sync/sync_internal_test.go b/pkg/extensions/sync/sync_internal_test.go index 9b6c2968a8..29ef4e7939 100644 --- a/pkg/extensions/sync/sync_internal_test.go +++ b/pkg/extensions/sync/sync_internal_test.go @@ -336,7 +336,9 @@ func TestLocalRegistry(t *testing.T) { Convey("trigger metaDB error on index manifest in CommitImage()", func() { registry := NewLocalRegistry(storage.StoreController{DefaultStore: syncImgStore}, mocks.MetaDBMock{ - SetRepoReferenceFn: func(repo, Reference string, manifestDigest godigest.Digest, mediaType string) error { + SetRepoReferenceFn: func(ctx context.Context, repo, + Reference string, manifestDigest godigest.Digest, mediaType string, + ) error { if Reference == "1.0" { return zerr.ErrRepoMetaNotFound } @@ -351,7 +353,9 @@ func TestLocalRegistry(t *testing.T) { Convey("trigger metaDB error on image manifest in CommitImage()", func() { registry := NewLocalRegistry(storage.StoreController{DefaultStore: syncImgStore}, mocks.MetaDBMock{ - SetRepoReferenceFn: func(repo, Reference string, manifestDigest godigest.Digest, mediaType string) error { + SetRepoReferenceFn: func(ctx context.Context, repo, + Reference string, manifestDigest godigest.Digest, mediaType string, + ) error { return zerr.ErrRepoMetaNotFound }, }, log) diff --git a/pkg/extensions/sync/sync_test.go b/pkg/extensions/sync/sync_test.go index d3ed4fa578..6ff2641f31 100644 --- a/pkg/extensions/sync/sync_test.go +++ b/pkg/extensions/sync/sync_test.go @@ -883,7 +883,7 @@ func TestOnDemand(t *testing.T) { return nil }, - SetRepoReferenceFn: func(repo, reference string, manifestDigest godigest.Digest, + SetRepoReferenceFn: func(ctx context.Context, repo, reference string, manifestDigest godigest.Digest, mediaType string, ) error { if strings.HasPrefix(reference, "sha256-") && diff --git a/pkg/meta/boltdb/boltdb.go b/pkg/meta/boltdb/boltdb.go index ca14e456da..eb245285de 100644 --- a/pkg/meta/boltdb/boltdb.go +++ b/pkg/meta/boltdb/boltdb.go @@ -471,14 +471,21 @@ func (bdw *BoltDB) RemoveRepoReference(repo, reference string, manifestDigest go return err } -func (bdw *BoltDB) SetRepoReference(repo string, reference string, manifestDigest godigest.Digest, +func (bdw *BoltDB) SetRepoReference(ctx context.Context, repo string, reference string, manifestDigest godigest.Digest, mediaType string, ) error { if err := common.ValidateRepoReferenceInput(repo, reference, manifestDigest); err != nil { return err } - err := bdw.DB.Update(func(tx *bbolt.Tx) error { + var userid string + + userAc, err := reqCtx.UserAcFromContext(ctx) + if err == nil { + userid = userAc.GetUsername() + } + + err = bdw.DB.Update(func(tx *bbolt.Tx) error { buck := tx.Bucket([]byte(RepoMetadataBucket)) repoMetaBlob := buck.Get([]byte(repo)) @@ -507,7 +514,13 @@ func (bdw *BoltDB) SetRepoReference(repo string, reference string, manifestDiges } if _, ok := repoMeta.Statistics[manifestDigest.String()]; !ok { - repoMeta.Statistics[manifestDigest.String()] = mTypes.DescriptorStatistics{DownloadCount: 0} + repoMeta.Statistics[manifestDigest.String()] = mTypes.DescriptorStatistics{ + DownloadCount: 0, + LastPullTimestamp: time.Time{}, + PushTimestamp: time.Now().Local(), + // Q do we care if it was pushed by sync or by metaDB.ParseStorage()? + PushedBy: userid, + } } if _, ok := repoMeta.Signatures[manifestDigest.String()]; !ok { @@ -752,7 +765,7 @@ func (bdw *BoltDB) GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta return foundRepos, err } -func (bdw *BoltDB) IncrementImageDownloads(repo string, reference string) error { +func (bdw *BoltDB) UpdateStatsOnDownload(repo string, reference string) error { err := bdw.DB.Update(func(tx *bbolt.Tx) error { buck := tx.Bucket([]byte(RepoMetadataBucket)) @@ -783,6 +796,7 @@ func (bdw *BoltDB) IncrementImageDownloads(repo string, reference string) error manifestStatistics := repoMeta.Statistics[manifestDigest] manifestStatistics.DownloadCount++ + manifestStatistics.LastPullTimestamp = time.Now().Local() repoMeta.Statistics[manifestDigest] = manifestStatistics repoMetaBlob, err = json.Marshal(repoMeta) diff --git a/pkg/meta/boltdb/boltdb_test.go b/pkg/meta/boltdb/boltdb_test.go index 4729e5a892..eb7fc4b43b 100644 --- a/pkg/meta/boltdb/boltdb_test.go +++ b/pkg/meta/boltdb/boltdb_test.go @@ -335,7 +335,7 @@ func TestWrapperErrors(t *testing.T) { }) So(err, ShouldBeNil) - err = boltdbWrapper.SetRepoReference("repo1", "tag", "digest", ispec.MediaTypeImageManifest) + err = boltdbWrapper.SetRepoReference(context.Background(), "repo1", "tag", "digest", ispec.MediaTypeImageManifest) So(err, ShouldNotBeNil) }) @@ -434,7 +434,7 @@ func TestWrapperErrors(t *testing.T) { So(err, ShouldNotBeNil) }) - Convey("IncrementImageDownloads", func() { + Convey("UpdateStatsOnDownload", func() { err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { repoBuck := tx.Bucket([]byte(boltdb.RepoMetadataBucket)) @@ -442,10 +442,10 @@ func TestWrapperErrors(t *testing.T) { }) So(err, ShouldBeNil) - err = boltdbWrapper.IncrementImageDownloads("repo2", "tag") + err = boltdbWrapper.UpdateStatsOnDownload("repo2", "tag") So(err, ShouldNotBeNil) - err = boltdbWrapper.IncrementImageDownloads("repo1", "tag") + err = boltdbWrapper.UpdateStatsOnDownload("repo1", "tag") So(err, ShouldNotBeNil) err = boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { @@ -455,7 +455,7 @@ func TestWrapperErrors(t *testing.T) { }) So(err, ShouldBeNil) - err = boltdbWrapper.IncrementImageDownloads("repo1", "tag") + err = boltdbWrapper.UpdateStatsOnDownload("repo1", "tag") So(err, ShouldNotBeNil) }) @@ -648,7 +648,8 @@ func TestWrapperErrors(t *testing.T) { Convey("Bad index data", func() { indexDigest := digest.FromString("indexDigest") - err := boltdbWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := boltdbWrapper.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + "tag1", indexDigest, ispec.MediaTypeImageIndex) So(err, ShouldBeNil) err = setBadIndexData(boltdbWrapper.DB, indexDigest.String()) @@ -664,7 +665,8 @@ func TestWrapperErrors(t *testing.T) { Convey("Bad indexBlob in IndexData", func() { indexDigest := digest.FromString("indexDigest") - err := boltdbWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := boltdbWrapper.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + "tag1", indexDigest, ispec.MediaTypeImageIndex) So(err, ShouldBeNil) err = boltdbWrapper.SetIndexData(indexDigest, mTypes.IndexData{ @@ -778,7 +780,8 @@ func TestWrapperErrors(t *testing.T) { Convey("FilterTags bad IndexData", func() { indexDigest := digest.FromString("indexDigest") - err := boltdbWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := boltdbWrapper.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + "tag1", indexDigest, ispec.MediaTypeImageIndex) So(err, ShouldBeNil) err = setBadIndexData(boltdbWrapper.DB, indexDigest.String()) @@ -792,7 +795,8 @@ func TestWrapperErrors(t *testing.T) { Convey("FilterTags bad indexBlob in IndexData", func() { indexDigest := digest.FromString("indexDigest") - err := boltdbWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := boltdbWrapper.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + "tag1", indexDigest, ispec.MediaTypeImageIndex) So(err, ShouldBeNil) err = boltdbWrapper.SetIndexData(indexDigest, mTypes.IndexData{ @@ -812,7 +816,8 @@ func TestWrapperErrors(t *testing.T) { manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndex2") ) - err := boltdbWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := boltdbWrapper.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + "tag1", indexDigest, ispec.MediaTypeImageIndex) So(err, ShouldBeNil) indexBlob, err := GetIndexBlobWithManifests([]digest.Digest{ @@ -979,7 +984,8 @@ func TestWrapperErrors(t *testing.T) { Convey("Unsuported type", func() { digest := digest.FromString("digest") - err := boltdbWrapper.SetRepoReference("repo", "tag1", digest, "invalid type") //nolint:contextcheck + err := boltdbWrapper.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + "tag1", digest, "invalid type") So(err, ShouldBeNil) _, _, _, err = boltdbWrapper.SearchRepos(ctx, "") diff --git a/pkg/meta/dynamodb/dynamodb.go b/pkg/meta/dynamodb/dynamodb.go index d96c2b1dfa..89d068c672 100644 --- a/pkg/meta/dynamodb/dynamodb.go +++ b/pkg/meta/dynamodb/dynamodb.go @@ -520,14 +520,21 @@ func (dwr *DynamoDB) RemoveRepoReference(repo, reference string, manifestDigest return err } -func (dwr *DynamoDB) SetRepoReference(repo string, reference string, manifestDigest godigest.Digest, - mediaType string, +func (dwr *DynamoDB) SetRepoReference(ctx context.Context, repo string, reference string, + manifestDigest godigest.Digest, mediaType string, ) error { if err := common.ValidateRepoReferenceInput(repo, reference, manifestDigest); err != nil { return err } - resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{ + var userid string + + userAc, err := reqCtx.UserAcFromContext(ctx) + if err == nil { + userid = userAc.GetUsername() + } + + resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{ //nolint:contextcheck TableName: aws.String(dwr.RepoMetaTablename), Key: map[string]types.AttributeValue{ "RepoName": &types.AttributeValueMemberS{Value: repo}, @@ -560,7 +567,13 @@ func (dwr *DynamoDB) SetRepoReference(repo string, reference string, manifestDig } if _, ok := repoMeta.Statistics[manifestDigest.String()]; !ok { - repoMeta.Statistics[manifestDigest.String()] = mTypes.DescriptorStatistics{DownloadCount: 0} + repoMeta.Statistics[manifestDigest.String()] = mTypes.DescriptorStatistics{ + DownloadCount: 0, + LastPullTimestamp: time.Time{}, + PushTimestamp: time.Now().Local(), + // Q do we care if it was pushed by sync or by metaDB.ParseStorage()? + PushedBy: userid, + } } if _, ok := repoMeta.Signatures[manifestDigest.String()]; !ok { @@ -571,7 +584,7 @@ func (dwr *DynamoDB) SetRepoReference(repo string, reference string, manifestDig repoMeta.Referrers[manifestDigest.String()] = []mTypes.ReferrerInfo{} } - err = dwr.SetRepoMeta(repo, repoMeta) + err = dwr.SetRepoMeta(repo, repoMeta) //nolint: contextcheck return err } @@ -682,7 +695,7 @@ func (dwr *DynamoDB) GetUserRepoMeta(ctx context.Context, repo string) (mTypes.R return repoMeta, nil } -func (dwr *DynamoDB) IncrementImageDownloads(repo string, reference string) error { +func (dwr *DynamoDB) UpdateStatsOnDownload(repo string, reference string) error { repoMeta, err := dwr.GetRepoMeta(repo) if err != nil { return err @@ -703,6 +716,7 @@ func (dwr *DynamoDB) IncrementImageDownloads(repo string, reference string) erro manifestStatistics := repoMeta.Statistics[descriptorDigest] manifestStatistics.DownloadCount++ + manifestStatistics.LastPullTimestamp = time.Now().Local() repoMeta.Statistics[descriptorDigest] = manifestStatistics return dwr.SetRepoMeta(repo, repoMeta) diff --git a/pkg/meta/dynamodb/dynamodb_test.go b/pkg/meta/dynamodb/dynamodb_test.go index 716dc0394f..b7a3a7efed 100644 --- a/pkg/meta/dynamodb/dynamodb_test.go +++ b/pkg/meta/dynamodb/dynamodb_test.go @@ -67,13 +67,13 @@ func TestIterator(t *testing.T) { So(dynamoWrapper.ResetManifestDataTable(), ShouldBeNil) So(dynamoWrapper.ResetRepoMetaTable(), ShouldBeNil) - err = dynamoWrapper.SetRepoReference("repo1", "tag1", "manifestType", "manifestDigest1") + err = dynamoWrapper.SetRepoReference(context.Background(), "repo1", "tag1", "manifestType", "manifestDigest1") So(err, ShouldBeNil) - err = dynamoWrapper.SetRepoReference("repo2", "tag2", "manifestType", "manifestDigest2") + err = dynamoWrapper.SetRepoReference(context.Background(), "repo2", "tag2", "manifestType", "manifestDigest2") So(err, ShouldBeNil) - err = dynamoWrapper.SetRepoReference("repo3", "tag3", "manifestType", "manifestDigest3") + err = dynamoWrapper.SetRepoReference(context.Background(), "repo3", "tag3", "manifestType", "manifestDigest3") So(err, ShouldBeNil) repoMetaAttributeIterator := mdynamodb.NewBaseDynamoAttributesIterator( @@ -521,7 +521,7 @@ func TestWrapperErrors(t *testing.T) { }) Convey("GetManifestMeta GetManifestData not found error", func() { - err := dynamoWrapper.SetRepoReference("repo", "tag", "dig", "") + err := dynamoWrapper.SetRepoReference(context.Background(), "repo", "tag", "dig", "") So(err, ShouldBeNil) _, err = dynamoWrapper.GetManifestMeta("repo", "dig") @@ -550,7 +550,8 @@ func TestWrapperErrors(t *testing.T) { Convey("SetRepoReference client error", func() { dynamoWrapper.RepoMetaTablename = badTablename digest := digest.FromString("str") - err := dynamoWrapper.SetRepoReference("repo", digest.String(), digest, ispec.MediaTypeImageManifest) + err := dynamoWrapper.SetRepoReference(context.Background(), "repo", + digest.String(), digest, ispec.MediaTypeImageManifest) So(err, ShouldNotBeNil) }) @@ -684,16 +685,16 @@ func TestWrapperErrors(t *testing.T) { So(err, ShouldNotBeNil) }) - Convey("IncrementImageDownloads GetRepoMeta error", func() { - err = dynamoWrapper.IncrementImageDownloads("repoNotFound", "") + Convey("UpdateStatsOnDownload GetRepoMeta error", func() { + err = dynamoWrapper.UpdateStatsOnDownload("repoNotFound", "") So(err, ShouldNotBeNil) }) - Convey("IncrementImageDownloads tag not found error", func() { - err := dynamoWrapper.SetRepoReference("repo", "tag", "dig", "") + Convey("UpdateStatsOnDownload tag not found error", func() { + err := dynamoWrapper.SetRepoReference(context.Background(), "repo", "tag", "dig", "") So(err, ShouldBeNil) - err = dynamoWrapper.IncrementImageDownloads("repo", "notFoundTag") + err = dynamoWrapper.UpdateStatsOnDownload("repo", "notFoundTag") So(err, ShouldNotBeNil) }) @@ -720,7 +721,7 @@ func TestWrapperErrors(t *testing.T) { }) Convey("AddManifestSignature GetRepoMeta error", func() { - err := dynamoWrapper.SetRepoReference("repo", "tag", "dig", "") + err := dynamoWrapper.SetRepoReference(context.Background(), "repo", "tag", "dig", "") So(err, ShouldBeNil) err = dynamoWrapper.AddManifestSignature("repoNotFound", "tag", mTypes.SignatureMetadata{}) @@ -728,7 +729,7 @@ func TestWrapperErrors(t *testing.T) { }) Convey("AddManifestSignature ManifestSignatures signedManifestDigest not found error", func() { - err := dynamoWrapper.SetRepoReference("repo", "tag", "dig", "") + err := dynamoWrapper.SetRepoReference(context.Background(), "repo", "tag", "dig", "") So(err, ShouldBeNil) err = dynamoWrapper.AddManifestSignature("repo", "tagNotFound", mTypes.SignatureMetadata{}) @@ -736,7 +737,7 @@ func TestWrapperErrors(t *testing.T) { }) Convey("AddManifestSignature SignatureType metadb.NotationType", func() { - err := dynamoWrapper.SetRepoReference("repo", "tag", "dig", "") + err := dynamoWrapper.SetRepoReference(context.Background(), "repo", "tag", "dig", "") So(err, ShouldBeNil) err = dynamoWrapper.AddManifestSignature("repo", "tagNotFound", mTypes.SignatureMetadata{ @@ -833,7 +834,8 @@ func TestWrapperErrors(t *testing.T) { }) Convey("SearchRepos GetManifestMeta error", func() { - err := dynamoWrapper.SetRepoReference("repo", "tag1", "notFoundDigest", //nolint:contextcheck + err := dynamoWrapper.SetRepoReference(context.Background(), "repo", "tag1", //nolint:contextcheck + "notFoundDigest", ispec.MediaTypeImageManifest) So(err, ShouldBeNil) @@ -845,7 +847,8 @@ func TestWrapperErrors(t *testing.T) { Convey("Unsuported type", func() { digest := digest.FromString("digest") - err := dynamoWrapper.SetRepoReference("repo", "tag1", digest, "invalid type") //nolint:contextcheck + err := dynamoWrapper.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + "tag1", digest, "invalid type") So(err, ShouldBeNil) _, _, _, err = dynamoWrapper.SearchRepos(ctx, "") @@ -862,7 +865,8 @@ func TestWrapperErrors(t *testing.T) { Convey("SearchRepos bad index data", func() { indexDigest := digest.FromString("indexDigest") - err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := dynamoWrapper.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + "tag1", indexDigest, ispec.MediaTypeImageIndex) So(err, ShouldBeNil) err = setBadIndexData(dynamoWrapper.Client, indexDataTablename, indexDigest.String()) //nolint:contextcheck @@ -875,7 +879,8 @@ func TestWrapperErrors(t *testing.T) { Convey("SearchRepos bad indexBlob in IndexData", func() { indexDigest := digest.FromString("indexDigest") - err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := dynamoWrapper.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + "tag1", indexDigest, ispec.MediaTypeImageIndex) So(err, ShouldBeNil) err = dynamoWrapper.SetIndexData(indexDigest, mTypes.IndexData{ //nolint:contextcheck @@ -897,7 +902,8 @@ func TestWrapperErrors(t *testing.T) { }) Convey("SearchTags GetManifestMeta error", func() { - err := dynamoWrapper.SetRepoReference("repo", "tag1", "manifestNotFound", //nolint:contextcheck + err := dynamoWrapper.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + "tag1", "manifestNotFound", ispec.MediaTypeImageManifest) So(err, ShouldBeNil) @@ -909,7 +915,8 @@ func TestWrapperErrors(t *testing.T) { Convey("SearchTags bad index data", func() { indexDigest := digest.FromString("indexDigest") - err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := dynamoWrapper.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + "tag1", indexDigest, ispec.MediaTypeImageIndex) So(err, ShouldBeNil) err = setBadIndexData(dynamoWrapper.Client, indexDataTablename, indexDigest.String()) //nolint:contextcheck @@ -922,7 +929,8 @@ func TestWrapperErrors(t *testing.T) { Convey("SearchTags bad indexBlob in IndexData", func() { indexDigest := digest.FromString("indexDigest") - err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := dynamoWrapper.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + "tag1", indexDigest, ispec.MediaTypeImageIndex) So(err, ShouldBeNil) err = dynamoWrapper.SetIndexData(indexDigest, mTypes.IndexData{ //nolint:contextcheck @@ -982,7 +990,7 @@ func TestWrapperErrors(t *testing.T) { }) Convey("FilterTags manifestMeta not found", func() { - err := dynamoWrapper.SetRepoReference("repo", "tag1", "manifestNotFound", //nolint:contextcheck + err := dynamoWrapper.SetRepoReference(context.Background(), "repo", "tag1", "manifestNotFound", //nolint:contextcheck ispec.MediaTypeImageManifest) So(err, ShouldBeNil) @@ -995,7 +1003,8 @@ func TestWrapperErrors(t *testing.T) { }) Convey("FilterTags manifestMeta unmarshal error", func() { - err := dynamoWrapper.SetRepoReference("repo", "tag1", "dig", ispec.MediaTypeImageManifest) //nolint:contextcheck + err := dynamoWrapper.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + "tag1", "dig", ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = setBadManifestData(dynamoWrapper.Client, manifestDataTablename, "dig") //nolint:contextcheck @@ -1013,7 +1022,8 @@ func TestWrapperErrors(t *testing.T) { Convey("FilterTags bad IndexData", func() { indexDigest := digest.FromString("indexDigest") - err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := dynamoWrapper.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + "tag1", indexDigest, ispec.MediaTypeImageIndex) So(err, ShouldBeNil) err = setBadIndexData(dynamoWrapper.Client, indexDataTablename, indexDigest.String()) //nolint:contextcheck @@ -1027,7 +1037,8 @@ func TestWrapperErrors(t *testing.T) { Convey("FilterTags bad indexBlob in IndexData", func() { indexDigest := digest.FromString("indexDigest") - err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := dynamoWrapper.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + "tag1", indexDigest, ispec.MediaTypeImageIndex) So(err, ShouldBeNil) err = dynamoWrapper.SetIndexData(indexDigest, mTypes.IndexData{ //nolint:contextcheck @@ -1047,7 +1058,8 @@ func TestWrapperErrors(t *testing.T) { manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndex2") ) - err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := dynamoWrapper.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + "tag1", indexDigest, ispec.MediaTypeImageIndex) So(err, ShouldBeNil) indexBlob, err := GetIndexBlobWithManifests([]digest.Digest{ @@ -1125,7 +1137,8 @@ func TestWrapperErrors(t *testing.T) { }) Convey("GetUserRepoMeta userMeta not found", func() { - err := dynamoWrapper.SetRepoReference("repo", "tag", digest.FromString("1"), ispec.MediaTypeImageManifest) + err := dynamoWrapper.SetRepoReference(context.Background(), "repo", + "tag", digest.FromString("1"), ispec.MediaTypeImageManifest) So(err, ShouldBeNil) dynamoWrapper.UserDataTablename = badTablename diff --git a/pkg/meta/hooks.go b/pkg/meta/hooks.go index 3a37814eb4..8476532eb4 100644 --- a/pkg/meta/hooks.go +++ b/pkg/meta/hooks.go @@ -1,6 +1,8 @@ package meta import ( + "context" + godigest "github.com/opencontainers/go-digest" "zotregistry.io/zot/pkg/log" @@ -12,7 +14,7 @@ import ( // OnUpdateManifest is called when a new manifest is added. It updates metadb according to the type // of image pushed(normal images, signatues, etc.). In care of any errors, it makes sure to keep // consistency between metadb and the image store. -func OnUpdateManifest(repo, reference, mediaType string, digest godigest.Digest, body []byte, +func OnUpdateManifest(ctx context.Context, repo, reference, mediaType string, digest godigest.Digest, body []byte, storeController storage.StoreController, metaDB mTypes.MetaDB, log log.Logger, ) error { imgStore := storeController.GetImageStore(repo) @@ -58,7 +60,7 @@ func OnUpdateManifest(repo, reference, mediaType string, digest godigest.Digest, } } } else { - err = SetImageMetaFromInput(repo, reference, mediaType, digest, body, + err = SetImageMetaFromInput(ctx, repo, reference, mediaType, digest, body, imgStore, metaDB, log) if err != nil { metadataSuccessfullySet = false @@ -155,7 +157,7 @@ func OnGetManifest(name, reference string, body []byte, } if !isSignature { - err := metaDB.IncrementImageDownloads(name, reference) + err := metaDB.UpdateStatsOnDownload(name, reference) if err != nil { log.Error().Err(err).Str("repository", name).Str("reference", reference). Msg("unexpected error for image") diff --git a/pkg/meta/hooks_test.go b/pkg/meta/hooks_test.go index 0530f3e498..096b976a0d 100644 --- a/pkg/meta/hooks_test.go +++ b/pkg/meta/hooks_test.go @@ -1,6 +1,7 @@ package meta_test import ( + "context" "encoding/json" "errors" "testing" @@ -56,7 +57,8 @@ func TestOnUpdateManifest(t *testing.T) { digest := godigest.FromBytes(manifestBlob) - err = meta.OnUpdateManifest("repo", "tag1", "", digest, manifestBlob, storeController, metaDB, log) + err = meta.OnUpdateManifest(context.Background(), "repo", + "tag1", "", digest, manifestBlob, storeController, metaDB, log) So(err, ShouldBeNil) repoMeta, err := metaDB.GetRepoMeta("repo") @@ -78,7 +80,7 @@ func TestOnUpdateManifest(t *testing.T) { }, } - err := meta.OnUpdateManifest("repo", "tag1", ispec.MediaTypeImageManifest, "digest", + err := meta.OnUpdateManifest(context.Background(), "repo", "tag1", ispec.MediaTypeImageManifest, "digest", []byte("{}"), storeController, metaDB, log) So(err, ShouldNotBeNil) }) @@ -103,7 +105,7 @@ func TestUpdateErrors(t *testing.T) { return nil } - err := meta.OnUpdateManifest("repo", "tag1", "digest", "media", badManifestBlob, + err := meta.OnUpdateManifest(context.Background(), "repo", "tag1", "digest", "media", badManifestBlob, storeController, metaDB, log) So(err, ShouldNotBeNil) }) @@ -124,7 +126,7 @@ func TestUpdateErrors(t *testing.T) { return badNotationManifestBlob, "", "", nil } - err = meta.OnUpdateManifest("repo", "tag1", "", "digest", badNotationManifestBlob, + err = meta.OnUpdateManifest(context.Background(), "repo", "tag1", "", "digest", badNotationManifestBlob, storeController, metaDB, log) So(err, ShouldNotBeNil) }) @@ -156,7 +158,7 @@ func TestUpdateErrors(t *testing.T) { return ErrTestError } - err = meta.OnUpdateManifest("repo", "tag1", "", "digest", notationManifestBlob, + err = meta.OnUpdateManifest(context.Background(), "repo", "tag1", "", "digest", notationManifestBlob, storeController, metaDB, log) So(err, ShouldNotBeNil) }) @@ -216,7 +218,7 @@ func TestUpdateErrors(t *testing.T) { metaDB := mocks.MetaDBMock{} log := log.NewLogger("debug", "") - err := meta.SetImageMetaFromInput("repo", "ref", ispec.MediaTypeImageManifest, "digest", + err := meta.SetImageMetaFromInput(context.Background(), "repo", "ref", ispec.MediaTypeImageManifest, "digest", []byte("BadManifestBlob"), imageStore, metaDB, log) So(err, ShouldNotBeNil) @@ -233,8 +235,8 @@ func TestUpdateErrors(t *testing.T) { return []byte("{}"), nil } - err = meta.SetImageMetaFromInput("repo", string(godigest.FromString("reference")), "", "digest", - manifestBlob, imageStore, metaDB, log) + err = meta.SetImageMetaFromInput(context.Background(), "repo", + string(godigest.FromString("reference")), "", "digest", manifestBlob, imageStore, metaDB, log) So(err, ShouldBeNil) }) @@ -247,7 +249,7 @@ func TestUpdateErrors(t *testing.T) { return ErrTestError }, } - err := meta.SetImageMetaFromInput("repo", "ref", ispec.MediaTypeImageManifest, "digest", + err := meta.SetImageMetaFromInput(context.Background(), "repo", "ref", ispec.MediaTypeImageManifest, "digest", []byte("{}"), imageStore, metaDB, log) So(err, ShouldNotBeNil) }) @@ -261,7 +263,7 @@ func TestUpdateErrors(t *testing.T) { return ErrTestError }, } - err := meta.SetImageMetaFromInput("repo", "ref", ispec.MediaTypeImageIndex, "digest", + err := meta.SetImageMetaFromInput(context.Background(), "repo", "ref", ispec.MediaTypeImageIndex, "digest", []byte("{}"), imageStore, metaDB, log) So(err, ShouldNotBeNil) }) @@ -280,7 +282,7 @@ func TestUpdateErrors(t *testing.T) { }, } - err := meta.SetImageMetaFromInput("repo", "ref", ispec.MediaTypeImageManifest, "digest", + err := meta.SetImageMetaFromInput(context.Background(), "repo", "ref", ispec.MediaTypeImageManifest, "digest", []byte(`{"subject": {"digest": "subjDigest"}}`), imageStore, metaDB, log) So(err, ShouldNotBeNil) }) diff --git a/pkg/meta/meta_test.go b/pkg/meta/meta_test.go index 8697ff3c89..6f562f57ab 100644 --- a/pkg/meta/meta_test.go +++ b/pkg/meta/meta_test.go @@ -528,7 +528,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func ) Convey("Setting a good repo", func() { - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) repoMeta, err := metaDB.GetRepoMeta(repo1) @@ -554,7 +554,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func So(err, ShouldNotBeNil) digest := godigest.FromString("digest") - err = metaDB.SetRepoReference(repo1, digest.String(), digest, + err = metaDB.SetRepoReference(context.Background(), repo1, digest.String(), digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) @@ -564,9 +564,9 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func }) Convey("Set multiple tags for repo", func() { - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) repoMeta, err := metaDB.GetRepoMeta(repo1) @@ -576,9 +576,9 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func }) Convey("Set multiple repos", func() { - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, tag2, manifestDigest2, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo2, tag2, manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) repoMeta1, err := metaDB.GetRepoMeta(repo1) @@ -592,17 +592,17 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func Convey("Setting a repo with invalid fields", func() { Convey("Repo name is not valid", func() { - err := metaDB.SetRepoReference("", tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), "", tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldNotBeNil) }) Convey("Tag is not valid", func() { - err := metaDB.SetRepoReference(repo1, "", manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, "", manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldNotBeNil) }) Convey("Manifest Digest is not valid", func() { - err := metaDB.SetRepoReference(repo1, tag1, "", ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, tag1, "", ispec.MediaTypeImageManifest) So(err, ShouldNotBeNil) }) }) @@ -621,10 +621,10 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func InexistentRepo = "InexistentRepo" ) - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, tag2, manifestDigest2, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo2, tag2, manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) Convey("Get a existent repo", func() { @@ -653,10 +653,10 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func manifestDigest2 = godigest.FromString("fake-manifest2") ) - err := metaDB.SetRepoReference(repo, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo, tag2, manifestDigest2, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo, tag2, manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) Convey("Delete reference from repo", func() { @@ -763,13 +763,13 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func manifestDigest2 = godigest.FromString("fake-manifest2") ) - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, tag2, manifestDigest2, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo2, tag2, manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) Convey("Get all Repometa", func() { @@ -803,7 +803,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func manifestDigest1 = godigest.FromString("fake-manifest1") ) - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = metaDB.IncrementRepoStars(repo1) @@ -835,7 +835,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func manifestDigest1 = godigest.FromString("fake-manifest1") ) - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = metaDB.IncrementRepoStars(repo1) @@ -870,7 +870,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func manifestDigest1 = godigest.FromString("fake-manifest1") ) - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = metaDB.IncrementRepoStars(repo1) @@ -930,10 +930,10 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func // anonymous user ctx3 := userAc.DeriveContext(context.Background()) - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo2, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) starCount, err := metaDB.GetRepoStars(repo1) @@ -1167,10 +1167,10 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func // anonymous user ctx3 := userAc.DeriveContext(context.Background()) - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo2, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) repos, err := metaDB.GetBookmarkedRepos(ctx1) @@ -1274,7 +1274,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func So(len(repos), ShouldEqual, 0) }) - Convey("Test IncrementImageDownloads", func() { + Convey("Test UpdateStatsOnDownload", func() { var ( repo1 = "repo1" tag1 = "0.0.1" @@ -1285,7 +1285,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func manifestDigest := godigest.FromBytes(manifestBlob) - err = metaDB.SetRepoReference(repo1, tag1, manifestDigest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, tag1, manifestDigest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = metaDB.SetManifestMeta(repo1, manifestDigest, mTypes.ManifestMetadata{ @@ -1294,7 +1294,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func }) So(err, ShouldBeNil) - err = metaDB.IncrementImageDownloads(repo1, tag1) + err = metaDB.UpdateStatsOnDownload(repo1, tag1) So(err, ShouldBeNil) repoMeta, err := metaDB.GetRepoMeta(repo1) @@ -1302,13 +1302,14 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func So(repoMeta.Statistics[manifestDigest.String()].DownloadCount, ShouldEqual, 1) - err = metaDB.IncrementImageDownloads(repo1, tag1) + err = metaDB.UpdateStatsOnDownload(repo1, tag1) So(err, ShouldBeNil) repoMeta, err = metaDB.GetRepoMeta(repo1) So(err, ShouldBeNil) So(repoMeta.Statistics[manifestDigest.String()].DownloadCount, ShouldEqual, 2) + So(time.Now().Local(), ShouldHappenAfter, repoMeta.Statistics[manifestDigest.String()].LastPullTimestamp) _, err = metaDB.GetManifestMeta(repo1, "badManiestDigest") So(err, ShouldNotBeNil) @@ -1321,7 +1322,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func manifestDigest1 = godigest.FromString("fake-manifest1") ) - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = metaDB.SetManifestMeta(repo1, manifestDigest1, mTypes.ManifestMetadata{}) @@ -1350,7 +1351,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func manifestDigest1 = godigest.FromString("dig") ) - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = metaDB.SetManifestMeta(repo1, manifestDigest1, mTypes.ManifestMetadata{ @@ -1385,7 +1386,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func repo := "repo" tag := "0.0.1" - err := metaDB.SetRepoReference(repo, tag, manifestDigest, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo, tag, manifestDigest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = metaDB.SetManifestMeta(repo, manifestDigest, mTypes.ManifestMetadata{ @@ -1496,7 +1497,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func }) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = metaDB.SetManifestData(manifestDigest1, mTypes.ManifestData{}) @@ -1518,7 +1519,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func manifestDigest1 = godigest.FromString("fake-manifest1") ) - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = metaDB.SetManifestData(manifestDigest1, mTypes.ManifestData{}) @@ -1579,11 +1580,14 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func } Convey("Search all repos", func() { - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + tag2, manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, tag3, manifestDigest3, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo2, //nolint:contextcheck + tag3, manifestDigest3, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = metaDB.SetManifestMeta(repo1, manifestDigest1, emptyRepoMeta) @@ -1603,7 +1607,8 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func }) Convey("Search a repo by name", func() { - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = metaDB.SetManifestMeta(repo1, manifestDigest1, emptyRepoMeta) @@ -1617,10 +1622,12 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func }) Convey("Search non-existing repo by name", func() { - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + tag2, manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) repos, manifestMetaMap, _, err := metaDB.SearchRepos(ctx, "RepoThatDoesntExist") @@ -1630,11 +1637,14 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func }) Convey("Search with partial match", func() { - err := metaDB.SetRepoReference("alpine", tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), "alpine", //nolint:contextcheck + tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("pine", tag2, manifestDigest2, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "pine", //nolint:contextcheck + tag2, manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("golang", tag3, manifestDigest3, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "golang", //nolint:contextcheck + tag3, manifestDigest3, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = metaDB.SetManifestMeta("alpine", manifestDigest1, emptyRepoMeta) @@ -1653,11 +1663,14 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func }) Convey("Search multiple repos that share manifests", func() { - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, tag2, manifestDigest1, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo2, //nolint:contextcheck + tag2, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo3, tag3, manifestDigest1, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo3, //nolint:contextcheck + tag3, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = metaDB.SetManifestMeta(repo1, manifestDigest1, emptyRepoMeta) @@ -1674,11 +1687,14 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func }) Convey("Search repos with access control", func() { - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, tag2, manifestDigest1, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo2, //nolint:contextcheck + tag2, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo3, tag3, manifestDigest1, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo3, //nolint:contextcheck + tag3, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = metaDB.SetManifestMeta(repo1, manifestDigest1, emptyRepoMeta) @@ -1763,10 +1779,12 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func }) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo", tag4, indexDigest, ispec.MediaTypeImageIndex) + err = metaDB.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + tag4, indexDigest, ispec.MediaTypeImageIndex) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo", tag5, manifestDigest3, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + tag5, manifestDigest3, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) repos, manifestMetaMap, indexDataMap, err := metaDB.SearchRepos(ctx, "repo") @@ -1806,17 +1824,23 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func ConfigBlob: emptyConfigBlob, } - err = metaDB.SetRepoReference(repo1, "0.0.1", manifestDigest1, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + "0.0.1", manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, "0.0.2", manifestDigest3, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + "0.0.2", manifestDigest3, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, "0.1.0", manifestDigest2, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + "0.1.0", manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, "1.0.0", manifestDigest2, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + "1.0.0", manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, "1.0.1", manifestDigest2, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + "1.0.1", manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, "0.0.1", manifestDigest3, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo2, //nolint:contextcheck + "0.0.1", manifestDigest3, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = metaDB.SetManifestMeta(repo1, manifestDigest1, emptyRepoMeta) @@ -1906,11 +1930,14 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func tag3 = "0.0.3" ) - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, tag2, manifestDigest1, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo2, //nolint:contextcheck + tag2, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo3, tag3, manifestDigest1, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo3, //nolint:contextcheck + tag3, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) config := ispec.Image{} @@ -2003,13 +2030,16 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func }) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo", tag4, indexDigest, ispec.MediaTypeImageIndex) + err = metaDB.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + tag4, indexDigest, ispec.MediaTypeImageIndex) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo", tag5, manifestDigest3, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + tag5, manifestDigest3, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo", tag6, manifestDigest4, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + tag6, manifestDigest4, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) repos, manifestMetaMap, indexDataMap, err := metaDB.SearchTags(ctx, "repo:0.0") @@ -2039,15 +2069,20 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func tag5 = "0.0.5" ) - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, tag2, manifestDigest1, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + tag2, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, tag3, manifestDigest1, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + tag3, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, tag4, manifestDigest1, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + tag4, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, tag5, manifestDigest1, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + tag5, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) config := ispec.Image{} @@ -2119,7 +2154,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func ConfigBlob: emptyConfigBlob, } - err = metaDB.SetRepoReference(repo1, "2.0.0", indexDigest, ispec.MediaTypeImageIndex) + err = metaDB.SetRepoReference(context.Background(), repo1, "2.0.0", indexDigest, ispec.MediaTypeImageIndex) So(err, ShouldBeNil) indexBlob, err := GetIndexBlobWithManifests([]godigest.Digest{ @@ -2133,17 +2168,23 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func }) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, "0.0.1", manifestDigest1, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + "0.0.1", manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, "0.0.2", manifestDigest3, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + "0.0.2", manifestDigest3, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, "0.1.0", manifestDigest2, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + "0.1.0", manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, "1.0.0", manifestDigest2, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + "1.0.0", manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, "1.0.1", manifestDigest2, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + "1.0.1", manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, "0.0.1", manifestDigest3, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo2, //nolint:contextcheck + "0.0.1", manifestDigest3, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = metaDB.SetManifestMeta(repo1, manifestDigest1, emptyManifestMeta) @@ -2289,7 +2330,8 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func err = metaDB.SetManifestData(referredDigest, manifestData) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo", "tag", referredDigest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + "tag", referredDigest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) // ------- Add Artifact 1 @@ -2463,10 +2505,12 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func So(err, ShouldBeNil) } - err = metaDB.SetRepoReference("repo", img.DigestStr(), imgDigest, img.Manifest.MediaType) + err = metaDB.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + img.DigestStr(), imgDigest, img.Manifest.MediaType) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo", multiarch.DigestStr(), multiarchDigest, ispec.MediaTypeImageIndex) + err = metaDB.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + multiarch.DigestStr(), multiarchDigest, ispec.MediaTypeImageIndex) So(err, ShouldBeNil) repoMetas, _, _, err := metaDB.FilterRepos(context.Background(), @@ -2492,7 +2536,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func }) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo99, "tag", manifestDigest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo99, "tag", manifestDigest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) repoMetas, _, _, err := metaDB.SearchRepos(ctx, repo99) @@ -2567,7 +2611,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func digest := godigest.FromString("1") - err := metaDB.SetRepoReference("repo", "tag", digest, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), "repo", "tag", digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) _, err = metaDB.ToggleBookmarkRepo(ctx, "repo") diff --git a/pkg/meta/parse.go b/pkg/meta/parse.go index d9317f65cf..c10c7ee80b 100644 --- a/pkg/meta/parse.go +++ b/pkg/meta/parse.go @@ -1,6 +1,7 @@ package meta import ( + "context" "encoding/json" "errors" "fmt" @@ -134,8 +135,8 @@ func ParseRepo(repo string, metaDB mTypes.MetaDB, storeController storage.StoreC reference = descriptor.Digest.String() } - err = SetImageMetaFromInput(repo, reference, descriptor.MediaType, descriptor.Digest, descriptorBlob, - imageStore, metaDB, log) + err = SetImageMetaFromInput(context.Background(), repo, reference, descriptor.MediaType, + descriptor.Digest, descriptorBlob, imageStore, metaDB, log) if err != nil { log.Error().Err(err).Str("repository", repo).Str("tag", tag). Msg("load-repo: failed to set metadata for image") @@ -380,8 +381,8 @@ func NewIndexData(repoName string, indexBlob []byte, imageStore storageTypes.Ima // SetMetadataFromInput tries to set manifest metadata and update repo metadata by adding the current tag // (in case the reference is a tag). The function expects image manifests and indexes (multi arch images). -func SetImageMetaFromInput(repo, reference, mediaType string, digest godigest.Digest, descriptorBlob []byte, - imageStore storageTypes.ImageStore, metaDB mTypes.MetaDB, log log.Logger, +func SetImageMetaFromInput(ctx context.Context, repo, reference, mediaType string, digest godigest.Digest, + descriptorBlob []byte, imageStore storageTypes.ImageStore, metaDB mTypes.MetaDB, log log.Logger, ) error { switch mediaType { case ispec.MediaTypeImageManifest: @@ -417,7 +418,7 @@ func SetImageMetaFromInput(repo, reference, mediaType string, digest godigest.Di } } - err = metaDB.SetRepoReference(repo, reference, digest, mediaType) + err = metaDB.SetRepoReference(ctx, repo, reference, digest, mediaType) if err != nil { log.Error().Err(err).Msg("metadb: error while putting repo meta") diff --git a/pkg/meta/parse_test.go b/pkg/meta/parse_test.go index fcccea0bf0..38bac696b1 100644 --- a/pkg/meta/parse_test.go +++ b/pkg/meta/parse_test.go @@ -8,6 +8,7 @@ import ( "os" "path" "testing" + "time" godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -160,7 +161,9 @@ func TestParseStorageErrors(t *testing.T) { } Convey("metaDB.SetRepoReference", func() { - metaDB.SetRepoReferenceFn = func(repo, tag string, manifestDigest godigest.Digest, mediaType string) error { + metaDB.SetRepoReferenceFn = func(ctx context.Context, repo, //nolint:contextcheck + tag string, manifestDigest godigest.Digest, mediaType string, + ) error { return ErrTestError } @@ -557,16 +560,16 @@ func RunParseStorageTests(rootDir string, metaDB mTypes.MetaDB) { err = WriteImageToFileSystem(image, repo, "tag", storeController) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo, "tag", manifestDigest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo, "tag", manifestDigest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = metaDB.IncrementRepoStars(repo) So(err, ShouldBeNil) - err = metaDB.IncrementImageDownloads(repo, "tag") + err = metaDB.UpdateStatsOnDownload(repo, "tag") So(err, ShouldBeNil) - err = metaDB.IncrementImageDownloads(repo, "tag") + err = metaDB.UpdateStatsOnDownload(repo, "tag") So(err, ShouldBeNil) - err = metaDB.IncrementImageDownloads(repo, "tag") + err = metaDB.UpdateStatsOnDownload(repo, "tag") So(err, ShouldBeNil) repoMeta, err := metaDB.GetRepoMeta(repo) @@ -574,6 +577,7 @@ func RunParseStorageTests(rootDir string, metaDB mTypes.MetaDB) { So(repoMeta.Statistics[manifestDigest.String()].DownloadCount, ShouldEqual, 3) So(repoMeta.Stars, ShouldEqual, 1) + So(time.Now().Local(), ShouldHappenAfter, repoMeta.Statistics[manifestDigest.String()].LastPullTimestamp) err = meta.ParseStorage(metaDB, storeController, log.NewLogger("debug", "")) So(err, ShouldBeNil) diff --git a/pkg/meta/types/types.go b/pkg/meta/types/types.go index 6e64280bef..4b036a0fbc 100644 --- a/pkg/meta/types/types.go +++ b/pkg/meta/types/types.go @@ -43,7 +43,8 @@ type MetaDB interface { //nolint:interfacebloat GetRepoStars(repo string) (int, error) // SetRepoReference sets the reference of a manifest in the tag list of a repo - SetRepoReference(repo string, reference string, manifestDigest godigest.Digest, mediaType string) error + SetRepoReference(ctx context.Context, repo string, reference string, + manifestDigest godigest.Digest, mediaType string) error /* RemoveRepoReference removes the tag from RepoMetadata if the reference is a tag, @@ -102,8 +103,8 @@ type MetaDB interface { //nolint:interfacebloat GetReferrersInfo(repo string, referredDigest godigest.Digest, artifactTypes []string) ( []ReferrerInfo, error) - // IncrementManifestDownloads adds 1 to the download count of a manifest - IncrementImageDownloads(repo string, reference string) error + // UpdateStatsOnDownload adds 1 to the download count of a manifest and sets the timestamp of download + UpdateStatsOnDownload(repo string, reference string) error // AddManifestSignature adds signature metadata to a given manifest in the database AddManifestSignature(repo string, signedManifestDigest godigest.Digest, sm SignatureMetadata) error @@ -212,7 +213,10 @@ type Descriptor struct { } type DescriptorStatistics struct { - DownloadCount int + DownloadCount int + LastPullTimestamp time.Time + PushTimestamp time.Time + PushedBy string } type ManifestSignatures map[string][]SignatureInfo diff --git a/pkg/storage/gc/gc.go b/pkg/storage/gc/gc.go index 220bfd6894..b16610f163 100644 --- a/pkg/storage/gc/gc.go +++ b/pkg/storage/gc/gc.go @@ -15,6 +15,7 @@ import ( oras "github.com/oras-project/artifacts-spec/specs-go/v1" zerr "zotregistry.io/zot/errors" + "zotregistry.io/zot/pkg/api/config" zcommon "zotregistry.io/zot/pkg/common" zlog "zotregistry.io/zot/pkg/log" mTypes "zotregistry.io/zot/pkg/meta/types" @@ -30,28 +31,32 @@ const ( ) type Options struct { - // will garbage collect referrers with missing subject older than Delay - Referrers bool + DryRun bool // will garbage collect blobs older than Delay Delay time.Duration // will garbage collect untagged manifests older than RetentionDelay - RetentionDelay time.Duration + UntaggedDelay time.Duration + + ImageRetention config.ImageRetention } type GarbageCollect struct { - imgStore types.ImageStore - opts Options - metaDB mTypes.MetaDB - log zlog.Logger + imgStore types.ImageStore + opts Options + metaDB mTypes.MetaDB + policyMgr policyManager + log zlog.Logger } func NewGarbageCollect(imgStore types.ImageStore, metaDB mTypes.MetaDB, opts Options, log zlog.Logger, ) GarbageCollect { + // init rules return GarbageCollect{ - imgStore: imgStore, - metaDB: metaDB, - opts: opts, - log: log, + imgStore: imgStore, + metaDB: metaDB, + opts: opts, + policyMgr: policyManager{opts.ImageRetention.Policies, log}, + log: log, } } @@ -120,11 +125,18 @@ func (gc GarbageCollect) cleanRepo(repo string) error { return err } - // update repos's index.json in storage - if err := gc.imgStore.PutIndexContent(repo, index); err != nil { + // apply tags retention + if err := gc.retainTags(repo, &index); err != nil { return err } + if !gc.opts.DryRun { + // update repos's index.json in storage + if err := gc.imgStore.PutIndexContent(repo, index); err != nil { + return err + } + } + // gc unreferenced blobs if err := gc.cleanBlobs(repo, index, gc.opts.Delay, gc.log); err != nil { return err @@ -142,7 +154,9 @@ func (gc GarbageCollect) cleanManifests(repo string, index *ispec.Index) error { for !stop { var gcedReferrer bool - if gc.opts.Referrers { + var gcedManifest bool + + if gc.policyMgr.hasDeleteReferrer(repo) { gc.log.Debug().Str("repository", repo).Msg("gc: manifests with missing referrers") gcedReferrer, err = gc.cleanIndexReferrers(repo, index, *index) @@ -151,18 +165,20 @@ func (gc GarbageCollect) cleanManifests(repo string, index *ispec.Index) error { } } - referenced := make(map[godigest.Digest]bool, 0) + if gc.policyMgr.hasDeleteUntagged(repo) { + referenced := make(map[godigest.Digest]bool, 0) - /* gather all manifests referenced in multiarch images/by other manifests - so that we can skip them in cleanUntaggedManifests */ - if err := gc.identifyManifestsReferencedInIndex(*index, repo, referenced); err != nil { - return err - } + /* gather all manifests referenced in multiarch images/by other manifests + so that we can skip them in cleanUntaggedManifests */ + if err := gc.identifyManifestsReferencedInIndex(*index, repo, referenced); err != nil { + return err + } - // apply image retention policy - gcedManifest, err := gc.cleanUntaggedManifests(repo, index, referenced) - if err != nil { - return err + // apply image retention policy + gcedManifest, err = gc.cleanUntaggedManifests(repo, index, referenced) + if err != nil { + return err + } } /* if we gced any manifest then loop again and gc manifests with @@ -263,11 +279,18 @@ func (gc GarbageCollect) cleanReferrer(repo string, index *ispec.Index, manifest if err != nil { return false, err } + + if gced { + gc.log.Info().Str("repository", repo). + Str("reference", manifestDesc.Digest.String()). + Str("subject", subject.Digest.String()). + Str("policy", "deleteReferrers").Msg("gc: removed manifest without reference") + } } } // cosign - tag, ok := manifestDesc.Annotations[ispec.AnnotationRefName] + tag, ok := getDescriptorTag(manifestDesc) if ok { if strings.HasPrefix(tag, "sha256-") && (strings.HasSuffix(tag, cosignSignatureTagSuffix) || strings.HasSuffix(tag, SBOMTagSuffix)) { @@ -279,6 +302,13 @@ func (gc GarbageCollect) cleanReferrer(repo string, index *ispec.Index, manifest if err != nil { return false, err } + + if gced { + gc.log.Info().Str("repository", repo). + Str("reference", tag). + Str("subject", subjectDigest.String()). + Str("policy", "deleteReferrers").Msg("gc: removed cosign manifest without reference") + } } } } @@ -295,13 +325,13 @@ func (gc GarbageCollect) gcManifest(repo string, index *ispec.Index, desc ispec. canGC, err := isBlobOlderThan(gc.imgStore, repo, desc.Digest, delay, gc.log) if err != nil { gc.log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()). - Str("delay", gc.opts.Delay.String()).Msg("gc: failed to check if blob is older than delay") + Str("delay", delay.String()).Msg("gc: failed to check if blob is older than delay") return false, err } if canGC { - if gced, err = gc.removeManifest(repo, index, desc, signatureType, subjectDigest); err != nil { + if gced, err = gc.removeManifest(repo, index, desc, desc.Digest.String(), signatureType, subjectDigest); err != nil { return false, err } } @@ -311,20 +341,24 @@ func (gc GarbageCollect) gcManifest(repo string, index *ispec.Index, desc ispec. // removeManifest removes a manifest entry from an index and syncs metaDB accordingly. func (gc GarbageCollect) removeManifest(repo string, index *ispec.Index, - desc ispec.Descriptor, signatureType string, subjectDigest godigest.Digest, + desc ispec.Descriptor, reference string, signatureType string, subjectDigest godigest.Digest, ) (bool, error) { - gc.log.Debug().Str("repository", repo).Str("digest", desc.Digest.String()).Msg("gc: removing manifest") + gc.log.Debug().Str("repository", repo).Bool("dryRun", gc.opts.DryRun). + Str("reference", reference).Msg("gc: removing manifest") - // remove from index - _, err := common.RemoveManifestDescByReference(index, desc.Digest.String(), true) + _, err := common.RemoveManifestDescByReference(index, reference, true) if err != nil { - if errors.Is(err, zerr.ErrManifestConflict) { + if errors.Is(err, zerr.ErrManifestConflict) { // check for manifest not found? { return false, nil } return false, err } + if gc.opts.DryRun { + return true, nil + } + // sync metaDB if gc.metaDB != nil { if signatureType != "" { @@ -338,9 +372,9 @@ func (gc GarbageCollect) removeManifest(repo string, index *ispec.Index, return false, err } } else { - err := gc.metaDB.RemoveRepoReference(repo, desc.Digest.String(), desc.Digest) + err := gc.metaDB.RemoveRepoReference(repo, reference, desc.Digest) if err != nil { - gc.log.Error().Err(err).Msg("gc, metadb: unable to remove repo reference in metaDB") + gc.log.Error().Err(err).Msg("gc,metadb: unable to remove repo reference in metaDB") return false, err } @@ -359,7 +393,6 @@ func (gc GarbageCollect) cleanUntaggedManifests(repo string, index *ispec.Index, gc.log.Debug().Str("repository", repo).Msg("gc: manifests without tags") - // first gather manifests part of image indexes and referrers, we want to skip checking them for _, desc := range index.Manifests { // skip manifests referenced in image indexes if _, referenced := referenced[desc.Digest]; referenced { @@ -368,12 +401,18 @@ func (gc GarbageCollect) cleanUntaggedManifests(repo string, index *ispec.Index, // remove untagged images if desc.MediaType == ispec.MediaTypeImageManifest || desc.MediaType == ispec.MediaTypeImageIndex { - _, ok := desc.Annotations[ispec.AnnotationRefName] + _, ok := getDescriptorTag(desc) if !ok { - gced, err = gc.gcManifest(repo, index, desc, "", "", gc.opts.RetentionDelay) + gced, err = gc.gcManifest(repo, index, desc, "", "", gc.opts.UntaggedDelay) if err != nil { return false, err } + + if gced { + gc.log.Info().Str("repository", repo). + Str("reference", desc.Digest.String()). + Str("policy", "deleteUntagged").Msg("gc: removed untagged manifest") + } } } } @@ -484,7 +523,8 @@ func (gc GarbageCollect) cleanBlobs(repo string, index ispec.Index, return err } - log.Info().Str("repository", repo).Int("count", reaped).Msg("gc: garbage collected blobs") + log.Info().Str("repository", repo).Int("count", reaped). + Msg("gc: garbage collected blobs") return nil } @@ -596,6 +636,36 @@ func (gc GarbageCollect) addORASImageManifestBlobsToReferences(repo string, mdig return nil } +func (gc GarbageCollect) retainTags(repo string, index *ispec.Index) error { + if !gc.policyMgr.hasTagRetention(repo) { + return nil + } + + repoMeta, err := gc.metaDB.GetUserRepoMeta(context.Background(), repo) + if err != nil { + gc.log.Error().Err(err).Str("repository", repo).Msg("gc: can't retrieve repoMeta for repo") + + return err + } + + retainTags := gc.policyMgr.getRetainedTags(repoMeta, *index) + + // remove + for _, desc := range index.Manifests { + // check tag + tag, ok := getDescriptorTag(desc) + if ok && !zcommon.Contains(retainTags, tag) { + // remove tags which should not be retained + _, err := gc.removeManifest(repo, index, desc, tag, "", "") + if err != nil && !errors.Is(err, zerr.ErrManifestNotFound) { + return err + } + } + } + + return nil +} + func isManifestReferencedInIndex(index *ispec.Index, digest godigest.Digest) bool { for _, manifest := range index.Manifests { if manifest.Digest == digest { @@ -631,6 +701,25 @@ func getSubjectFromCosignTag(tag string) godigest.Digest { return godigest.NewDigestFromEncoded(godigest.Algorithm(alg), encoded) } +func getDescriptorTag(desc ispec.Descriptor) (string, bool) { + tag, ok := desc.Annotations[ispec.AnnotationRefName] + + return tag, ok +} + +func getIndexTags(index ispec.Index) []string { + tags := make([]string, 0) + + for _, desc := range index.Manifests { + tag, ok := getDescriptorTag(desc) + if ok { + tags = append(tags, tag) + } + } + + return tags +} + /* GCTaskGenerator takes all repositories found in the storage.imagestore @@ -704,5 +793,5 @@ func NewGCTask(imgStore types.ImageStore, gc GarbageCollect, repo string, func (gct *gcTask) DoWork(ctx context.Context) error { // run task - return gct.gc.CleanRepo(gct.repo) + return gct.gc.CleanRepo(gct.repo) //nolint: contextcheck } diff --git a/pkg/storage/gc/gc_internal_test.go b/pkg/storage/gc/gc_internal_test.go index 9618f32331..0b72d62c66 100644 --- a/pkg/storage/gc/gc_internal_test.go +++ b/pkg/storage/gc/gc_internal_test.go @@ -15,6 +15,7 @@ import ( "github.com/rs/zerolog" . "github.com/smartystreets/goconvey/convey" + "zotregistry.io/zot/pkg/api/config" zcommon "zotregistry.io/zot/pkg/common" "zotregistry.io/zot/pkg/extensions/monitoring" "zotregistry.io/zot/pkg/log" @@ -47,9 +48,16 @@ func TestGarbageCollectManifestErrors(t *testing.T) { imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, Options{ - Referrers: true, - Delay: storageConstants.DefaultGCDelay, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + Delay: storageConstants.DefaultGCDelay, + UntaggedDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + ImageRetention: config.ImageRetention{ + Policies: []config.GCPolicy{ + { + RepoNames: []string{"**"}, + DeleteReferrers: true, + }, + }, + }, }, log) Convey("trigger repo not found in addImageIndexBlobsToReferences()", func() { @@ -174,9 +182,16 @@ func TestGarbageCollectIndexErrors(t *testing.T) { imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, Options{ - Referrers: true, - Delay: storageConstants.DefaultGCDelay, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + Delay: storageConstants.DefaultGCDelay, + UntaggedDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + ImageRetention: config.ImageRetention{ + Policies: []config.GCPolicy{ + { + RepoNames: []string{"**"}, + DeleteReferrers: true, + }, + }, + }, }, log) content := []byte("this is a blob") @@ -273,6 +288,19 @@ func TestGarbageCollectWithMockedImageStore(t *testing.T) { Convey("Cover gc error paths", t, func(c C) { log := log.Logger{Logger: zerolog.New(os.Stdout)} + gcOptions := Options{ + Delay: storageConstants.DefaultGCDelay, + UntaggedDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + ImageRetention: config.ImageRetention{ + Policies: []config.GCPolicy{ + { + RepoNames: []string{"**"}, + DeleteReferrers: true, + }, + }, + }, + } + Convey("Error on PutIndexContent in gc.cleanRepo()", func() { returnedIndexJSON := ispec.Index{} @@ -288,11 +316,7 @@ func TestGarbageCollectWithMockedImageStore(t *testing.T) { }, } - gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, Options{ - Referrers: true, - Delay: storageConstants.DefaultGCDelay, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, - }, log) + gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gcOptions, log) err = gc.cleanRepo(repoName) So(err, ShouldNotBeNil) @@ -316,11 +340,7 @@ func TestGarbageCollectWithMockedImageStore(t *testing.T) { }, } - gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, Options{ - Referrers: true, - Delay: storageConstants.DefaultGCDelay, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, - }, log) + gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gcOptions, log) err = gc.cleanRepo(repoName) So(err, ShouldNotBeNil) @@ -333,11 +353,7 @@ func TestGarbageCollectWithMockedImageStore(t *testing.T) { }, } - gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, Options{ - Referrers: true, - Delay: storageConstants.DefaultGCDelay, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, - }, log) + gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gcOptions, log) err := gc.cleanRepo(repoName) So(err, ShouldNotBeNil) @@ -369,11 +385,15 @@ func TestGarbageCollectWithMockedImageStore(t *testing.T) { }, } - gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, Options{ - Referrers: false, - Delay: storageConstants.DefaultGCDelay, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, - }, log) + gcOptions.ImageRetention = config.ImageRetention{ + Policies: []config.GCPolicy{ + { + RepoNames: []string{"**"}, + DeleteUntagged: true, + }, + }, + } + gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gcOptions, log) err = gc.cleanManifests(repoName, &ispec.Index{ Manifests: []ispec.Descriptor{ @@ -393,11 +413,16 @@ func TestGarbageCollectWithMockedImageStore(t *testing.T) { }, } - gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, Options{ - Referrers: false, - Delay: storageConstants.DefaultGCDelay, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, - }, log) + gcOptions.ImageRetention = config.ImageRetention{ + Policies: []config.GCPolicy{ + { + RepoNames: []string{"**"}, + DeleteUntagged: true, + }, + }, + } + + gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gcOptions, log) err := gc.cleanManifests(repoName, &ispec.Index{ Manifests: []ispec.Descriptor{ @@ -430,11 +455,15 @@ func TestGarbageCollectWithMockedImageStore(t *testing.T) { }, } - gc := NewGarbageCollect(imgStore, metaDB, Options{ - Referrers: false, - Delay: storageConstants.DefaultGCDelay, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, - }, log) + gcOptions.ImageRetention = config.ImageRetention{ + Policies: []config.GCPolicy{ + { + RepoNames: []string{"**"}, + DeleteUntagged: true, + }, + }, + } + gc := NewGarbageCollect(imgStore, metaDB, gcOptions, log) err = gc.cleanManifests(repoName, &ispec.Index{ Manifests: []ispec.Descriptor{ @@ -467,11 +496,8 @@ func TestGarbageCollectWithMockedImageStore(t *testing.T) { }, } - gc := NewGarbageCollect(imgStore, metaDB, Options{ - Referrers: false, - Delay: storageConstants.DefaultGCDelay, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, - }, log) + gcOptions.ImageRetention = config.ImageRetention{} + gc := NewGarbageCollect(imgStore, metaDB, gcOptions, log) desc := ispec.Descriptor{ MediaType: ispec.MediaTypeImageManifest, @@ -481,7 +507,7 @@ func TestGarbageCollectWithMockedImageStore(t *testing.T) { index := &ispec.Index{ Manifests: []ispec.Descriptor{desc}, } - _, err = gc.removeManifest(repoName, index, desc, storage.NotationType, + _, err = gc.removeManifest(repoName, index, desc, desc.Digest.String(), storage.NotationType, godigest.FromBytes([]byte("digest2"))) So(err, ShouldNotBeNil) @@ -515,11 +541,7 @@ func TestGarbageCollectWithMockedImageStore(t *testing.T) { }, } - gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, Options{ - Referrers: true, - Delay: storageConstants.DefaultGCDelay, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, - }, log) + gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gcOptions, log) err = gc.cleanManifests(repoName, &returnedIndexImage) So(err, ShouldNotBeNil) @@ -550,11 +572,7 @@ func TestGarbageCollectWithMockedImageStore(t *testing.T) { }, } - gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, Options{ - Referrers: true, - Delay: storageConstants.DefaultGCDelay, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, - }, log) + gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gcOptions, log) err = gc.cleanManifests(repoName, &ispec.Index{ Manifests: []ispec.Descriptor{ diff --git a/pkg/storage/gc/policy.go b/pkg/storage/gc/policy.go new file mode 100644 index 0000000000..b2e527c2e0 --- /dev/null +++ b/pkg/storage/gc/policy.go @@ -0,0 +1,198 @@ +package gc + +import ( + glob "github.com/bmatcuk/doublestar/v4" + ispec "github.com/opencontainers/image-spec/specs-go/v1" + + zerr "zotregistry.io/zot/errors" + "zotregistry.io/zot/pkg/api/config" + zcommon "zotregistry.io/zot/pkg/common" + zlog "zotregistry.io/zot/pkg/log" + mTypes "zotregistry.io/zot/pkg/meta/types" + "zotregistry.io/zot/pkg/storage/gc/retention/candidate" + "zotregistry.io/zot/pkg/storage/gc/retention/rules" + "zotregistry.io/zot/pkg/storage/gc/retention/rules/dayspull" + "zotregistry.io/zot/pkg/storage/gc/retention/rules/dayspush" + "zotregistry.io/zot/pkg/storage/gc/retention/rules/latestpull" + "zotregistry.io/zot/pkg/storage/gc/retention/rules/latestpush" + "zotregistry.io/zot/pkg/storage/gc/retention/rules/name" + "zotregistry.io/zot/pkg/storage/gc/retention/rules/retainall" +) + +type policyManager struct { + repoPolicies []config.GCPolicy + log zlog.Logger +} + +func (p policyManager) hasDeleteUntagged(repo string) bool { + if policy, err := p.getRepoPolicy(repo); err == nil { + return policy.DeleteUntagged + } + + // default + return false +} + +func (p policyManager) hasDeleteReferrer(repo string) bool { + if policy, err := p.getRepoPolicy(repo); err == nil { + return policy.DeleteReferrers + } + + // default + return false +} + +func (p policyManager) hasTagRetention(repo string) bool { + if policy, err := p.getRepoPolicy(repo); err == nil { + return len(policy.TagsRetention) > 0 + } + + // default + return false +} + +func (p policyManager) getNameRule(repo, tag string) rules.Rule { + repoPolicy, err := p.getRepoPolicy(repo) + if err != nil { + // if repo policy not found, then retain all + return retainall.New(true) + } + + tagPolicy, err := p.getTagPolicy(tag, repoPolicy.TagsRetention) + if err != nil { + // if repo policy found but tag not found, then remove all tags + return retainall.New(false) + } + + // name rule + return name.New(tagPolicy.Names) +} + +func (p policyManager) getRules(repo string, tag string) []rules.Rule { + rules := make([]rules.Rule, 0) + + repoPolicy, err := p.getRepoPolicy(repo) + if err != nil { + // if repo policy not found, then do not apply any tag retention policy + return rules + } + + tagPolicy, err := p.getTagPolicy(tag, repoPolicy.TagsRetention) + if err != nil { + // if tag policy not found, gc all tags by default + return append(rules, retainall.New(false)) + } + + if tagPolicy.MostRecentlyPulledCount != nil { + rules = append(rules, latestpull.New(*tagPolicy.MostRecentlyPulledCount)) + } + + if tagPolicy.MostRecentlyPushedCount != nil { + rules = append(rules, latestpush.New(*tagPolicy.MostRecentlyPushedCount)) + } + + if tagPolicy.PulledWithinLastNrDays != nil { + rules = append(rules, dayspull.New(*tagPolicy.PulledWithinLastNrDays)) + } + + if tagPolicy.PushedWithinLastNrDays != nil { + rules = append(rules, dayspush.New(*tagPolicy.PushedWithinLastNrDays)) + } + + return rules +} + +func (p policyManager) getRepoPolicy(repo string) (config.GCPolicy, error) { + for _, policy := range p.repoPolicies { + for _, pattern := range policy.RepoNames { + matched, err := glob.Match(pattern, repo) + if err == nil && matched { + return policy, nil + } + } + } + + return config.GCPolicy{}, zerr.ErrGCPolicyNotFound +} + +func (p policyManager) getTagPolicy(tag string, tagPolicies []config.TagsRetentionPolicy, +) (config.TagsRetentionPolicy, error) { + for _, tagPolicy := range tagPolicies { + if matches(tag, tagPolicy.Names) { + return tagPolicy, nil + } + } + + return config.TagsRetentionPolicy{}, zerr.ErrGCPolicyNotFound +} + +func (p policyManager) getRetainedTags(repoMeta mTypes.RepoMetadata, index ispec.Index) []string { + repo := repoMeta.Name + + candidates := candidate.GetCandidates(repoMeta) + retainTags := make([]string, 0) + + // we need to make sure tags for which we can not find statistics in repoDB are not removed + actualTags := getIndexTags(index) + + // find tags which are not in candidates list, if they are not in repoDB we want to keep them + for _, tag := range actualTags { + found := false + + for _, candidate := range candidates { + if candidate.Tag == tag { + found = true + } + } + + if !found { + retainTags = append(retainTags, tag) + } + } + + for _, candidate := range candidates { + retainCandidates := candidates // copy + + // first filter candidates by tag glob pattern + nameRule := p.getNameRule(repo, candidate.Tag) + retainCandidates = nameRule.Perform(retainCandidates) + + // get corresponding rules from this policy + rules := p.getRules(repo, candidate.Tag) + + // we retain candidates if any of the below rules are met (OR logic between rules) + for _, rule := range rules { + ruleCandidates := rule.Perform(retainCandidates) + + retainCandidates = append(retainCandidates, ruleCandidates...) + } + + for _, retainCandidate := range retainCandidates { + // there may be duplicates + if !zcommon.Contains(retainTags, retainCandidate.Tag) { + // unique tags + p.log.Info().Str("repository", repo). + Str("tag", retainCandidate.Tag). + Str("policy", retainCandidate.RetainedBy).Msg("gc: retained tag") + + retainTags = append(retainTags, retainCandidate.Tag) + } + } + } + + return retainTags +} + +func matches(tag string, patterns []string) bool { + if len(patterns) == 0 { + return true + } + + for _, pattern := range patterns { + if matched, err := glob.Match(pattern, tag); err == nil && matched { + return true + } + } + + return false +} diff --git a/pkg/storage/gc/retention/candidate/candidate.go b/pkg/storage/gc/retention/candidate/candidate.go new file mode 100644 index 0000000000..9555d0ea60 --- /dev/null +++ b/pkg/storage/gc/retention/candidate/candidate.go @@ -0,0 +1,38 @@ +package candidate + +import ( + "time" + + mTypes "zotregistry.io/zot/pkg/meta/types" +) + +// helper struct. +type Candidate struct { + DigestStr string + Tag string + PushTimestamp time.Time + PullTimestamp time.Time + RetainedBy string +} + +func GetCandidates(repoMeta mTypes.RepoMetadata) []Candidate { + candidates := make([]Candidate, 0) + + // collect all statistic of repo's manifests + for tag, desc := range repoMeta.Tags { + for digestStr, stats := range repoMeta.Statistics { + if digestStr == desc.Digest { + candidate := Candidate{ + DigestStr: digestStr, + Tag: tag, + PushTimestamp: stats.PushTimestamp, + PullTimestamp: stats.LastPullTimestamp, + } + + candidates = append(candidates, candidate) + } + } + } + + return candidates +} diff --git a/pkg/storage/gc/retention/rules/dayspull/dayspull.go b/pkg/storage/gc/retention/rules/dayspull/dayspull.go new file mode 100644 index 0000000000..40ea91998c --- /dev/null +++ b/pkg/storage/gc/retention/rules/dayspull/dayspull.go @@ -0,0 +1,39 @@ +package dayspull + +import ( + "time" + + "zotregistry.io/zot/pkg/storage/gc/retention/candidate" +) + +const ( + ruleName = "mostRecentlyPulledCount" + day = time.Hour * 24 +) + +type rule struct { + count int +} + +func New(count int) rule { + return rule{count: count} +} + +func (r rule) Name() string { + return ruleName +} + +func (r rule) Perform(candidates []candidate.Candidate) []candidate.Candidate { + filtered := make([]candidate.Candidate, 0) + + timestamp := time.Now().Add(time.Duration(-r.count) * day) + + for _, candidate := range candidates { + if candidate.PullTimestamp.After(timestamp) { + candidate.RetainedBy = r.Name() + filtered = append(filtered, candidate) + } + } + + return filtered +} diff --git a/pkg/storage/gc/retention/rules/dayspush/dayspush.go b/pkg/storage/gc/retention/rules/dayspush/dayspush.go new file mode 100644 index 0000000000..122afa55d6 --- /dev/null +++ b/pkg/storage/gc/retention/rules/dayspush/dayspush.go @@ -0,0 +1,38 @@ +package dayspush + +import ( + "time" + + "zotregistry.io/zot/pkg/storage/gc/retention/candidate" +) + +const ( + ruleName = "mostRecentlyPushedCount" + day = time.Hour * 24 +) + +type rule struct { + count int +} + +func New(count int) rule { + return rule{count: count} +} + +func (r rule) Name() string { + return ruleName +} + +func (r rule) Perform(candidates []candidate.Candidate) []candidate.Candidate { + filtered := make([]candidate.Candidate, 0) + + timestamp := time.Now().Add(time.Duration(-r.count) * day) + + for _, candidate := range candidates { + if candidate.PushTimestamp.After(timestamp) { + filtered = append(filtered, candidate) + } + } + + return filtered +} diff --git a/pkg/storage/gc/retention/rules/latestpull/latestpull.go b/pkg/storage/gc/retention/rules/latestpull/latestpull.go new file mode 100644 index 0000000000..90a833e5f1 --- /dev/null +++ b/pkg/storage/gc/retention/rules/latestpull/latestpull.go @@ -0,0 +1,35 @@ +package latestpull + +import ( + "sort" + + "zotregistry.io/zot/pkg/storage/gc/retention/candidate" +) + +const ruleName = "pulledWithinLastNrDays" + +type rule struct { + count int +} + +func New(count int) rule { + return rule{count: count} +} + +func (r rule) Name() string { + return ruleName +} + +func (r rule) Perform(candidates []candidate.Candidate) []candidate.Candidate { + sort.Slice(candidates, func(i, j int) bool { + return candidates[i].PullTimestamp.Before(candidates[j].PullTimestamp) + }) + + // take top count candidates + upper := r.count + if r.count > len(candidates) { + upper = len(candidates) + } + + return candidates[:upper] +} diff --git a/pkg/storage/gc/retention/rules/latestpush/latestpush.go b/pkg/storage/gc/retention/rules/latestpush/latestpush.go new file mode 100644 index 0000000000..d0ef7d83f9 --- /dev/null +++ b/pkg/storage/gc/retention/rules/latestpush/latestpush.go @@ -0,0 +1,41 @@ +package latestpush + +import ( + "sort" + + "zotregistry.io/zot/pkg/storage/gc/retention/candidate" +) + +const ruleName = "pulledWithinLastNrDays" + +type rule struct { + count int +} + +func New(count int) rule { + return rule{count: count} +} + +func (r rule) Name() string { + return ruleName +} + +func (r rule) Perform(candidates []candidate.Candidate) []candidate.Candidate { + sort.Slice(candidates, func(i, j int) bool { + return candidates[i].PushTimestamp.Before(candidates[j].PushTimestamp) + }) + + // take top count candidates + upper := r.count + if r.count > len(candidates) { + upper = len(candidates) + } + + candidates = candidates[:upper] + + for _, candidate := range candidates { + candidate.RetainedBy = ruleName + } + + return candidates +} diff --git a/pkg/storage/gc/retention/rules/name/name.go b/pkg/storage/gc/retention/rules/name/name.go new file mode 100644 index 0000000000..3f8de7440b --- /dev/null +++ b/pkg/storage/gc/retention/rules/name/name.go @@ -0,0 +1,47 @@ +package name + +import ( + glob "github.com/bmatcuk/doublestar/v4" + + "zotregistry.io/zot/pkg/storage/gc/retention/candidate" +) + +const ruleName = "name" + +type rule struct { + names []string +} + +func New(names []string) rule { + return rule{names: names} +} + +func (r rule) Name() string { + return ruleName +} + +func (r rule) Perform(candidates []candidate.Candidate) []candidate.Candidate { + filtered := make([]candidate.Candidate, 0) + + for _, candidate := range candidates { + if matches(candidate.Tag, r.names) { + filtered = append(filtered, candidate) + } + } + + return filtered +} + +func matches(tag string, patterns []string) bool { + if len(patterns) == 0 { + return true + } + + for _, pattern := range patterns { + if matched, err := glob.Match(pattern, tag); err == nil && matched { + return true + } + } + + return false +} diff --git a/pkg/storage/gc/retention/rules/retainall/retainall.go b/pkg/storage/gc/retention/rules/retainall/retainall.go new file mode 100644 index 0000000000..b0274b01ac --- /dev/null +++ b/pkg/storage/gc/retention/rules/retainall/retainall.go @@ -0,0 +1,27 @@ +package retainall + +import ( + "zotregistry.io/zot/pkg/storage/gc/retention/candidate" +) + +const ruleName = "alwaysRetain" + +type rule struct { + retain bool +} + +func New(retain bool) rule { + return rule{retain} +} + +func (r rule) Name() string { + return ruleName +} + +func (r rule) Perform(candidates []candidate.Candidate) []candidate.Candidate { + if r.retain { + return candidates + } + + return []candidate.Candidate{} +} diff --git a/pkg/storage/gc/retention/rules/rule.go b/pkg/storage/gc/retention/rules/rule.go new file mode 100644 index 0000000000..f18a3dbf65 --- /dev/null +++ b/pkg/storage/gc/retention/rules/rule.go @@ -0,0 +1,8 @@ +package rules + +import "zotregistry.io/zot/pkg/storage/gc/retention/candidate" + +type Rule interface { + Name() string + Perform(candidates []candidate.Candidate) []candidate.Candidate +} diff --git a/pkg/storage/imagestore/imagestore.go b/pkg/storage/imagestore/imagestore.go index a65762c10b..70d3e0a0c1 100644 --- a/pkg/storage/imagestore/imagestore.go +++ b/pkg/storage/imagestore/imagestore.go @@ -1512,7 +1512,8 @@ func (is *ImageStore) CleanupRepo(repo string, blobs []godigest.Digest, removeRe count := 0 for _, digest := range blobs { - is.log.Debug().Str("repository", repo).Str("digest", digest.String()).Msg("perform GC on blob") + is.log.Debug().Str("repository", repo). + Str("digest", digest.String()).Msg("perform GC on blob") if err := is.deleteBlob(repo, digest); err != nil { if errors.Is(err, zerr.ErrBlobReferenced) { @@ -1544,6 +1545,8 @@ func (is *ImageStore) CleanupRepo(repo string, blobs []godigest.Digest, removeRe // if removeRepo flag is true and we cleanup all blobs and there are no blobs currently being uploaded. if removeRepo && count == len(blobs) && count > 0 && len(blobUploads) == 0 { + is.log.Info().Str("repository", repo).Msg("removed all blobs, removing repo") + if err := is.storeDriver.Delete(path.Join(is.rootDir, repo)); err != nil { is.log.Error().Err(err).Str("repository", repo).Msg("unable to remove repo") diff --git a/pkg/storage/local/local_test.go b/pkg/storage/local/local_test.go index 571475be31..c0909edd4e 100644 --- a/pkg/storage/local/local_test.go +++ b/pkg/storage/local/local_test.go @@ -47,6 +47,16 @@ const ( repoName = "test" ) +var DeleteReferrers = config.ImageRetention{ //nolint: gochecknoglobals + Policies: []config.GCPolicy{ + { + RepoNames: []string{"**"}, + DeleteReferrers: true, + DeleteUntagged: true, + }, + }, +} + var errCache = errors.New("new cache error") func runAndGetScheduler() (*scheduler.Scheduler, context.CancelFunc) { @@ -1129,9 +1139,9 @@ func FuzzRunGCRepo(f *testing.F) { imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ - Referrers: true, Delay: storageConstants.DefaultGCDelay, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + UntaggedDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + ImageRetention: DeleteReferrers, }, log) if err := gc.CleanRepo(data); err != nil { @@ -2006,9 +2016,9 @@ func TestGarbageCollectForImageStore(t *testing.T) { repoName := "gc-all-repos-short" gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ - Referrers: true, Delay: 1 * time.Second, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + UntaggedDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + ImageRetention: DeleteReferrers, }, log) image := CreateDefaultVulnerableImage() @@ -2051,9 +2061,9 @@ func TestGarbageCollectForImageStore(t *testing.T) { repoName := "gc-all-repos-short" gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ - Referrers: true, Delay: 1 * time.Second, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + UntaggedDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + ImageRetention: DeleteReferrers, }, log) image := CreateDefaultVulnerableImage() @@ -2092,9 +2102,9 @@ func TestGarbageCollectForImageStore(t *testing.T) { repoName := "gc-sig" gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ - Referrers: true, Delay: 1 * time.Second, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + UntaggedDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + ImageRetention: DeleteReferrers, }, log) storeController := storage.StoreController{DefaultStore: imgStore} @@ -2161,9 +2171,9 @@ func TestGarbageCollectImageUnknownManifest(t *testing.T) { } gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ - Referrers: true, Delay: 1 * time.Second, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + UntaggedDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + ImageRetention: DeleteReferrers, }, log) unsupportedMediaType := "application/vnd.oci.artifact.manifest.v1+json" @@ -2336,9 +2346,9 @@ func TestGarbageCollectErrors(t *testing.T) { repoName := "gc-index" gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ - Referrers: true, Delay: 500 * time.Millisecond, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + UntaggedDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + ImageRetention: DeleteReferrers, }, log) // create a blob/layer diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go index ce8fef9a69..91657d1cba 100644 --- a/pkg/storage/storage_test.go +++ b/pkg/storage/storage_test.go @@ -28,6 +28,7 @@ import ( "gopkg.in/resty.v1" zerr "zotregistry.io/zot/errors" + "zotregistry.io/zot/pkg/api/config" "zotregistry.io/zot/pkg/extensions/monitoring" "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/storage" @@ -43,6 +44,16 @@ import ( "zotregistry.io/zot/pkg/test/mocks" ) +var DeleteReferrers = config.ImageRetention{ //nolint: gochecknoglobals + Policies: []config.GCPolicy{ + { + RepoNames: []string{"**"}, + DeleteReferrers: true, + DeleteUntagged: true, + }, + }, +} + func cleanupStorage(store driver.StorageDriver, name string) { _ = store.Delete(context.Background(), name) } @@ -1291,9 +1302,16 @@ func TestGarbageCollectImageManifest(t *testing.T) { } gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ - Referrers: true, - Delay: storageConstants.DefaultGCDelay, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + Delay: storageConstants.DefaultGCDelay, + UntaggedDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + ImageRetention: config.ImageRetention{ + Policies: []config.GCPolicy{ + { + RepoNames: []string{"**"}, + DeleteReferrers: true, + }, + }, + }, }, log) repoName := "gc-long" @@ -1454,9 +1472,9 @@ func TestGarbageCollectImageManifest(t *testing.T) { } gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ - Referrers: true, Delay: gcDelay, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + UntaggedDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + ImageRetention: DeleteReferrers, }, log) // upload orphan blob @@ -1764,9 +1782,9 @@ func TestGarbageCollectImageManifest(t *testing.T) { } gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ - Referrers: true, Delay: gcDelay, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + UntaggedDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + ImageRetention: DeleteReferrers, }, log) // first upload an image to the first repo and wait for GC timeout @@ -2000,9 +2018,9 @@ func TestGarbageCollectImageIndex(t *testing.T) { } gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ - Referrers: true, Delay: storageConstants.DefaultGCDelay, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + UntaggedDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + ImageRetention: DeleteReferrers, }, log) repoName := "gc-long" @@ -2130,9 +2148,9 @@ func TestGarbageCollectImageIndex(t *testing.T) { } gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ - Referrers: true, Delay: gcDelay, - RetentionDelay: imageRetentionDelay, + UntaggedDelay: imageRetentionDelay, + ImageRetention: DeleteReferrers, }, log) // upload orphan blob @@ -2440,9 +2458,9 @@ func TestGarbageCollectChainedImageIndexes(t *testing.T) { } gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ - Referrers: true, Delay: gcDelay, - RetentionDelay: imageRetentionDelay, + UntaggedDelay: imageRetentionDelay, + ImageRetention: DeleteReferrers, }, log) // upload orphan blob diff --git a/pkg/test/image-utils/upload_test.go b/pkg/test/image-utils/upload_test.go index 178e08efaa..7291bc412f 100644 --- a/pkg/test/image-utils/upload_test.go +++ b/pkg/test/image-utils/upload_test.go @@ -81,17 +81,24 @@ func TestUploadImage(t *testing.T) { conf.HTTP.Port = port conf.Storage.RootDirectory = tempDir - err := os.Chmod(tempDir, 0o400) - if err != nil { - t.Fatal(err) - } - ctlr := api.NewController(conf) ctlrManager := tcommon.NewControllerManager(ctlr) ctlrManager.StartAndWait(port) defer ctlrManager.StopServer() + err := os.Chmod(tempDir, 0o400) + if err != nil { + t.Fatal(err) + } + + defer func() { + err := os.Chmod(tempDir, 0o700) + if err != nil { + t.Fatal(err) + } + }() + img := Image{ Layers: make([][]byte, 10), } diff --git a/pkg/test/mocks/repo_db_mock.go b/pkg/test/mocks/repo_db_mock.go index 9d40b4c1da..badfdba40b 100644 --- a/pkg/test/mocks/repo_db_mock.go +++ b/pkg/test/mocks/repo_db_mock.go @@ -19,7 +19,8 @@ type MetaDBMock struct { SetRepoLogoFn func(repo string, logoPath string) error - SetRepoReferenceFn func(repo string, Reference string, manifestDigest godigest.Digest, mediaType string) error + SetRepoReferenceFn func(ctx context.Context, repo string, Reference string, + manifestDigest godigest.Digest, mediaType string) error RemoveRepoReferenceFn func(repo, reference string, manifestDigest godigest.Digest) error @@ -55,7 +56,7 @@ type MetaDBMock struct { GetReferrersInfoFn func(repo string, referredDigest godigest.Digest, artifactTypes []string) ( []mTypes.ReferrerInfo, error) - IncrementImageDownloadsFn func(repo string, reference string) error + UpdateStatsOnDownloadFn func(repo string, reference string) error UpdateSignaturesValidityFn func(repo string, manifestDigest godigest.Digest) error @@ -160,11 +161,11 @@ func (sdm MetaDBMock) GetRepoStars(repo string) (int, error) { return 0, nil } -func (sdm MetaDBMock) SetRepoReference(repo string, reference string, manifestDigest godigest.Digest, - mediaType string, +func (sdm MetaDBMock) SetRepoReference(ctx context.Context, repo string, reference string, + manifestDigest godigest.Digest, mediaType string, ) error { if sdm.SetRepoReferenceFn != nil { - return sdm.SetRepoReferenceFn(repo, reference, manifestDigest, mediaType) + return sdm.SetRepoReferenceFn(ctx, repo, reference, manifestDigest, mediaType) } return nil @@ -251,9 +252,9 @@ func (sdm MetaDBMock) SetManifestMeta(repo string, manifestDigest godigest.Diges return nil } -func (sdm MetaDBMock) IncrementImageDownloads(repo string, reference string) error { - if sdm.IncrementImageDownloadsFn != nil { - return sdm.IncrementImageDownloadsFn(repo, reference) +func (sdm MetaDBMock) UpdateStatsOnDownload(repo string, reference string) error { + if sdm.UpdateStatsOnDownloadFn != nil { + return sdm.UpdateStatsOnDownloadFn(repo, reference) } return nil diff --git a/test/blackbox/garbage_collect.bats b/test/blackbox/garbage_collect.bats index 61ca08d5e3..5faeeaabc5 100644 --- a/test/blackbox/garbage_collect.bats +++ b/test/blackbox/garbage_collect.bats @@ -34,10 +34,18 @@ function setup_file() { "storage": { "rootDirectory": "${zot_root_dir}", "gc": true, - "gcReferrers": true, "gcDelay": "30s", "untaggedImageRetentionDelay": "40s", - "gcInterval": "1s" + "gcInterval": "1s", + "retention": { + "policies": [ + { + "repoNames": ["**"], + "deleteReferrers": true, + "deleteUntagged": true + } + ] + } }, "http": { "address": "0.0.0.0", diff --git a/test/cluster/config-minio.json b/test/cluster/config-minio.json index 618cf2cb1e..53464a7cfe 100644 --- a/test/cluster/config-minio.json +++ b/test/cluster/config-minio.json @@ -2,7 +2,7 @@ "distSpecVersion": "1.1.0-dev", "storage": { "rootDirectory": "/tmp/zot", - "gc": false, + "gc": true, "dedupe": false, "storageDriver": { "name": "s3", diff --git a/test/gc-stress/config-gc-bench-local.json b/test/gc-stress/config-gc-bench-local.json index ae1010c155..3d1dbb9901 100644 --- a/test/gc-stress/config-gc-bench-local.json +++ b/test/gc-stress/config-gc-bench-local.json @@ -3,7 +3,6 @@ "storage": { "rootDirectory": "/tmp/zot/local", "gc": true, - "gcReferrers": false, "gcDelay": "20s", "untaggedImageRetentionDelay": "20s", "gcInterval": "1s" diff --git a/test/gc-stress/config-gc-bench-s3-localstack.json b/test/gc-stress/config-gc-bench-s3-localstack.json index 11151bdce2..9bb3ad3636 100644 --- a/test/gc-stress/config-gc-bench-s3-localstack.json +++ b/test/gc-stress/config-gc-bench-s3-localstack.json @@ -3,7 +3,6 @@ "storage": { "rootDirectory": "/tmp/zot/s3", "gc": true, - "gcReferrers": false, "gcDelay": "50m", "untaggedImageRetentionDelay": "50m", "gcInterval": "2m", @@ -20,7 +19,13 @@ "name": "dynamodb", "endpoint": "http://localhost:4566", "region": "us-east-2", - "cacheTablename": "BlobTable" + "cacheTablename": "BlobTable", + "repoMetaTablename": "RepoMetadataTable", + "indexDataTablename": "IndexDataTable", + "manifestDataTablename": "ManifestDataTable", + "apikeytablename": "ApiKeyDataTable", + "userdatatablename": "UserDataTable", + "versionTablename": "VersionTable" } }, "http": { diff --git a/test/gc-stress/config-gc-bench-s3-minio.json b/test/gc-stress/config-gc-bench-s3-minio.json index 7c59e661ad..8fbeb0f549 100644 --- a/test/gc-stress/config-gc-bench-s3-minio.json +++ b/test/gc-stress/config-gc-bench-s3-minio.json @@ -3,7 +3,6 @@ "storage": { "rootDirectory": "/tmp/zot/s3", "gc": true, - "gcReferrers": false, "gcDelay": "3m", "untaggedImageRetentionDelay": "3m", "gcInterval": "1s", @@ -22,7 +21,13 @@ "name": "dynamodb", "endpoint": "http://localhost:4566", "region": "us-east-2", - "cacheTablename": "BlobTable" + "cacheTablename": "BlobTable", + "repoMetaTablename": "RepoMetadataTable", + "indexDataTablename": "IndexDataTable", + "manifestDataTablename": "ManifestDataTable", + "apikeytablename": "ApiKeyDataTable", + "userdatatablename": "UserDataTable", + "versionTablename": "VersionTable" } }, "http": { diff --git a/test/gc-stress/config-gc-referrers-bench-local.json b/test/gc-stress/config-gc-referrers-bench-local.json index 909be1a085..5263ce5791 100644 --- a/test/gc-stress/config-gc-referrers-bench-local.json +++ b/test/gc-stress/config-gc-referrers-bench-local.json @@ -3,10 +3,17 @@ "storage": { "rootDirectory": "/tmp/zot/local", "gc": true, - "gcReferrers": true, "gcDelay": "20s", "untaggedImageRetentionDelay": "20s", - "gcInterval": "1s" + "gcInterval": "1s", + "retention": { + "policies": [ + { + "repoNames": ["**"], + "deleteReferrers": true + } + ] + } }, "http": { "address": "127.0.0.1", diff --git a/test/gc-stress/config-gc-referrers-bench-s3-localstack.json b/test/gc-stress/config-gc-referrers-bench-s3-localstack.json index 98cee6bc48..bdfb5ff765 100644 --- a/test/gc-stress/config-gc-referrers-bench-s3-localstack.json +++ b/test/gc-stress/config-gc-referrers-bench-s3-localstack.json @@ -3,10 +3,17 @@ "storage": { "rootDirectory": "/tmp/zot/s3", "gc": true, - "gcReferrers": true, "gcDelay": "50m", "untaggedImageRetentionDelay": "50m", "gcInterval": "2m", + "retention": { + "policies": [ + { + "repoNames": ["**"], + "deleteReferrers": true + } + ] + }, "storageDriver": { "name": "s3", "rootdirectory": "/zot", @@ -20,7 +27,13 @@ "name": "dynamodb", "endpoint": "http://localhost:4566", "region": "us-east-2", - "cacheTablename": "BlobTable" + "cacheTablename": "BlobTable", + "repoMetaTablename": "RepoMetadataTable", + "indexDataTablename": "IndexDataTable", + "manifestDataTablename": "ManifestDataTable", + "apikeytablename": "ApiKeyDataTable", + "userdatatablename": "UserDataTable", + "versionTablename": "VersionTable" } }, "http": { diff --git a/test/gc-stress/config-gc-referrers-bench-s3-minio.json b/test/gc-stress/config-gc-referrers-bench-s3-minio.json index 58ba09934d..93e6a176c2 100644 --- a/test/gc-stress/config-gc-referrers-bench-s3-minio.json +++ b/test/gc-stress/config-gc-referrers-bench-s3-minio.json @@ -3,10 +3,17 @@ "storage": { "rootDirectory": "/tmp/zot/s3", "gc": true, - "gcReferrers": true, "gcDelay": "3m", "untaggedImageRetentionDelay": "3m", "gcInterval": "1s", + "retention": { + "policies": [ + { + "repoNames": ["**"], + "deleteReferrers": true + } + ] + }, "storageDriver": { "name": "s3", "rootdirectory": "/zot", @@ -22,7 +29,13 @@ "name": "dynamodb", "endpoint": "http://localhost:4566", "region": "us-east-2", - "cacheTablename": "BlobTable" + "cacheTablename": "BlobTable", + "repoMetaTablename": "RepoMetadataTable", + "indexDataTablename": "IndexDataTable", + "manifestDataTablename": "ManifestDataTable", + "apikeytablename": "ApiKeyDataTable", + "userdatatablename": "UserDataTable", + "versionTablename": "VersionTable" } }, "http": {