diff --git a/pkg/api/controller.go b/pkg/api/controller.go index e958048d77..a7ff3e689b 100644 --- a/pkg/api/controller.go +++ b/pkg/api/controller.go @@ -372,6 +372,10 @@ func (c *Controller) StartBackgroundTasks(reloadCtx context.Context) { ext.EnableMetricsExtension(c.Config, c.Log, c.Config.Storage.RootDirectory) ext.EnableSearchExtension(c.Config, c.StoreController, c.MetaDB, taskScheduler, c.CveScanner, c.Log) } + // runs once if metrics are enabled + if c.Config.IsMetricsEnabled() { + c.StoreController.DefaultStore.PopulateStorageMetrics(time.Duration(0), taskScheduler) + } if c.Config.Storage.SubPaths != nil { for route, storageConfig := range c.Config.Storage.SubPaths { @@ -396,6 +400,10 @@ func (c *Controller) StartBackgroundTasks(reloadCtx context.Context) { substore := c.StoreController.SubStore[route] if substore != nil { substore.RunDedupeBlobs(time.Duration(0), taskScheduler) + + if c.Config.IsMetricsEnabled() { + substore.PopulateStorageMetrics(time.Duration(0), taskScheduler) + } } } } diff --git a/pkg/storage/common/common.go b/pkg/storage/common/common.go index 8573b4a6ba..cc9e3650e1 100644 --- a/pkg/storage/common/common.go +++ b/pkg/storage/common/common.go @@ -6,8 +6,10 @@ import ( "encoding/json" "errors" "fmt" + "math/rand" "path" "strings" + "time" "github.com/docker/distribution/registry/storage/driver" godigest "github.com/opencontainers/go-digest" @@ -18,6 +20,7 @@ import ( zerr "zotregistry.io/zot/errors" zcommon "zotregistry.io/zot/pkg/common" + "zotregistry.io/zot/pkg/extensions/monitoring" zlog "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/scheduler" storageConstants "zotregistry.io/zot/pkg/storage/constants" @@ -1052,3 +1055,76 @@ func (dt *dedupeTask) DoWork(ctx context.Context) error { return err } + +type StorageMetricsInitGenerator struct { + ImgStore storageTypes.ImageStore + done bool + Metrics monitoring.MetricServer + lastRepo string + nextRun time.Time + rand *rand.Rand + Log zlog.Logger +} + +func (gen *StorageMetricsInitGenerator) getRandomDelay() int { + maxDelay := 30 + + return gen.rand.Intn(maxDelay) +} + +func (gen *StorageMetricsInitGenerator) Next() (scheduler.Task, error) { + if gen.lastRepo == "" && gen.nextRun.IsZero() { + gen.rand = rand.New(rand.NewSource(time.Now().UTC().UnixNano())) //nolint: gosec + } + + delay := gen.getRandomDelay() + + gen.nextRun = time.Now().Add(time.Duration(delay) * time.Second) + + repo, err := gen.ImgStore.GetNextRepository(gen.lastRepo) + if err != nil { + return nil, err + } + + gen.Log.Debug().Str("repo", repo).Int("randomDelay", delay).Msg("StorageMetricsInitGenerator") + + if repo == "" { + gen.done = true + + return nil, nil + } + gen.lastRepo = repo + + return NewStorageMetricsTask(gen.ImgStore, gen.Metrics, repo), nil +} + +func (gen *StorageMetricsInitGenerator) IsDone() bool { + return gen.done +} + +func (gen *StorageMetricsInitGenerator) IsReady() bool { + return true +} + +func (gen *StorageMetricsInitGenerator) Reset() { + gen.lastRepo = "" + gen.done = false +} + +type smTask struct { + imgStore storageTypes.ImageStore + metrics monitoring.MetricServer + repo string +} + +func NewStorageMetricsTask(imgStore storageTypes.ImageStore, metrics monitoring.MetricServer, repo string, +) *smTask { + return &smTask{imgStore, metrics, repo} +} + +func (smt *smTask) DoWork(ctx context.Context) error { + // run task + monitoring.SetStorageUsage(smt.metrics, smt.imgStore.RootDir(), smt.repo) + + return nil +} diff --git a/pkg/storage/imagestore/imagestore.go b/pkg/storage/imagestore/imagestore.go index 2508a59fd3..9b28c93963 100644 --- a/pkg/storage/imagestore/imagestore.go +++ b/pkg/storage/imagestore/imagestore.go @@ -1929,6 +1929,16 @@ func (is *ImageStore) RunDedupeBlobs(interval time.Duration, sch *scheduler.Sche sch.SubmitGenerator(generator, interval, scheduler.MediumPriority) } +func (is *ImageStore) PopulateStorageMetrics(interval time.Duration, sch *scheduler.Scheduler) { + generator := &common.StorageMetricsInitGenerator{ + ImgStore: is, + Metrics: is.metrics, + Log: is.log, + } + + sch.SubmitGenerator(generator, interval, scheduler.LowPriority) +} + type blobStream struct { reader io.Reader closer io.Closer diff --git a/pkg/storage/types/types.go b/pkg/storage/types/types.go index 4f7c364120..03f7648918 100644 --- a/pkg/storage/types/types.go +++ b/pkg/storage/types/types.go @@ -61,6 +61,7 @@ type ImageStore interface { //nolint:interfacebloat RunDedupeForDigest(digest godigest.Digest, dedupe bool, duplicateBlobs []string) error GetNextDigestWithBlobPaths(repos []string, lastDigests []godigest.Digest) (godigest.Digest, []string, error) GetAllBlobs(repo string) ([]string, error) + PopulateStorageMetrics(interval time.Duration, sch *scheduler.Scheduler) } type Driver interface { //nolint:interfacebloat diff --git a/pkg/test/mocks/image_store_mock.go b/pkg/test/mocks/image_store_mock.go index e5eda2c5c4..5451c10bf0 100644 --- a/pkg/test/mocks/image_store_mock.go +++ b/pkg/test/mocks/image_store_mock.go @@ -55,6 +55,7 @@ type MockedImageStore struct { GetAllBlobsFn func(repo string) ([]string, error) CleanupRepoFn func(repo string, blobs []godigest.Digest, removeRepo bool) (int, error) PutIndexContentFn func(repo string, index ispec.Index) error + RunStorageMetricsFn func(interval time.Duration, sch *scheduler.Scheduler) } func (is MockedImageStore) Lock(t *time.Time) { @@ -405,3 +406,9 @@ func (is MockedImageStore) PutIndexContent(repo string, index ispec.Index) error return nil } + +func (is MockedImageStore) PopulateStorageMetrics(interval time.Duration, sch *scheduler.Scheduler) { + if is.RunStorageMetricsFn != nil { + is.RunStorageMetricsFn(interval, sch) + } +}