From e5af24d49e26fd09a19bab3c7a0f0660f78e88c0 Mon Sep 17 00:00:00 2001 From: Eusebiu Petu Date: Mon, 4 Nov 2024 23:16:07 +0200 Subject: [PATCH] wip zot catalog pagination Signed-off-by: Eusebiu Petu --- pkg/api/routes.go | 118 +++++++++++++++++++++++++-- pkg/storage/imagestore/imagestore.go | 69 ++++++++++++++++ pkg/storage/storage_controller.go | 20 ++++- pkg/storage/types/types.go | 1 + pkg/test/mocks/image_store_mock.go | 29 ++++--- 5 files changed, 219 insertions(+), 18 deletions(-) diff --git a/pkg/api/routes.go b/pkg/api/routes.go index 164b6bf9a..27deeb71d 100644 --- a/pkg/api/routes.go +++ b/pkg/api/routes.go @@ -47,6 +47,7 @@ import ( mTypes "zotregistry.dev/zot/pkg/meta/types" zreg "zotregistry.dev/zot/pkg/regexp" reqCtx "zotregistry.dev/zot/pkg/requestcontext" + "zotregistry.dev/zot/pkg/storage" storageCommon "zotregistry.dev/zot/pkg/storage/common" storageTypes "zotregistry.dev/zot/pkg/storage/types" "zotregistry.dev/zot/pkg/test/inject" @@ -1776,12 +1777,116 @@ func (rh *RouteHandler) ListRepositories(response http.ResponseWriter, request * return } + q := request.URL.Query() + + lastEntry := q.Get("last") + maxEntries, err := strconv.Atoi(q.Get("n")) + if err != nil { + maxEntries = -1 + } + + if lastEntry != "" || maxEntries > 0 { + storePath := rh.c.StoreController.GetStorePath(lastEntry) + + combineRepoList := make([]string, 0) + + var moreEntries bool = false + + var remainder int + + singleStore := rh.c.StoreController.DefaultStore + if singleStore != nil && storePath == storage.DefaultStorePath { // route is default + var err error + + var repos []string + + repos, moreEntries, err = singleStore.GetNextRepositories(lastEntry, maxEntries) + if err != nil { + response.WriteHeader(http.StatusInternalServerError) + + return + } + + // compute remainder + remainder = maxEntries - len(repos) + + if moreEntries { + // maxEntries has been hit + lastEntry = repos[len(repos)-1] + } else { + // reset for the next substores + lastEntry = "" + } + + combineRepoList = append(combineRepoList, repos...) + } + + subStore := rh.c.StoreController.SubStore + for subPath, imgStore := range subStore { + if subPath != storePath || remainder <= 0 { + continue + } + + var err error + + var repos []string + + repos, moreEntries, err = imgStore.GetNextRepositories(lastEntry, remainder) + if err != nil { + response.WriteHeader(http.StatusInternalServerError) + + return + } + + // compute remainder + remainder = maxEntries - len(repos) + + if moreEntries { + // maxEntries has been hit + lastEntry = repos[len(repos)-1] + } else { + // reset for the next substores + lastEntry = "" + } + + combineRepoList = append(combineRepoList, repos...) + } + + repos := make([]string, 0) + // authz context + userAc, err := reqCtx.UserAcFromContext(request.Context()) + if err != nil { + response.WriteHeader(http.StatusInternalServerError) + + return + } + + if userAc != nil { + for _, r := range combineRepoList { + if userAc.Can(constants.ReadPermission, r) { + repos = append(repos, r) + } + } + } else { + repos = combineRepoList + } + + is := RepositoryList{Repositories: repos} + + zcommon.WriteJSON(response, http.StatusOK, is) + + return + } + combineRepoList := make([]string, 0) - subStore := rh.c.StoreController.SubStore + singleStore := rh.c.StoreController.DefaultStore + if singleStore != nil { // route is default + var err error - for _, imgStore := range subStore { - repos, err := imgStore.GetRepositories() + var repos []string + + repos, err = singleStore.GetRepositories() if err != nil { response.WriteHeader(http.StatusInternalServerError) @@ -1791,9 +1896,9 @@ func (rh *RouteHandler) ListRepositories(response http.ResponseWriter, request * combineRepoList = append(combineRepoList, repos...) } - singleStore := rh.c.StoreController.DefaultStore - if singleStore != nil { - repos, err := singleStore.GetRepositories() + subStore := rh.c.StoreController.SubStore + for _, imgStore := range subStore { + repos, err := imgStore.GetRepositories() if err != nil { response.WriteHeader(http.StatusInternalServerError) @@ -1801,6 +1906,7 @@ func (rh *RouteHandler) ListRepositories(response http.ResponseWriter, request * } combineRepoList = append(combineRepoList, repos...) + } repos := make([]string, 0) diff --git a/pkg/storage/imagestore/imagestore.go b/pkg/storage/imagestore/imagestore.go index 7664aeb73..73c4b5dbe 100644 --- a/pkg/storage/imagestore/imagestore.go +++ b/pkg/storage/imagestore/imagestore.go @@ -265,6 +265,75 @@ func (is *ImageStore) ValidateRepo(name string) (bool, error) { return true, nil } +func (is *ImageStore) GetNextRepositories(lastRepo string, maxEntries int) ([]string, bool, error) { + var lockLatency time.Time + + dir := is.rootDir + + is.RLock(&lockLatency) + defer is.RUnlock(&lockLatency) + + stores := make([]string, 0) + + moreEntries := false + entries := 0 + found := false + err := is.storeDriver.Walk(dir, func(fileInfo driver.FileInfo) error { + if entries == maxEntries { + moreEntries = true + + return io.EOF + } + + if !fileInfo.IsDir() { + return nil + } + + // skip .sync and .uploads dirs no need to try to validate them + if strings.HasSuffix(fileInfo.Path(), syncConstants.SyncBlobUploadDir) || + strings.HasSuffix(fileInfo.Path(), ispec.ImageBlobsDir) || + strings.HasSuffix(fileInfo.Path(), storageConstants.BlobUploadDir) { + return driver.ErrSkipDir + } + + rel, err := filepath.Rel(is.rootDir, fileInfo.Path()) + if err != nil { + return nil //nolint:nilerr // ignore paths that are not under root dir + } + + if ok, err := is.ValidateRepo(rel); !ok || err != nil { + return nil //nolint:nilerr // ignore invalid repos + } + + if lastRepo == "" || lastRepo == rel { + found = true + } + + if found { + entries++ + + stores = append(stores, rel) + } + + return nil + }) + + // if the root directory is not yet created then return an empty slice of repositories + driverErr := &driver.Error{} + if errors.As(err, &driver.PathNotFoundError{}) { + is.log.Debug().Msg("empty rootDir") + + return stores, moreEntries, nil + } + + if errors.Is(err, io.EOF) || + (errors.As(err, driverErr) && errors.Is(driverErr.Detail, io.EOF)) { + return stores, moreEntries, nil + } + + return stores, moreEntries, err +} + // GetRepositories returns a list of all the repositories under this store. func (is *ImageStore) GetRepositories() ([]string, error) { var lockLatency time.Time diff --git a/pkg/storage/storage_controller.go b/pkg/storage/storage_controller.go index 721a1a60f..75f10f91b 100644 --- a/pkg/storage/storage_controller.go +++ b/pkg/storage/storage_controller.go @@ -7,8 +7,9 @@ import ( ) const ( - CosignType = "cosign" - NotationType = "notation" + CosignType = "cosign" + NotationType = "notation" + DefaultStorePath = "/" ) type StoreController struct { @@ -29,6 +30,21 @@ func GetRoutePrefix(name string) string { return "/" + names[0] } +func (sc StoreController) GetStorePath(name string) string { + if sc.SubStore != nil && name != "" { + subStorePath := GetRoutePrefix(name) + + _, ok := sc.SubStore[subStorePath] + if !ok { + return DefaultStorePath + } + + return subStorePath + } + + return DefaultStorePath +} + func (sc StoreController) GetImageStore(name string) storageTypes.ImageStore { if sc.SubStore != nil { // SubStore is being provided, now we need to find equivalent image store and this will be found by splitting name diff --git a/pkg/storage/types/types.go b/pkg/storage/types/types.go index 9d5cd4882..a5af75b36 100644 --- a/pkg/storage/types/types.go +++ b/pkg/storage/types/types.go @@ -30,6 +30,7 @@ type ImageStore interface { //nolint:interfacebloat ValidateRepo(name string) (bool, error) GetRepositories() ([]string, error) GetNextRepository(repo string) (string, error) + GetNextRepositories(repo string, maxEntries int) ([]string, bool, error) GetImageTags(repo string) ([]string, error) GetImageManifest(repo, reference string) ([]byte, godigest.Digest, string, error) PutImageManifest(repo, reference, mediaType string, body []byte) (godigest.Digest, godigest.Digest, error) diff --git a/pkg/test/mocks/image_store_mock.go b/pkg/test/mocks/image_store_mock.go index d94b540d8..80d15ad21 100644 --- a/pkg/test/mocks/image_store_mock.go +++ b/pkg/test/mocks/image_store_mock.go @@ -12,16 +12,17 @@ import ( ) type MockedImageStore struct { - NameFn func() string - DirExistsFn func(d string) bool - RootDirFn func() string - InitRepoFn func(name string) error - ValidateRepoFn func(name string) (bool, error) - GetRepositoriesFn func() ([]string, error) - GetNextRepositoryFn func(repo string) (string, error) - GetImageTagsFn func(repo string) ([]string, error) - GetImageManifestFn func(repo string, reference string) ([]byte, godigest.Digest, string, error) - PutImageManifestFn func(repo string, reference string, mediaType string, body []byte) (godigest.Digest, + NameFn func() string + DirExistsFn func(d string) bool + RootDirFn func() string + InitRepoFn func(name string) error + ValidateRepoFn func(name string) (bool, error) + GetRepositoriesFn func() ([]string, error) + GetNextRepositoryFn func(repo string) (string, error) + GetNextRepositoriesFn func(lastRepo string, maxEntries int) ([]string, bool, error) + GetImageTagsFn func(repo string) ([]string, error) + GetImageManifestFn func(repo string, reference string) ([]byte, godigest.Digest, string, error) + PutImageManifestFn func(repo string, reference string, mediaType string, body []byte) (godigest.Digest, godigest.Digest, error) DeleteImageManifestFn func(repo string, reference string, detectCollision bool) error BlobUploadPathFn func(repo string, uuid string) string @@ -138,6 +139,14 @@ func (is MockedImageStore) GetNextRepository(repo string) (string, error) { return "", nil } +func (is MockedImageStore) GetNextRepositories(lastRepo string, maxEntries int) ([]string, bool, error) { + if is.GetNextRepositoriesFn != nil { + return is.GetNextRepositoriesFn(lastRepo, maxEntries) + } + + return []string{}, false, nil +} + func (is MockedImageStore) GetImageManifest(repo string, reference string) ([]byte, godigest.Digest, string, error) { if is.GetImageManifestFn != nil { return is.GetImageManifestFn(repo, reference)