diff --git a/errors/errors.go b/errors/errors.go index 6a0a2d100e..db9f162c3a 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -118,9 +118,9 @@ var ( ErrEmptyTag = errors.New("metadb: tag can't be empty string") ErrEmptyDigest = errors.New("metadb: digest can't be empty string") ErrInvalidRepoRefFormat = errors.New("invalid image reference format [repo:tag] or [repo@digest]") - ErrLimitIsNegative = errors.New("pageturner: limit has negative value") - ErrOffsetIsNegative = errors.New("pageturner: offset has negative value") - ErrSortCriteriaNotSupported = errors.New("pageturner: the sort criteria is not supported") + ErrLimitIsNegative = errors.New("pagefinder: limit has negative value") + ErrOffsetIsNegative = errors.New("pagefinder: offset has negative value") + ErrSortCriteriaNotSupported = errors.New("pagefinder: the sort criteria is not supported") ErrMediaTypeNotSupported = errors.New("metadb: media type is not supported") ErrTimeout = errors.New("operation timeout") ErrNotImplemented = errors.New("not implemented") diff --git a/pkg/extensions/search/convert/metadb.go b/pkg/extensions/search/convert/metadb.go index b3e9e38e61..6d997c608d 100644 --- a/pkg/extensions/search/convert/metadb.go +++ b/pkg/extensions/search/convert/metadb.go @@ -175,12 +175,12 @@ func GetFullImageMeta(tag string, repoMeta mTypes.RepoMeta, imageMeta mTypes.Ima } } -func GetFullManifestMeta(repoMeta mTypes.RepoMeta, manifests []mTypes.ManifestData) []mTypes.FullManifestMeta { +func GetFullManifestMeta(repoMeta mTypes.RepoMeta, manifests []mTypes.ManifestMeta) []mTypes.FullManifestMeta { results := make([]mTypes.FullManifestMeta, 0, len(manifests)) for i := range manifests { results = append(results, mTypes.FullManifestMeta{ - ManifestData: manifests[i], + ManifestMeta: manifests[i], Referrers: repoMeta.Referrers[manifests[i].Digest.String()], Statistics: repoMeta.Statistics[manifests[i].Digest.String()], Signatures: repoMeta.Signatures[manifests[i].Digest.String()], diff --git a/pkg/extensions/search/cve/cve_internal_test.go b/pkg/extensions/search/cve/cve_internal_test.go index c4cbc1d2fa..d242efc3e0 100644 --- a/pkg/extensions/search/cve/cve_internal_test.go +++ b/pkg/extensions/search/cve/cve_internal_test.go @@ -3,17 +3,64 @@ package cveinfo import ( + "errors" "testing" "time" + "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" . "github.com/smartystreets/goconvey/convey" cvemodel "zotregistry.io/zot/pkg/extensions/search/cve/model" + "zotregistry.io/zot/pkg/meta/types" + "zotregistry.io/zot/pkg/test/mocks" ) +var ErrTestError = errors.New("test error") + func TestUtils(t *testing.T) { Convey("Utils", t, func() { + Convey("getConfigAndDigest", func() { + _, _, err := getConfigAndDigest(mocks.MetaDBMock{}, "bad-digest") + So(err, ShouldNotBeNil) + + _, _, err = getConfigAndDigest(mocks.MetaDBMock{ + GetImageMetaFn: func(digest digest.Digest) (types.ImageMeta, error) { + return types.ImageMeta{}, ErrTestError + }, + }, ispec.DescriptorEmptyJSON.Digest.String()) + So(err, ShouldNotBeNil) + + // bad media type of config + _, _, err = getConfigAndDigest(mocks.MetaDBMock{ + GetImageMetaFn: func(digest digest.Digest) (types.ImageMeta, error) { + return types.ImageMeta{Manifests: []types.ManifestMeta{ + {Manifest: ispec.Manifest{Config: ispec.Descriptor{MediaType: "bad-type"}}}, + }}, nil + }, + }, ispec.DescriptorEmptyJSON.Digest.String()) + So(err, ShouldNotBeNil) + }) + Convey("getIndexContent", func() { + _, err := getIndexContent(mocks.MetaDBMock{}, "bad-digest") + So(err, ShouldNotBeNil) + + _, err = getIndexContent(mocks.MetaDBMock{ + GetImageMetaFn: func(digest digest.Digest) (types.ImageMeta, error) { + return types.ImageMeta{}, ErrTestError + }, + }, ispec.DescriptorEmptyJSON.Digest.String()) + So(err, ShouldNotBeNil) + + // nil index + _, err = getIndexContent(mocks.MetaDBMock{ + GetImageMetaFn: func(digest digest.Digest) (types.ImageMeta, error) { + return types.ImageMeta{}, nil + }, + }, ispec.DescriptorEmptyJSON.Digest.String()) + So(err, ShouldNotBeNil) + }) + Convey("mostRecentUpdate", func() { // empty timestamp := mostRecentUpdate([]cvemodel.DescriptorInfo{}) diff --git a/pkg/extensions/search/cve/trivy/scanner.go b/pkg/extensions/search/cve/trivy/scanner.go index 6ae1a74398..38c6f1e9b9 100644 --- a/pkg/extensions/search/cve/trivy/scanner.go +++ b/pkg/extensions/search/cve/trivy/scanner.go @@ -243,10 +243,6 @@ func (scanner Scanner) isManifestScanable(digestStr string) (bool, error) { return false, err } - if manifestData.MediaType != ispec.MediaTypeImageManifest { - return false, zerr.ErrUnexpectedMediaType - } - for _, imageLayer := range manifestData.Manifests[0].Manifest.Layers { switch imageLayer.MediaType { case ispec.MediaTypeImageLayerGzip, ispec.MediaTypeImageLayer, string(regTypes.DockerLayer): @@ -259,15 +255,11 @@ func (scanner Scanner) isManifestScanable(digestStr string) (bool, error) { return true, nil } -func (scanner Scanner) isManifestDataScannable(manifestData mTypes.ManifestData) (bool, error) { +func (scanner Scanner) isManifestDataScannable(manifestData mTypes.ManifestMeta) (bool, error) { if scanner.cache.Get(manifestData.Digest.String()) != nil { return true, nil } - if manifestData.Manifest.MediaType != ispec.MediaTypeImageManifest { - return false, zerr.ErrScanNotSupported - } - for _, imageLayer := range manifestData.Manifest.Layers { switch imageLayer.MediaType { case ispec.MediaTypeImageLayerGzip, ispec.MediaTypeImageLayer, string(regTypes.DockerLayer): @@ -290,7 +282,7 @@ func (scanner Scanner) isIndexScannable(digestStr string) (bool, error) { return false, err } - if indexData.MediaType != ispec.MediaTypeImageIndex || indexData.Index == nil { + if indexData.Index == nil { return false, zerr.ErrUnexpectedMediaType } diff --git a/pkg/extensions/search/cve/trivy/scanner_test.go b/pkg/extensions/search/cve/trivy/scanner_test.go index 028549c4b9..a4aa7470a7 100644 --- a/pkg/extensions/search/cve/trivy/scanner_test.go +++ b/pkg/extensions/search/cve/trivy/scanner_test.go @@ -3,6 +3,7 @@ package trivy_test import ( + "errors" "path/filepath" "testing" "time" @@ -19,13 +20,17 @@ import ( "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/meta" "zotregistry.io/zot/pkg/meta/boltdb" + "zotregistry.io/zot/pkg/meta/types" "zotregistry.io/zot/pkg/storage" "zotregistry.io/zot/pkg/storage/local" . "zotregistry.io/zot/pkg/test/common" "zotregistry.io/zot/pkg/test/deprecated" . "zotregistry.io/zot/pkg/test/image-utils" + "zotregistry.io/zot/pkg/test/mocks" ) +var ErrTestError = errors.New("test error") + func TestScanBigTestFile(t *testing.T) { Convey("Scan zot-test", t, func() { projRootDir, err := GetProjectRootDir() @@ -200,3 +205,74 @@ func TestVulnerableLayer(t *testing.T) { So(cveMap, ShouldContainKey, "CVE-2023-3446") }) } + +func TestScannerErrors(t *testing.T) { + Convey("Errors", t, func() { + storeController := storage.StoreController{} + metaDB := mocks.MetaDBMock{} + log := log.NewLogger("debug", "") + + Convey("IsImageFormatScannable", func() { + storeController.DefaultStore = mocks.MockedImageStore{} + metaDB.GetImageMetaFn = func(digest godigest.Digest) (types.ImageMeta, error) { + return types.ImageMeta{}, ErrTestError + } + scanner := trivy.NewScanner(storeController, metaDB, "ghcr.io/project-zot/trivy-db", "", log) + + _, err := scanner.IsImageFormatScannable("repo", godigest.FromString("dig").String()) + So(err, ShouldNotBeNil) + }) + Convey("IsImageMediaScannable", func() { + storeController.DefaultStore = mocks.MockedImageStore{} + metaDB.GetImageMetaFn = func(digest godigest.Digest) (types.ImageMeta, error) { + return types.ImageMeta{}, ErrTestError + } + scanner := trivy.NewScanner(storeController, metaDB, "ghcr.io/project-zot/trivy-db", "", log) + + Convey("Manifest", func() { + _, err := scanner.IsImageMediaScannable("repo", godigest.FromString("dig").String(), ispec.MediaTypeImageManifest) + So(err, ShouldNotBeNil) + }) + Convey("Index", func() { + _, err := scanner.IsImageMediaScannable("repo", godigest.FromString("dig").String(), ispec.MediaTypeImageIndex) + So(err, ShouldNotBeNil) + }) + Convey("Index with nil index", func() { + metaDB.GetImageMetaFn = func(digest godigest.Digest) (types.ImageMeta, error) { + return types.ImageMeta{}, nil + } + scanner := trivy.NewScanner(storeController, metaDB, "ghcr.io/project-zot/trivy-db", "", log) + + _, err := scanner.IsImageMediaScannable("repo", godigest.FromString("dig").String(), ispec.MediaTypeImageIndex) + So(err, ShouldNotBeNil) + }) + Convey("Index with good index", func() { + metaDB.GetImageMetaFn = func(digest godigest.Digest) (types.ImageMeta, error) { + return types.ImageMeta{ + Index: &ispec.Index{ + Manifests: []ispec.Descriptor{{MediaType: ispec.MediaTypeImageLayer}}, + }, + Manifests: []types.ManifestMeta{{Manifest: ispec.Manifest{ + Layers: []ispec.Descriptor{{MediaType: ispec.MediaTypeImageLayer}}, + }}}, + }, nil + } + scanner := trivy.NewScanner(storeController, metaDB, "ghcr.io/project-zot/trivy-db", "", log) + + _, err := scanner.IsImageMediaScannable("repo", godigest.FromString("dig").String(), ispec.MediaTypeImageIndex) + So(err, ShouldBeNil) + }) + }) + Convey("ScanImage", func() { + storeController.DefaultStore = mocks.MockedImageStore{} + metaDB.GetImageMetaFn = func(digest godigest.Digest) (types.ImageMeta, error) { + return types.ImageMeta{}, ErrTestError + } + + scanner := trivy.NewScanner(storeController, metaDB, "ghcr.io/project-zot/trivy-db", "", log) + + _, err := scanner.ScanImage("image@" + godigest.FromString("digest").String()) + So(err, ShouldNotBeNil) + }) + }) +} diff --git a/pkg/extensions/search/resolver_test.go b/pkg/extensions/search/resolver_test.go index 02d8bd5a78..0a7e7fd2d5 100644 --- a/pkg/extensions/search/resolver_test.go +++ b/pkg/extensions/search/resolver_test.go @@ -244,6 +244,39 @@ func TestRepoListWithNewestImage(t *testing.T) { }) } +func TestGetFilteredPaginatedRepos(t *testing.T) { + ctx := context.Background() + log := log.NewLogger("debug", "") + + Convey("getFilteredPaginatedRepos", t, func() { + metaDB := mocks.MetaDBMock{} + + Convey("FilterRepos", func() { + metaDB.FilterReposFn = func(ctx context.Context, rankName mTypes.FilterRepoNameFunc, + filterFunc mTypes.FilterFullRepoFunc, + ) ([]mTypes.RepoMeta, error) { + return nil, ErrTestError + } + _, err := getFilteredPaginatedRepos(ctx, nil, func(repo string) bool { return true }, log, + &gql_generated.PageInput{}, metaDB) + So(err, ShouldNotBeNil) + }) + Convey("FilterImageMeta", func() { + metaDB.FilterImageMetaFn = func(ctx context.Context, digests []string) (map[string]mTypes.ImageMeta, error) { + return nil, ErrTestError + } + _, err := getFilteredPaginatedRepos(ctx, nil, func(repo string) bool { return true }, log, + &gql_generated.PageInput{}, metaDB) + So(err, ShouldNotBeNil) + }) + Convey("PaginatedRepoMeta2RepoSummaries", func() { + _, err := getFilteredPaginatedRepos(ctx, nil, func(repo string) bool { return true }, log, + &gql_generated.PageInput{Limit: ref(-10)}, metaDB) + So(err, ShouldNotBeNil) + }) + }) +} + func TestGetBookmarkedRepos(t *testing.T) { Convey("getBookmarkedRepos", t, func() { responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, @@ -426,6 +459,24 @@ func TestImageListForDigest(t *testing.T) { }) } +func TestGetImageSummaryError(t *testing.T) { + Convey("getImageSummary", t, func() { + metaDB := mocks.MetaDBMock{ + GetRepoMetaFn: func(ctx context.Context, repo string) (mTypes.RepoMeta, error) { + return mTypes.RepoMeta{Tags: map[string]mTypes.Descriptor{"tag": {}}}, nil + }, + FilterImageMetaFn: func(ctx context.Context, digests []string) (map[string]mTypes.ImageMeta, error) { + return nil, ErrTestError + }, + } + log := log.NewLogger("debug", "") + + _, err := getImageSummary(context.Background(), "repo", "tag", nil, convert.SkipQGLField{}, + metaDB, nil, log) + So(err, ShouldNotBeNil) + }) +} + func TestImageListError(t *testing.T) { Convey("getImageList", t, func() { testLogger := log.NewLogger("debug", "/dev/null") @@ -588,6 +639,18 @@ func TestQueryResolverErrors(t *testing.T) { So(err, ShouldNotBeNil) }) + Convey("GlobalSearch error filte image meta", func() { + resolverConfig := NewResolver(log, storage.StoreController{}, mocks.MetaDBMock{ + FilterImageMetaFn: func(ctx context.Context, digests []string) (map[string]mTypes.ImageMeta, error) { + return nil, ErrTestError + }, + }, mocks.CveInfoMock{}) + resolver := queryResolver{resolverConfig} + + _, err := resolver.GlobalSearch(ctx, "some_string", &gql_generated.Filter{}, getPageInput(1, 1)) + So(err, ShouldNotBeNil) + }) + Convey("ImageListForCve error in GetMultipleRepoMeta", func() { resolverConfig := NewResolver( log, @@ -651,6 +714,30 @@ func TestQueryResolverErrors(t *testing.T) { So(err, ShouldNotBeNil) }) + Convey("RepoListWithNewestImage repoListWithNewestImage() filter image meta error", func() { + resolverConfig := NewResolver( + log, + storage.StoreController{ + DefaultStore: mocks.MockedImageStore{}, + }, + mocks.MetaDBMock{ + SearchReposFn: func(ctx context.Context, searchText string, + ) ([]mTypes.RepoMeta, error) { + return []mTypes.RepoMeta{}, nil + }, + FilterImageMetaFn: func(ctx context.Context, digests []string) (map[string]mTypes.ImageMeta, error) { + return nil, ErrTestError + }, + }, + mocks.CveInfoMock{}, + ) + + qr := queryResolver{resolverConfig} + + _, err := qr.RepoListWithNewestImage(ctx, &gql_generated.PageInput{}) + So(err, ShouldNotBeNil) + }) + Convey("RepoListWithNewestImage repoListWithNewestImage() errors mocked StoreController", func() { resolverConfig := NewResolver( log, diff --git a/pkg/meta/boltdb/boltdb.go b/pkg/meta/boltdb/boltdb.go index 904f5e3b02..9531fae919 100644 --- a/pkg/meta/boltdb/boltdb.go +++ b/pkg/meta/boltdb/boltdb.go @@ -222,10 +222,7 @@ func (bdw *BoltDB) SetRepoReference(ctx context.Context, repo string, reference return err } - protoRepoMeta, repoBlobs, err = common.AddImageMetaToRepoMeta(protoRepoMeta, repoBlobs, reference, imageMeta) - if err != nil { - return err - } + protoRepoMeta, repoBlobs = common.AddImageMetaToRepoMeta(protoRepoMeta, repoBlobs, reference, imageMeta) err = setProtoRepoBlobs(repoBlobs, repoBlobsBuck) if err != nil { @@ -282,7 +279,7 @@ func unmarshalProtoRepoMeta(repo string, repoMetaBlob []byte) (*proto_go.RepoMet if len(repoMetaBlob) > 0 { err := proto.Unmarshal(repoMetaBlob, protoRepoMeta) if err != nil { - return nil, err + return protoRepoMeta, err } } @@ -698,7 +695,7 @@ func (bdw *BoltDB) GetFullImageMeta(ctx context.Context, repo string, tag string repoMetaBlob := buck.Get([]byte(repo)) // object not found - if repoMetaBlob == nil { + if len(repoMetaBlob) == 0 { return zerr.ErrRepoMetaNotFound } @@ -922,7 +919,7 @@ func (bdw *BoltDB) DeleteSignature(repo string, signedManifestDigest godigest.Di repoMetaBuck := tx.Bucket([]byte(RepoMetaBuck)) repoMetaBlob := repoMetaBuck.Get([]byte(repo)) - if repoMetaBlob == nil { + if len(repoMetaBlob) == 0 { return zerr.ErrManifestMetaNotFound } @@ -961,7 +958,7 @@ func (bdw *BoltDB) IncrementRepoStars(repo string) error { repoMetaBuck := tx.Bucket([]byte(RepoMetaBuck)) repoMetaBlob := repoMetaBuck.Get([]byte(repo)) - if repoMetaBlob == nil { + if len(repoMetaBlob) == 0 { return zerr.ErrRepoMetaNotFound } @@ -983,7 +980,7 @@ func (bdw *BoltDB) DecrementRepoStars(repo string) error { repoMetaBuck := tx.Bucket([]byte(RepoMetaBuck)) repoMetaBlob := repoMetaBuck.Get([]byte(repo)) - if repoMetaBlob == nil { + if len(repoMetaBlob) == 0 { return zerr.ErrRepoMetaNotFound } @@ -1091,7 +1088,7 @@ func (bdw *BoltDB) UpdateStatsOnDownload(repo string, reference string) error { repoMetaBuck := tx.Bucket([]byte(RepoMetaBuck)) repoMetaBlob := repoMetaBuck.Get([]byte(repo)) - if repoMetaBlob == nil { + if len(repoMetaBlob) == 0 { return zerr.ErrRepoMetaNotFound } @@ -1102,8 +1099,7 @@ func (bdw *BoltDB) UpdateStatsOnDownload(repo string, reference string) error { manifestDigest := reference - if !common.ReferenceIsDigest(reference) { - // search digest for tag + if common.ReferenceIsTag(reference) { descriptor, found := protoRepoMeta.Tags[reference] if !found { @@ -1156,7 +1152,7 @@ func (bdw *BoltDB) UpdateSignaturesValidity(repo string, manifestDigest godigest repoBuck := transaction.Bucket([]byte(RepoMetaBuck)) repoMetaBlob := repoBuck.Get([]byte(repo)) - if repoMetaBlob == nil { + if len(repoMetaBlob) == 0 { return zerr.ErrRepoMetaNotFound } @@ -1290,10 +1286,7 @@ func (bdw *BoltDB) RemoveRepoReference(repo, reference string, manifestDigest go return err } - protoRepoMeta, repoBlobs, err = common.RemoveImageFromRepoMeta(protoRepoMeta, repoBlobs, reference) - if err != nil { - return err - } + protoRepoMeta, repoBlobs = common.RemoveImageFromRepoMeta(protoRepoMeta, repoBlobs, reference) repoBlobsBytes, err = proto.Marshal(repoBlobs) if err != nil { @@ -1359,7 +1352,7 @@ func (bdw *BoltDB) ToggleStarRepo(ctx context.Context, repo string) (mTypes.Togg repoBuck := tx.Bucket([]byte(RepoMetaBuck)) repoMetaBlob := repoBuck.Get([]byte(repo)) - if repoMetaBlob == nil { + if len(repoMetaBlob) == 0 { return zerr.ErrRepoMetaNotFound } @@ -1935,16 +1928,12 @@ func (bdw *BoltDB) ResetDB() error { } func resetBucket(transaction *bbolt.Tx, bucketName string) error { - bucket := transaction.Bucket([]byte(bucketName)) - - if bucket != nil { - err := transaction.DeleteBucket([]byte(bucketName)) - if err != nil { - return err - } + err := transaction.DeleteBucket([]byte(bucketName)) + if err != nil { + return err } - _, err := transaction.CreateBucketIfNotExists([]byte(bucketName)) + _, err = transaction.CreateBucketIfNotExists([]byte(bucketName)) return err } diff --git a/pkg/meta/boltdb/boltdb_test.go b/pkg/meta/boltdb/boltdb_test.go index f606df1d3f..bd99373554 100644 --- a/pkg/meta/boltdb/boltdb_test.go +++ b/pkg/meta/boltdb/boltdb_test.go @@ -8,27 +8,42 @@ import ( "testing" "time" - "github.com/opencontainers/go-digest" + godigest "github.com/opencontainers/go-digest" + ispec "github.com/opencontainers/image-spec/specs-go/v1" . "github.com/smartystreets/goconvey/convey" "go.etcd.io/bbolt" + "google.golang.org/protobuf/proto" zerr "zotregistry.io/zot/errors" "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/meta/boltdb" + proto_go "zotregistry.io/zot/pkg/meta/proto/gen" mTypes "zotregistry.io/zot/pkg/meta/types" reqCtx "zotregistry.io/zot/pkg/requestcontext" + . "zotregistry.io/zot/pkg/test/image-utils" ) type imgTrustStore struct{} func (its imgTrustStore) VerifySignature( - signatureType string, rawSignature []byte, sigKey string, manifestDigest digest.Digest, imageMeta mTypes.ImageMeta, + signatureType string, rawSignature []byte, sigKey string, manifestDigest godigest.Digest, imageMeta mTypes.ImageMeta, repo string, ) (string, time.Time, bool, error) { return "", time.Time{}, false, nil } func TestWrapperErrors(t *testing.T) { + image := CreateDefaultImage() + imageMeta := image.AsImageMeta() + multiarchImageMeta := CreateMultiarchWith().Images([]Image{image}).Build().AsImageMeta() + + badProtoBlob := []byte("bad-repo-meta") + + goodRepoMetaBlob, err := proto.Marshal(&proto_go.RepoMeta{Name: "repo"}) + if err != nil { + t.FailNow() + } + Convey("Errors", t, func() { tmpDir := t.TempDir() boltDBParams := boltdb.DBParameters{RootDir: tmpDir} @@ -48,6 +63,468 @@ func TestWrapperErrors(t *testing.T) { ctx := userAc.DeriveContext(context.Background()) + Convey("ResetDB", func() { + Convey("RepoMetaBuck", func() { + err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { + return tx.DeleteBucket([]byte(boltdb.RepoMetaBuck)) + }) + So(err, ShouldBeNil) + + err = boltdbWrapper.ResetDB() + So(err, ShouldNotBeNil) + }) + + Convey("ImageMetaBuck", func() { + err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { + return tx.DeleteBucket([]byte(boltdb.ImageMetaBuck)) + }) + So(err, ShouldBeNil) + + err = boltdbWrapper.ResetDB() + So(err, ShouldNotBeNil) + }) + + Convey("RepoBlobsBuck", func() { + err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { + return tx.DeleteBucket([]byte(boltdb.RepoBlobsBuck)) + }) + So(err, ShouldBeNil) + + err = boltdbWrapper.ResetDB() + So(err, ShouldNotBeNil) + }) + + Convey("UserAPIKeysBucket", func() { + err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { + return tx.DeleteBucket([]byte(boltdb.UserAPIKeysBucket)) + }) + So(err, ShouldBeNil) + + err = boltdbWrapper.ResetDB() + So(err, ShouldNotBeNil) + }) + + Convey("UserDataBucket", func() { + err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { + return tx.DeleteBucket([]byte(boltdb.UserDataBucket)) + }) + So(err, ShouldBeNil) + + err = boltdbWrapper.ResetDB() + So(err, ShouldNotBeNil) + }) + }) + + Convey("RemoveRepoReference", func() { + Convey("getProtoRepoMeta errors", func() { + err := setRepoMeta("repo", badProtoBlob, boltdbWrapper.DB) + So(err, ShouldBeNil) + + err = boltdbWrapper.RemoveRepoReference("repo", "ref", imageMeta.Digest) + So(err, ShouldNotBeNil) + }) + + Convey("getProtoImageMeta errors", func() { + err := boltdbWrapper.SetRepoMeta("repo", mTypes.RepoMeta{ + Name: "repo", + Tags: map[string]mTypes.Descriptor{ + "tag": { + MediaType: ispec.MediaTypeImageManifest, + Digest: imageMeta.Digest.String(), + }, + }, + }) + So(err, ShouldBeNil) + + err = setImageMeta(imageMeta.Digest, badProtoBlob, boltdbWrapper.DB) + So(err, ShouldBeNil) + + err = boltdbWrapper.RemoveRepoReference("repo", "ref", imageMeta.Digest) + So(err, ShouldNotBeNil) + }) + + Convey("unmarshalProtoRepoBlobs errors", func() { + err := boltdbWrapper.SetRepoReference(ctx, "repo", "tag", imageMeta) + So(err, ShouldBeNil) + + err = setRepoBlobInfo("repo", badProtoBlob, boltdbWrapper.DB) + So(err, ShouldBeNil) + + err = boltdbWrapper.RemoveRepoReference("repo", "ref", imageMeta.Digest) + So(err, ShouldNotBeNil) + }) + }) + + Convey("UpdateSignaturesValidity", func() { + boltdbWrapper.SetImageTrustStore(imgTrustStore{}) + digest := image.Digest() + + Convey("image meta blob not found", func() { + err := boltdbWrapper.UpdateSignaturesValidity("repo", digest) + So(err, ShouldBeNil) + }) + + Convey("image meta unmarshal fail", func() { + err := setImageMeta(digest, badProtoBlob, boltdbWrapper.DB) + So(err, ShouldBeNil) + + err = boltdbWrapper.UpdateSignaturesValidity("repo", digest) + So(err, ShouldNotBeNil) + }) + + Convey("repo meta blob not found", func() { + err := boltdbWrapper.SetImageMeta(digest, imageMeta) + So(err, ShouldBeNil) + + err = boltdbWrapper.UpdateSignaturesValidity("repo", digest) + So(err, ShouldNotBeNil) + }) + + Convey("repo meta unmarshal fail", func() { + err := boltdbWrapper.SetImageMeta(digest, imageMeta) + So(err, ShouldBeNil) + + err = setRepoMeta("repo", badProtoBlob, boltdbWrapper.DB) + So(err, ShouldBeNil) + + err = boltdbWrapper.UpdateSignaturesValidity("repo", digest) + So(err, ShouldNotBeNil) + }) + }) + + Convey("UpdateStatsOnDownload", func() { + Convey("repo meta not found", func() { + err = boltdbWrapper.UpdateStatsOnDownload("repo", "ref") + So(err, ShouldNotBeNil) + }) + + Convey("unmarshalProtoRepoMeta error", func() { + err := setRepoMeta("repo", badProtoBlob, boltdbWrapper.DB) + So(err, ShouldBeNil) + + err = boltdbWrapper.UpdateStatsOnDownload("repo", "ref") + So(err, ShouldNotBeNil) + }) + + Convey("ref is tag and tag is not found", func() { + err := boltdbWrapper.SetRepoReference(ctx, "repo", "tag", imageMeta) + So(err, ShouldBeNil) + + err = boltdbWrapper.UpdateStatsOnDownload("repo", "not-found-tag") + So(err, ShouldNotBeNil) + }) + + Convey("digest not found in statistics", func() { + err := boltdbWrapper.SetRepoReference(ctx, "repo", "tag", imageMeta) + So(err, ShouldBeNil) + + err = boltdbWrapper.UpdateStatsOnDownload("repo", godigest.FromString("not-found").String()) + So(err, ShouldNotBeNil) + }) + }) + + Convey("GetReferrersInfo", func() { + Convey("unmarshalProtoRepoMeta error", func() { + err := setRepoMeta("repo", badProtoBlob, boltdbWrapper.DB) + So(err, ShouldBeNil) + + _, err = boltdbWrapper.GetReferrersInfo("repo", "refDig", []string{}) + So(err, ShouldNotBeNil) + }) + }) + + Convey("ResetRepoReferences", func() { + Convey("unmarshalProtoRepoMeta error", func() { + err := setRepoMeta("repo", badProtoBlob, boltdbWrapper.DB) + So(err, ShouldBeNil) + + err = boltdbWrapper.ResetRepoReferences("repo") + So(err, ShouldNotBeNil) + }) + }) + + Convey("DecrementRepoStars", func() { + Convey("unmarshalProtoRepoMeta error", func() { + err := setRepoMeta("repo", badProtoBlob, boltdbWrapper.DB) + So(err, ShouldBeNil) + + err = boltdbWrapper.DecrementRepoStars("repo") + So(err, ShouldNotBeNil) + }) + }) + + Convey("IncrementRepoStars", func() { + Convey("unmarshalProtoRepoMeta error", func() { + err := setRepoMeta("repo", badProtoBlob, boltdbWrapper.DB) + So(err, ShouldBeNil) + + err = boltdbWrapper.IncrementRepoStars("repo") + So(err, ShouldNotBeNil) + }) + }) + + Convey("DeleteSignature", func() { + Convey("repo meta not found", func() { + err = boltdbWrapper.DeleteSignature("repo", godigest.FromString("dig"), mTypes.SignatureMetadata{}) + So(err, ShouldNotBeNil) + }) + + Convey("unmarshalProtoRepoMeta error", func() { + err := setRepoMeta("repo", badProtoBlob, boltdbWrapper.DB) + So(err, ShouldBeNil) + + err = boltdbWrapper.DeleteSignature("repo", godigest.FromString("dig"), mTypes.SignatureMetadata{}) + So(err, ShouldNotBeNil) + }) + }) + + Convey("AddManifestSignature", func() { + Convey("unmarshalProtoRepoMeta error", func() { + err := setRepoMeta("repo", badProtoBlob, boltdbWrapper.DB) + So(err, ShouldBeNil) + + err = boltdbWrapper.AddManifestSignature("repo", godigest.FromString("dig"), mTypes.SignatureMetadata{}) + So(err, ShouldNotBeNil) + }) + }) + + Convey("GetMultipleRepoMeta", func() { + Convey("unmarshalProtoRepoMeta error", func() { + err := setRepoMeta("repo", badProtoBlob, boltdbWrapper.DB) + So(err, ShouldBeNil) + + _, err = boltdbWrapper.GetMultipleRepoMeta(ctx, func(repoMeta mTypes.RepoMeta) bool { return true }) + So(err, ShouldNotBeNil) + }) + }) + + Convey("GetFullImageMeta", func() { + Convey("repo meta not found", func() { + _, err := boltdbWrapper.GetFullImageMeta(ctx, "repo", "tag") + So(err, ShouldNotBeNil) + }) + + Convey("unmarshalProtoRepoMeta fails", func() { + err := setRepoMeta("repo", badProtoBlob, boltdbWrapper.DB) + So(err, ShouldBeNil) + + _, err = boltdbWrapper.GetFullImageMeta(ctx, "repo", "tag") + So(err, ShouldNotBeNil) + }) + + Convey("tag not found", func() { + err := setRepoMeta("repo", goodRepoMetaBlob, boltdbWrapper.DB) + So(err, ShouldBeNil) + + _, err = boltdbWrapper.GetFullImageMeta(ctx, "repo", "tag-not-found") + So(err, ShouldNotBeNil) + }) + + Convey("getProtoImageMeta fails", func() { + err := boltdbWrapper.SetRepoMeta("repo", mTypes.RepoMeta{ + Name: "repo", + Tags: map[string]mTypes.Descriptor{ + "tag": { + MediaType: ispec.MediaTypeImageManifest, + Digest: godigest.FromString("not-found").String(), + }, + }, + }) + So(err, ShouldBeNil) + + _, err = boltdbWrapper.GetFullImageMeta(ctx, "repo", "tag") + So(err, ShouldNotBeNil) + }) + + Convey("image is index, fail to get manifests", func() { + err := boltdbWrapper.SetImageMeta(multiarchImageMeta.Digest, multiarchImageMeta) + So(err, ShouldBeNil) + + err = boltdbWrapper.SetRepoMeta("repo", mTypes.RepoMeta{ + Name: "repo", + Tags: map[string]mTypes.Descriptor{ + "tag": { + MediaType: ispec.MediaTypeImageIndex, + Digest: multiarchImageMeta.Digest.String(), + }, + }, + }) + So(err, ShouldBeNil) + + _, err = boltdbWrapper.GetFullImageMeta(ctx, "repo", "tag") + So(err, ShouldNotBeNil) + }) + }) + + Convey("FilterRepos", func() { + err := setRepoMeta("repo", badProtoBlob, boltdbWrapper.DB) + So(err, ShouldBeNil) + + _, err = boltdbWrapper.FilterRepos(ctx, mTypes.AcceptAllRepoNames, mTypes.AcceptAllRepoMeta) + So(err, ShouldNotBeNil) + }) + + Convey("SearchTags", func() { + Convey("unmarshalProtoRepoMeta fails", func() { + err := setRepoMeta("repo", badProtoBlob, boltdbWrapper.DB) + So(err, ShouldBeNil) + + // manifests are missing + _, err = boltdbWrapper.SearchTags(ctx, "repo:") + So(err, ShouldNotBeNil) + }) + + Convey("found repo meta", func() { + Convey("bad image manifest", func() { + badImageDigest := godigest.FromString("bad-image-manifest") + err := boltdbWrapper.SetRepoMeta("repo", mTypes.RepoMeta{ + Name: "repo", + Tags: map[string]mTypes.Descriptor{ + "bad-image-manifest": { + MediaType: ispec.MediaTypeImageManifest, + Digest: badImageDigest.String(), + }, + }, + }) + So(err, ShouldBeNil) + + err = setImageMeta(badImageDigest, badProtoBlob, boltdbWrapper.DB) + So(err, ShouldBeNil) + + _, err = boltdbWrapper.SearchTags(ctx, "repo:") + So(err, ShouldNotBeNil) + }) + Convey("bad image index", func() { + badIndexDigest := godigest.FromString("bad-image-manifest") + err := boltdbWrapper.SetRepoMeta("repo", mTypes.RepoMeta{ + Name: "repo", + Tags: map[string]mTypes.Descriptor{ + "bad-image-index": { + MediaType: ispec.MediaTypeImageIndex, + Digest: badIndexDigest.String(), + }, + }, + }) + So(err, ShouldBeNil) + + err = setImageMeta(badIndexDigest, badProtoBlob, boltdbWrapper.DB) + So(err, ShouldBeNil) + + _, err = boltdbWrapper.SearchTags(ctx, "repo:") + So(err, ShouldNotBeNil) + }) + Convey("good image index, bad inside manifest", func() { + goodIndexBadManifestDigest := godigest.FromString("good-index-bad-manifests") + err := boltdbWrapper.SetRepoMeta("repo", mTypes.RepoMeta{ + Name: "repo", + Tags: map[string]mTypes.Descriptor{ + "good-index-bad-manifests": { + MediaType: ispec.MediaTypeImageIndex, + Digest: goodIndexBadManifestDigest.String(), + }, + }, + }) + So(err, ShouldBeNil) + + err = boltdbWrapper.SetImageMeta(goodIndexBadManifestDigest, multiarchImageMeta) + So(err, ShouldBeNil) + + err = setImageMeta(image.Digest(), badProtoBlob, boltdbWrapper.DB) + So(err, ShouldBeNil) + + _, err = boltdbWrapper.SearchTags(ctx, "repo:") + So(err, ShouldNotBeNil) + }) + Convey("bad media type", func() { + err := boltdbWrapper.SetRepoMeta("repo", mTypes.RepoMeta{ + Name: "repo", + Tags: map[string]mTypes.Descriptor{ + "mad-media-type": { + MediaType: "bad media type", + Digest: godigest.FromString("dig").String(), + }, + }, + }) + So(err, ShouldBeNil) + + _, err = boltdbWrapper.SearchTags(ctx, "repo:") + So(err, ShouldBeNil) + }) + }) + }) + + Convey("FilterTags", func() { + Convey("unmarshalProtoRepoMeta fails", func() { + err := setRepoMeta("repo", badProtoBlob, boltdbWrapper.DB) + So(err, ShouldBeNil) + + _, err = boltdbWrapper.FilterTags(ctx, mTypes.AcceptAllRepoTag, mTypes.AcceptAllImageMeta) + So(err, ShouldNotBeNil) + }) + + Convey("bad media Type fails", func() { + err := boltdbWrapper.SetRepoMeta("repo", mTypes.RepoMeta{ + Name: "repo", + Tags: map[string]mTypes.Descriptor{ + "bad-repo-meta": { + MediaType: "bad media type", + Digest: godigest.FromString("dig").String(), + }, + }, + }) + So(err, ShouldBeNil) + + _, err = boltdbWrapper.FilterTags(ctx, mTypes.AcceptAllRepoTag, mTypes.AcceptAllImageMeta) + So(err, ShouldBeNil) + }) + }) + + Convey("SearchRepos", func() { + Convey("unmarshalProtoRepoMeta fails", func() { + err := setRepoMeta("repo", badProtoBlob, boltdbWrapper.DB) + So(err, ShouldBeNil) + + // manifests are missing + _, err = boltdbWrapper.SearchRepos(ctx, "repo") + So(err, ShouldNotBeNil) + }) + }) + + Convey("FilterImageMeta", func() { + Convey("MediaType ImageIndex, getProtoImageMeta fails", func() { + err := boltdbWrapper.SetImageMeta(multiarchImageMeta.Digest, multiarchImageMeta) + So(err, ShouldBeNil) + + err = setImageMeta(image.Digest(), badProtoBlob, boltdbWrapper.DB) + So(err, ShouldBeNil) + + // manifests are missing + _, err = boltdbWrapper.FilterImageMeta(ctx, []string{multiarchImageMeta.Digest.String()}) + So(err, ShouldNotBeNil) + }) + }) + + Convey("SetRepoReference", func() { + Convey("getProtoRepoMeta errors", func() { + err := setRepoMeta("repo", badProtoBlob, boltdbWrapper.DB) + So(err, ShouldBeNil) + + err = boltdbWrapper.SetRepoReference(ctx, "repo", "tag", imageMeta) + So(err, ShouldNotBeNil) + }) + + Convey("unmarshalProtoRepoBlobs errors", func() { + err := setRepoMeta("repo", goodRepoMetaBlob, boltdbWrapper.DB) + So(err, ShouldBeNil) + + err = setRepoBlobInfo("repo", badProtoBlob, boltdbWrapper.DB) + So(err, ShouldBeNil) + + err = boltdbWrapper.SetRepoReference(ctx, "repo", "tag", imageMeta) + So(err, ShouldNotBeNil) + }) + }) + Convey("AddUserAPIKey", func() { Convey("no userid found", func() { userAc := reqCtx.NewUserAccessControl() @@ -389,3 +866,33 @@ func TestWrapperErrors(t *testing.T) { }) }) } + +func setRepoMeta(repo string, blob []byte, db *bbolt.DB) error { //nolint: unparam + err := db.Update(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(boltdb.RepoMetaBuck)) + + return buck.Put([]byte(repo), blob) + }) + + return err +} + +func setImageMeta(digest godigest.Digest, blob []byte, db *bbolt.DB) error { + err := db.Update(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(boltdb.ImageMetaBuck)) + + return buck.Put([]byte(digest.String()), blob) + }) + + return err +} + +func setRepoBlobInfo(repo string, blob []byte, db *bbolt.DB) error { + err := db.Update(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(boltdb.RepoBlobsBuck)) + + return buck.Put([]byte(repo), blob) + }) + + return err +} diff --git a/pkg/meta/common/common.go b/pkg/meta/common/common.go index a1c093340a..6920a5fd48 100644 --- a/pkg/meta/common/common.go +++ b/pkg/meta/common/common.go @@ -40,6 +40,10 @@ func ReferenceIsDigest(reference string) bool { return err == nil } +func ReferenceIsTag(reference string) bool { + return !ReferenceIsDigest(reference) +} + func ValidateRepoReferenceInput(repo, reference string, manifestDigest godigest.Digest) error { if repo == "" { return zerr.ErrEmptyRepoName @@ -188,7 +192,7 @@ func CheckImageLastUpdated(repoLastUpdated time.Time, isSigned bool, noImageChec func AddImageMetaToRepoMeta(repoMeta *proto_go.RepoMeta, repoBlobs *proto_go.RepoBlobs, reference string, imageMeta mTypes.ImageMeta, -) (*proto_go.RepoMeta, *proto_go.RepoBlobs, error) { +) (*proto_go.RepoMeta, *proto_go.RepoBlobs) { switch imageMeta.MediaType { case ispec.MediaTypeImageManifest: manifestData := imageMeta.Manifests[0] @@ -241,7 +245,7 @@ func AddImageMetaToRepoMeta(repoMeta *proto_go.RepoMeta, repoBlobs *proto_go.Rep // update info only when a tag is added if zcommon.IsDigest(reference) { - return repoMeta, repoBlobs, nil + return repoMeta, repoBlobs } size, platforms, vendors := recalculateAggregateFields(repoMeta, repoBlobs) @@ -258,11 +262,11 @@ func AddImageMetaToRepoMeta(repoMeta *proto_go.RepoMeta, repoBlobs *proto_go.Rep Tag: reference, }) - return repoMeta, repoBlobs, nil + return repoMeta, repoBlobs } func RemoveImageFromRepoMeta(repoMeta *proto_go.RepoMeta, repoBlobs *proto_go.RepoBlobs, ref string, -) (*proto_go.RepoMeta, *proto_go.RepoBlobs, error) { +) (*proto_go.RepoMeta, *proto_go.RepoBlobs) { var updatedLastImage *proto_go.RepoLastUpdatedImage updatedBlobs := map[string]*proto_go.BlobInfo{} @@ -308,7 +312,7 @@ func RemoveImageFromRepoMeta(repoMeta *proto_go.RepoMeta, repoBlobs *proto_go.Re repoBlobs.Blobs = updatedBlobs - return repoMeta, repoBlobs, nil + return repoMeta, repoBlobs } func recalculateAggregateFields(repoMeta *proto_go.RepoMeta, repoBlobs *proto_go.RepoBlobs, diff --git a/pkg/meta/convert/convert.go b/pkg/meta/convert/convert.go index 6ce297fd23..07db40e079 100644 --- a/pkg/meta/convert/convert.go +++ b/pkg/meta/convert/convert.go @@ -327,7 +327,7 @@ func GetImageManifestMeta(manifestContent ispec.Manifest, configContent ispec.Im MediaType: ispec.MediaTypeImageManifest, Digest: digest, Size: size, - Manifests: []mTypes.ManifestData{ + Manifests: []mTypes.ManifestMeta{ { Digest: digest, Size: size, @@ -361,11 +361,11 @@ func GetTags(tags map[string]*proto_go.TagDescriptor) map[string]mTypes.Descript return resultMap } -func GetManifests(descriptors []ispec.Descriptor) []mTypes.ManifestData { - manifestList := []mTypes.ManifestData{} +func GetManifests(descriptors []ispec.Descriptor) []mTypes.ManifestMeta { + manifestList := []mTypes.ManifestMeta{} for _, manifest := range descriptors { - manifestList = append(manifestList, mTypes.ManifestData{ + manifestList = append(manifestList, mTypes.ManifestMeta{ Digest: manifest.Digest, Size: manifest.Size, }) @@ -384,6 +384,10 @@ func GetTime(time *timestamppb.Timestamp) *time.Time { func GetFullImageMetaFromProto(tag string, protoRepoMeta *proto_go.RepoMeta, protoImageMeta *proto_go.ImageMeta, ) mTypes.FullImageMeta { + if protoRepoMeta == nil { + return mTypes.FullImageMeta{} + } + imageMeta := GetImageMeta(protoImageMeta) imageDigest := imageMeta.Digest.String() @@ -404,7 +408,7 @@ func GetFullImageMetaFromProto(tag string, protoRepoMeta *proto_go.RepoMeta, pro } } -func GetFullManifestData(protoRepoMeta *proto_go.RepoMeta, manifestData []mTypes.ManifestData, +func GetFullManifestData(protoRepoMeta *proto_go.RepoMeta, manifestData []mTypes.ManifestMeta, ) []mTypes.FullManifestMeta { if protoRepoMeta == nil { return []mTypes.FullManifestMeta{} @@ -414,7 +418,7 @@ func GetFullManifestData(protoRepoMeta *proto_go.RepoMeta, manifestData []mTypes for i := range manifestData { results = append(results, mTypes.FullManifestMeta{ - ManifestData: manifestData[i], + ManifestMeta: manifestData[i], Referrers: GetImageReferrers(protoRepoMeta.Referrers[manifestData[i].Digest.String()]), Statistics: GetImageStatistics(protoRepoMeta.Statistics[manifestData[i].Digest.String()]), Signatures: GetImageSignatures(protoRepoMeta.Signatures[manifestData[i].Digest.String()]), @@ -511,6 +515,10 @@ func GetLastUpdatedImage(protoLastUpdated *proto_go.RepoLastUpdatedImage) *mType } func GetImageMeta(dbImageMeta *proto_go.ImageMeta) mTypes.ImageMeta { + if dbImageMeta == nil { + return mTypes.ImageMeta{} + } + imageMeta := mTypes.ImageMeta{ MediaType: dbImageMeta.MediaType, Size: GetImageManifestSize(dbImageMeta), @@ -538,10 +546,10 @@ func GetImageMeta(dbImageMeta *proto_go.ImageMeta) mTypes.ImageMeta { } } - manifestDataList := make([]mTypes.ManifestData, 0, len(dbImageMeta.Manifests)) + manifestDataList := make([]mTypes.ManifestMeta, 0, len(dbImageMeta.Manifests)) for _, manifest := range dbImageMeta.Manifests { - manifestDataList = append(manifestDataList, mTypes.ManifestData{ + manifestDataList = append(manifestDataList, mTypes.ManifestMeta{ Size: manifest.Size, Digest: godigest.Digest(manifest.Digest), Manifest: ispec.Manifest{ diff --git a/pkg/meta/convert/convert_test.go b/pkg/meta/convert/convert_test.go new file mode 100644 index 0000000000..a8b7aec2fa --- /dev/null +++ b/pkg/meta/convert/convert_test.go @@ -0,0 +1,72 @@ +package convert_test + +import ( + "testing" + + ispec "github.com/opencontainers/image-spec/specs-go/v1" + . "github.com/smartystreets/goconvey/convey" + + "zotregistry.io/zot/pkg/meta/convert" + "zotregistry.io/zot/pkg/meta/proto/gen" +) + +func TestConvertErrors(t *testing.T) { + Convey("Errors", t, func() { + Convey("GetImageArtifactType", func() { + str := convert.GetImageArtifactType(&gen.ImageMeta{MediaType: "bad-media-type"}) + So(str, ShouldResemble, "") + }) + Convey("GetImageManifestSize", func() { + size := convert.GetImageManifestSize(&gen.ImageMeta{MediaType: "bad-media-type"}) + So(size, ShouldEqual, 0) + }) + Convey("GetImageDigest", func() { + dig := convert.GetImageDigest(&gen.ImageMeta{MediaType: "bad-media-type"}) + So(dig.String(), ShouldResemble, "") + }) + Convey("GetImageDigestStr", func() { + digStr := convert.GetImageDigestStr(&gen.ImageMeta{MediaType: "bad-media-type"}) + So(digStr, ShouldResemble, "") + }) + Convey("GetImageAnnotations", func() { + annot := convert.GetImageAnnotations(&gen.ImageMeta{MediaType: "bad-media-type"}) + So(annot, ShouldBeEmpty) + }) + Convey("GetImageSubject", func() { + subjs := convert.GetImageSubject(&gen.ImageMeta{MediaType: "bad-media-type"}) + So(subjs, ShouldBeNil) + }) + Convey("GetDescriptorRef", func() { + ref := convert.GetDescriptorRef(nil) + So(ref, ShouldBeNil) + }) + Convey("GetPlatform", func() { + platf := convert.GetPlatform(nil) + So(platf, ShouldEqual, ispec.Platform{}) + }) + Convey("GetPlatformRef", func() { + platf := convert.GetPlatform(&gen.Platform{Architecture: "arch"}) + So(platf.Architecture, ShouldResemble, "arch") + }) + Convey("GetImageReferrers", func() { + ref := convert.GetImageReferrers(nil) + So(ref, ShouldNotBeNil) + }) + Convey("GetImageSignatures", func() { + sigs := convert.GetImageSignatures(nil) + So(sigs, ShouldNotBeNil) + }) + Convey("GetImageStatistics", func() { + sigs := convert.GetImageStatistics(nil) + So(sigs, ShouldNotBeNil) + }) + Convey("GetFullImageMetaFromProto", func() { + imageMeta := convert.GetFullImageMetaFromProto("tag", nil, nil) + So(imageMeta.Digest.String(), ShouldResemble, "") + }) + Convey("GetFullManifestData", func() { + imageMeta := convert.GetFullManifestData(nil, nil) + So(len(imageMeta), ShouldEqual, 0) + }) + }) +} diff --git a/pkg/meta/dynamodb/dynamodb.go b/pkg/meta/dynamodb/dynamodb.go index 92ec2271ca..9b0c66fe56 100644 --- a/pkg/meta/dynamodb/dynamodb.go +++ b/pkg/meta/dynamodb/dynamodb.go @@ -41,8 +41,7 @@ type DynamoDB struct { Log log.Logger } -func New( - client *dynamodb.Client, params DBDriverParameters, log log.Logger, +func New(client *dynamodb.Client, params DBDriverParameters, log log.Logger, ) (*DynamoDB, error) { dynamoWrapper := DynamoDB{ Client: client, @@ -110,27 +109,27 @@ func (dwr *DynamoDB) SetProtoImageMeta(digest godigest.Digest, protoImageMeta *p return err } - _, err = dwr.Client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{ + _, err = dwr.Client.UpdateItem(context.Background(), &dynamodb.UpdateItemInput{ ExpressionAttributeNames: map[string]string{ - "#ID": "ImageMeta", + "#IM": "ImageMeta", }, ExpressionAttributeValues: map[string]types.AttributeValue{ ":ImageMeta": mdAttributeValue, }, Key: map[string]types.AttributeValue{ "Key": &types.AttributeValueMemberS{ - Value: mConvert.GetImageDigestStr(protoImageMeta), + Value: digest.String(), }, }, TableName: aws.String(dwr.ImageMetaTablename), - UpdateExpression: aws.String("SET #ID = :ImageMeta"), + UpdateExpression: aws.String("SET #IM = :ImageMeta"), }) return err } func (dwr *DynamoDB) SetImageMeta(digest godigest.Digest, imageMeta mTypes.ImageMeta) error { - return dwr.SetProtoImageMeta(imageMeta.Digest, mConvert.GetProtoImageMeta(imageMeta)) + return dwr.SetProtoImageMeta(digest, mConvert.GetProtoImageMeta(imageMeta)) } func (dwr *DynamoDB) GetProtoImageMeta(ctx context.Context, digest godigest.Digest) (*proto_go.ImageMeta, error) { @@ -180,10 +179,10 @@ func (dwr *DynamoDB) setProtoRepoMeta(repo string, repoMeta *proto_go.RepoMeta) _, err = dwr.Client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{ ExpressionAttributeNames: map[string]string{ - "#RM": "RepoMetadata", + "#RM": "RepoMeta", }, ExpressionAttributeValues: map[string]types.AttributeValue{ - ":RepoMetadata": repoAttributeValue, + ":RepoMeta": repoAttributeValue, }, Key: map[string]types.AttributeValue{ "Key": &types.AttributeValueMemberS{ @@ -191,7 +190,7 @@ func (dwr *DynamoDB) setProtoRepoMeta(repo string, repoMeta *proto_go.RepoMeta) }, }, TableName: aws.String(dwr.RepoMetaTablename), - UpdateExpression: aws.String("SET #RM = :RepoMetadata"), + UpdateExpression: aws.String("SET #RM = :RepoMeta"), }) return err @@ -215,7 +214,7 @@ func (dwr *DynamoDB) getProtoRepoMeta(ctx context.Context, repo string) (*proto_ blob := []byte{} if resp.Item != nil { - err = attributevalue.Unmarshal(resp.Item["RepoMetadata"], &blob) + err = attributevalue.Unmarshal(resp.Item["RepoMeta"], &blob) if err != nil { return nil, err } @@ -343,10 +342,7 @@ func (dwr *DynamoDB) SetRepoReference(ctx context.Context, repo string, referenc return err } - repoMeta, repoBlobs, err = common.AddImageMetaToRepoMeta(repoMeta, repoBlobs, reference, imageMeta) - if err != nil { - return err - } + repoMeta, repoBlobs = common.AddImageMetaToRepoMeta(repoMeta, repoBlobs, reference, imageMeta) err = dwr.setRepoBlobsInfo(repo, repoBlobs) //nolint: contextcheck if err != nil { @@ -431,7 +427,7 @@ func (dwr *DynamoDB) SearchRepos(ctx context.Context, searchText string) ([]mTyp userStars := getUserStars(ctx, dwr) repoMetaAttributeIterator := NewBaseDynamoAttributesIterator( - dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log, + dwr.Client, dwr.RepoMetaTablename, "RepoMeta", 0, dwr.Log, ) repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx) @@ -568,7 +564,7 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterRepoTag mTypes.Filter var viewError error repoMetaAttributeIterator := NewBaseDynamoAttributesIterator( - dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log, + dwr.Client, dwr.RepoMetaTablename, "RepoMeta", 0, dwr.Log, ) repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx) @@ -816,7 +812,7 @@ func (dwr *DynamoDB) GetMultipleRepoMeta(ctx context.Context, filter func(repoMe ) repoMetaAttributeIterator = NewBaseDynamoAttributesIterator( - dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log, + dwr.Client, dwr.RepoMetaTablename, "RepoMeta", 0, dwr.Log, ) repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx) @@ -864,7 +860,7 @@ func (dwr *DynamoDB) FilterRepos(ctx context.Context, acceptName mTypes.FilterRe userStars := getUserStars(ctx, dwr) repoMetaAttributeIterator := NewBaseDynamoAttributesIterator( - dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log, + dwr.Client, dwr.RepoMetaTablename, "RepoMeta", 0, dwr.Log, ) repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx) @@ -1296,10 +1292,7 @@ func (dwr *DynamoDB) RemoveRepoReference(repo, reference string, manifestDigest return err } - protoRepoMeta, repoBlobsInfo, err = common.RemoveImageFromRepoMeta(protoRepoMeta, repoBlobsInfo, reference) - if err != nil { - return err - } + protoRepoMeta, repoBlobsInfo = common.RemoveImageFromRepoMeta(protoRepoMeta, repoBlobsInfo, reference) err = dwr.setRepoBlobsInfo(repo, repoBlobsInfo) //nolint: contextcheck if err != nil { @@ -1459,10 +1452,10 @@ func (dwr *DynamoDB) ToggleStarRepo(ctx context.Context, repo string) ( // Update Repo Meta with updated repo stars Update: &types.Update{ ExpressionAttributeNames: map[string]string{ - "#RM": "RepoMetadata", + "#RM": "RepoMeta", }, ExpressionAttributeValues: map[string]types.AttributeValue{ - ":RepoMetadata": repoAttributeValue, + ":RepoMeta": repoAttributeValue, }, Key: map[string]types.AttributeValue{ "Key": &types.AttributeValueMemberS{ @@ -1470,7 +1463,7 @@ func (dwr *DynamoDB) ToggleStarRepo(ctx context.Context, repo string) ( }, }, TableName: aws.String(dwr.RepoMetaTablename), - UpdateExpression: aws.String("SET #RM = :RepoMetadata"), + UpdateExpression: aws.String("SET #RM = :RepoMeta"), }, }, }, diff --git a/pkg/meta/dynamodb/dynamodb_test.go b/pkg/meta/dynamodb/dynamodb_test.go index 7fb3aadd93..f1e716c940 100644 --- a/pkg/meta/dynamodb/dynamodb_test.go +++ b/pkg/meta/dynamodb/dynamodb_test.go @@ -11,6 +11,8 @@ import ( "github.com/aws/aws-sdk-go-v2/service/dynamodb" "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" guuid "github.com/gofrs/uuid" + godigest "github.com/opencontainers/go-digest" + ispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/rs/zerolog" . "github.com/smartystreets/goconvey/convey" @@ -77,7 +79,7 @@ func TestIterator(t *testing.T) { repoMetaAttributeIterator := mdynamodb.NewBaseDynamoAttributesIterator( dynamoWrapper.Client, dynamoWrapper.RepoMetaTablename, - "RepoMetadata", + "RepoMeta", 1, log, ) @@ -118,7 +120,7 @@ func TestIteratorErrors(t *testing.T) { repoMetaAttributeIterator := mdynamodb.NewBaseDynamoAttributesIterator( dynamodb.NewFromConfig(cfg), "RepoMetadataTable", - "RepoMetadata", + "RepoMeta", 1, log.Logger{Logger: zerolog.New(os.Stdout)}, ) @@ -132,6 +134,7 @@ func TestWrapperErrors(t *testing.T) { tskip.SkipDynamo(t) const region = "us-east-2" + endpoint := os.Getenv("DYNAMODBMOCK_ENDPOINT") uuid, err := guuid.NewV4() @@ -148,6 +151,17 @@ func TestWrapperErrors(t *testing.T) { repoBlobsTablename := "RepoBlobs" + uuid.String() log := log.NewLogger("debug", "") + testDigest := godigest.FromString("str") + image := CreateDefaultImage() + multi := CreateMultiarchWith().Images([]Image{image}).Build() + imageMeta := image.AsImageMeta() + multiarchImageMeta := multi.AsImageMeta() + + badProtoBlob := []byte("bad-repo-meta") + // goodRepoMetaBlob, err := proto.Marshal(&proto_go.RepoMeta{Name: "repo"}) + // if err != nil { + // t.FailNow() + // } Convey("Errors", t, func() { params := mdynamodb.DBDriverParameters{ //nolint:contextcheck @@ -172,6 +186,7 @@ func TestWrapperErrors(t *testing.T) { dynamoWrapper.SetImageTrustStore(imgTrustStore) So(dynamoWrapper.ResetTable(dynamoWrapper.RepoMetaTablename), ShouldBeNil) //nolint:contextcheck + So(dynamoWrapper.ResetTable(dynamoWrapper.RepoBlobsTablename), ShouldBeNil) //nolint:contextcheck So(dynamoWrapper.ResetTable(dynamoWrapper.ImageMetaTablename), ShouldBeNil) //nolint:contextcheck So(dynamoWrapper.ResetTable(dynamoWrapper.UserDataTablename), ShouldBeNil) //nolint:contextcheck @@ -179,6 +194,458 @@ func TestWrapperErrors(t *testing.T) { userAc.SetUsername("test") ctx := userAc.DeriveContext(context.Background()) + Convey("RemoveRepoReference", func() { + Convey("getProtoRepoMeta errors", func() { + err := setRepoMeta("repo", badProtoBlob, dynamoWrapper) + So(err, ShouldBeNil) + + err = dynamoWrapper.RemoveRepoReference("repo", "ref", imageMeta.Digest) + So(err, ShouldNotBeNil) + }) + + Convey("getProtoImageMeta errors", func() { + err := dynamoWrapper.SetRepoMeta("repo", mTypes.RepoMeta{ //nolint: contextcheck + Name: "repo", + Tags: map[string]mTypes.Descriptor{ + "tag": { + MediaType: ispec.MediaTypeImageManifest, + Digest: imageMeta.Digest.String(), + }, + }, + }) + So(err, ShouldBeNil) + + err = setImageMeta(imageMeta.Digest, badProtoBlob, dynamoWrapper) + So(err, ShouldBeNil) + + err = dynamoWrapper.RemoveRepoReference("repo", "ref", imageMeta.Digest) + So(err, ShouldNotBeNil) + }) + + Convey("unmarshalProtoRepoBlobs errors", func() { + err := dynamoWrapper.SetRepoReference(ctx, "repo", "tag", imageMeta) + So(err, ShouldBeNil) + + err = setRepoBlobInfo("repo", badProtoBlob, dynamoWrapper) //nolint: contextcheck + So(err, ShouldBeNil) + + err = dynamoWrapper.RemoveRepoReference("repo", "ref", imageMeta.Digest) //nolint: contextcheck + So(err, ShouldNotBeNil) + }) + }) + Convey("FilterImageMeta", func() { + Convey("manifest meta unmarshal error", func() { + err = setImageMeta(image.Digest(), badProtoBlob, dynamoWrapper) //nolint: contextcheck + So(err, ShouldBeNil) + + _, err = dynamoWrapper.FilterImageMeta(ctx, []string{image.DigestStr()}) + So(err, ShouldNotBeNil) + }) + Convey("MediaType ImageIndex, getProtoImageMeta fails", func() { + err := dynamoWrapper.SetImageMeta(multiarchImageMeta.Digest, multiarchImageMeta) //nolint: contextcheck + So(err, ShouldBeNil) + + err = setImageMeta(image.Digest(), badProtoBlob, dynamoWrapper) //nolint: contextcheck + So(err, ShouldBeNil) + + // manifests are missing + _, err = dynamoWrapper.FilterImageMeta(ctx, []string{multiarchImageMeta.Digest.String()}) + So(err, ShouldNotBeNil) + }) + }) + Convey("UpdateSignaturesValidity", func() { + digest := image.Digest() + + Convey("image meta blob not found", func() { + err := dynamoWrapper.UpdateSignaturesValidity("repo", digest) + So(err, ShouldNotBeNil) + }) + + Convey("image meta unmarshal fail", func() { + err := setImageMeta(digest, badProtoBlob, dynamoWrapper) + So(err, ShouldBeNil) + + err = dynamoWrapper.UpdateSignaturesValidity("repo", digest) + So(err, ShouldNotBeNil) + }) + + Convey("repo meta blob not found", func() { + err := dynamoWrapper.SetImageMeta(digest, imageMeta) + So(err, ShouldBeNil) + + err = dynamoWrapper.UpdateSignaturesValidity("repo", digest) + So(err, ShouldNotBeNil) + }) + + Convey("repo meta unmarshal fail", func() { + err := dynamoWrapper.SetImageMeta(digest, imageMeta) + So(err, ShouldBeNil) + + err = setRepoMeta("repo", badProtoBlob, dynamoWrapper) + So(err, ShouldBeNil) + + err = dynamoWrapper.UpdateSignaturesValidity("repo", digest) + So(err, ShouldNotBeNil) + }) + }) + + Convey("UpdateStatsOnDownload", func() { + Convey("unmarshalProtoRepoMeta error", func() { + err := setRepoMeta("repo", badProtoBlob, dynamoWrapper) + So(err, ShouldBeNil) + + err = dynamoWrapper.UpdateStatsOnDownload("repo", "ref") + So(err, ShouldNotBeNil) + }) + + Convey("ref is tag and tag is not found", func() { + err := dynamoWrapper.SetRepoReference(ctx, "repo", "tag", imageMeta) + So(err, ShouldBeNil) + + err = dynamoWrapper.UpdateStatsOnDownload("repo", "not-found-tag") //nolint: contextcheck + So(err, ShouldNotBeNil) + }) + + Convey("digest not found in statistics", func() { + err := dynamoWrapper.SetRepoReference(ctx, "repo", "tag", imageMeta) + So(err, ShouldBeNil) + + err = dynamoWrapper.UpdateStatsOnDownload("repo", godigest.FromString("not-found").String()) //nolint: contextcheck + So(err, ShouldNotBeNil) + }) + }) + Convey("GetReferrersInfo", func() { + Convey("unmarshalProtoRepoMeta error", func() { + err := setRepoMeta("repo", badProtoBlob, dynamoWrapper) + So(err, ShouldBeNil) + + _, err = dynamoWrapper.GetReferrersInfo("repo", "refDig", []string{}) + So(err, ShouldNotBeNil) + }) + }) + Convey("DecrementRepoStars", func() { + Convey("unmarshalProtoRepoMeta error", func() { + err := setRepoMeta("repo", badProtoBlob, dynamoWrapper) + So(err, ShouldBeNil) + + err = dynamoWrapper.DecrementRepoStars("repo") + So(err, ShouldNotBeNil) + }) + }) + + Convey("IncrementRepoStars", func() { + Convey("unmarshalProtoRepoMeta error", func() { + err := setRepoMeta("repo", badProtoBlob, dynamoWrapper) + So(err, ShouldBeNil) + + err = dynamoWrapper.IncrementRepoStars("repo") + So(err, ShouldNotBeNil) + }) + }) + + Convey("GetMultipleRepoMeta", func() { + Convey("repoMetaAttributeIterator.First fails", func() { + dynamoWrapper.RepoMetaTablename = badTablename + _, err := dynamoWrapper.GetMultipleRepoMeta(ctx, func(repoMeta mTypes.RepoMeta) bool { return true }) + So(err, ShouldNotBeNil) + }) + Convey("repo meta unmarshal fails", func() { + err := setRepoMeta("repo", badProtoBlob, dynamoWrapper) //nolint: contextcheck + So(err, ShouldBeNil) + + _, err = dynamoWrapper.GetMultipleRepoMeta(ctx, func(repoMeta mTypes.RepoMeta) bool { return true }) + So(err, ShouldNotBeNil) + }) + }) + Convey("GetImageMeta", func() { + Convey("get image meta fails", func() { + _, err := dynamoWrapper.GetImageMeta(testDigest) + So(err, ShouldNotBeNil) + }) + Convey("image index, get manifest meta fails", func() { + err := dynamoWrapper.SetRepoReference(ctx, "repo", "tag", multiarchImageMeta) + So(err, ShouldBeNil) + + _, err = dynamoWrapper.GetImageMeta(multiarchImageMeta.Digest) //nolint: contextcheck + So(err, ShouldNotBeNil) + }) + }) + Convey("GetFullImageMeta", func() { + Convey("repo meta not found", func() { + _, err := dynamoWrapper.GetFullImageMeta(ctx, "repo", "tag") + So(err, ShouldNotBeNil) + }) + + Convey("unmarshalProtoRepoMeta fails", func() { + err := setRepoMeta("repo", badProtoBlob, dynamoWrapper) //nolint: contextcheck + So(err, ShouldBeNil) + + _, err = dynamoWrapper.GetFullImageMeta(ctx, "repo", "tag") + So(err, ShouldNotBeNil) + }) + + Convey("tag not found", func() { + err := dynamoWrapper.SetRepoReference(ctx, "repo", "tag", imageMeta) + So(err, ShouldBeNil) + + _, err = dynamoWrapper.GetFullImageMeta(ctx, "repo", "tag-not-found") + So(err, ShouldNotBeNil) + }) + + Convey("getProtoImageMeta fails", func() { + err := dynamoWrapper.SetRepoMeta("repo", mTypes.RepoMeta{ //nolint: contextcheck + Name: "repo", + Tags: map[string]mTypes.Descriptor{ + "tag": { + MediaType: ispec.MediaTypeImageManifest, + Digest: godigest.FromString("not-found").String(), + }, + }, + }) + So(err, ShouldBeNil) + + _, err = dynamoWrapper.GetFullImageMeta(ctx, "repo", "tag") + So(err, ShouldNotBeNil) + }) + + Convey("image is index, fail to get manifests", func() { + err := dynamoWrapper.SetImageMeta(multiarchImageMeta.Digest, multiarchImageMeta) //nolint: contextcheck + So(err, ShouldBeNil) + + err = dynamoWrapper.SetRepoMeta("repo", mTypes.RepoMeta{ //nolint: contextcheck + Name: "repo", + Tags: map[string]mTypes.Descriptor{ + "tag": { + MediaType: ispec.MediaTypeImageIndex, + Digest: multiarchImageMeta.Digest.String(), + }, + }, + }) + So(err, ShouldBeNil) + + _, err = dynamoWrapper.GetFullImageMeta(ctx, "repo", "tag") + So(err, ShouldNotBeNil) + }) + }) + + Convey("FilterTags", func() { + Convey("repoMetaAttributeIterator.First fails", func() { + dynamoWrapper.RepoMetaTablename = badTablename + + _, err = dynamoWrapper.FilterTags(ctx, mTypes.AcceptAllRepoTag, mTypes.AcceptAllImageMeta) + So(err, ShouldNotBeNil) + }) + Convey("repo meta unmarshal fails", func() { + err := setRepoMeta("repo", badProtoBlob, dynamoWrapper) //nolint: contextcheck + So(err, ShouldBeNil) + + _, err = dynamoWrapper.FilterTags(ctx, mTypes.AcceptAllRepoTag, mTypes.AcceptAllImageMeta) + So(err, ShouldNotBeNil) + }) + Convey("found repo meta", func() { + Convey("bad image manifest", func() { + badImageDigest := godigest.FromString("bad-image-manifest") + err := dynamoWrapper.SetRepoMeta("repo", mTypes.RepoMeta{ //nolint: contextcheck + Name: "repo", + Tags: map[string]mTypes.Descriptor{ + "bad-image-manifest": { + MediaType: ispec.MediaTypeImageManifest, + Digest: badImageDigest.String(), + }, + }, + }) + So(err, ShouldBeNil) + + err = setImageMeta(badImageDigest, badProtoBlob, dynamoWrapper) //nolint: contextcheck + So(err, ShouldBeNil) + + _, err = dynamoWrapper.FilterTags(ctx, mTypes.AcceptAllRepoTag, mTypes.AcceptAllImageMeta) + So(err, ShouldNotBeNil) + }) + Convey("bad image index", func() { + badIndexDigest := godigest.FromString("bad-image-manifest") + err := dynamoWrapper.SetRepoMeta("repo", mTypes.RepoMeta{ //nolint: contextcheck + Name: "repo", + Tags: map[string]mTypes.Descriptor{ + "bad-image-index": { + MediaType: ispec.MediaTypeImageIndex, + Digest: badIndexDigest.String(), + }, + }, + }) + So(err, ShouldBeNil) + + err = setImageMeta(badIndexDigest, badProtoBlob, dynamoWrapper) //nolint: contextcheck + So(err, ShouldBeNil) + + _, err = dynamoWrapper.FilterTags(ctx, mTypes.AcceptAllRepoTag, mTypes.AcceptAllImageMeta) + So(err, ShouldNotBeNil) + }) + Convey("good image index, bad inside manifest", func() { + goodIndexBadManifestDigest := godigest.FromString("good-index-bad-manifests") + err := dynamoWrapper.SetRepoMeta("repo", mTypes.RepoMeta{ //nolint: contextcheck + Name: "repo", + Tags: map[string]mTypes.Descriptor{ + "good-index-bad-manifests": { + MediaType: ispec.MediaTypeImageIndex, + Digest: goodIndexBadManifestDigest.String(), + }, + }, + }) + So(err, ShouldBeNil) + + err = dynamoWrapper.SetImageMeta(goodIndexBadManifestDigest, multiarchImageMeta) //nolint: contextcheck + So(err, ShouldBeNil) + + err = setImageMeta(image.Digest(), badProtoBlob, dynamoWrapper) //nolint: contextcheck + So(err, ShouldBeNil) + + _, err = dynamoWrapper.FilterTags(ctx, mTypes.AcceptAllRepoTag, mTypes.AcceptAllImageMeta) + So(err, ShouldNotBeNil) + }) + }) + }) + + Convey("SearchTags", func() { + Convey("getProtoRepoMeta errors", func() { + dynamoWrapper.RepoMetaTablename = badTablename + + _, err := dynamoWrapper.SearchTags(ctx, "repo") + So(err, ShouldNotBeNil) + }) + Convey("found repo meta", func() { + Convey("bad image manifest", func() { + badImageDigest := godigest.FromString("bad-image-manifest") + err := dynamoWrapper.SetRepoMeta("repo", mTypes.RepoMeta{ //nolint: contextcheck + Name: "repo", + Tags: map[string]mTypes.Descriptor{ + "bad-image-manifest": { + MediaType: ispec.MediaTypeImageManifest, + Digest: badImageDigest.String(), + }, + }, + }) + So(err, ShouldBeNil) + + err = setImageMeta(badImageDigest, badProtoBlob, dynamoWrapper) //nolint: contextcheck + So(err, ShouldBeNil) + + _, err = dynamoWrapper.SearchTags(ctx, "repo:") + So(err, ShouldNotBeNil) + }) + Convey("bad image index", func() { + badIndexDigest := godigest.FromString("bad-image-manifest") + err := dynamoWrapper.SetRepoMeta("repo", mTypes.RepoMeta{ //nolint: contextcheck + Name: "repo", + Tags: map[string]mTypes.Descriptor{ + "bad-image-index": { + MediaType: ispec.MediaTypeImageIndex, + Digest: badIndexDigest.String(), + }, + }, + }) + So(err, ShouldBeNil) + + err = setImageMeta(badIndexDigest, badProtoBlob, dynamoWrapper) //nolint: contextcheck + So(err, ShouldBeNil) + + _, err = dynamoWrapper.SearchTags(ctx, "repo:") + So(err, ShouldNotBeNil) + }) + Convey("good image index, bad inside manifest", func() { + goodIndexBadManifestDigest := godigest.FromString("good-index-bad-manifests") + err := dynamoWrapper.SetRepoMeta("repo", mTypes.RepoMeta{ //nolint: contextcheck + Name: "repo", + Tags: map[string]mTypes.Descriptor{ + "good-index-bad-manifests": { + MediaType: ispec.MediaTypeImageIndex, + Digest: goodIndexBadManifestDigest.String(), + }, + }, + }) + So(err, ShouldBeNil) + + err = dynamoWrapper.SetImageMeta(goodIndexBadManifestDigest, multiarchImageMeta) //nolint: contextcheck + So(err, ShouldBeNil) + + err = setImageMeta(image.Digest(), badProtoBlob, dynamoWrapper) //nolint: contextcheck + So(err, ShouldBeNil) + + _, err = dynamoWrapper.SearchTags(ctx, "repo:") + So(err, ShouldNotBeNil) + }) + Convey("bad media type", func() { + err := dynamoWrapper.SetRepoMeta("repo", mTypes.RepoMeta{ //nolint: contextcheck + Name: "repo", + Tags: map[string]mTypes.Descriptor{ + "mad-media-type": { + MediaType: "bad media type", + Digest: godigest.FromString("dig").String(), + }, + }, + }) + So(err, ShouldBeNil) + + _, err = dynamoWrapper.SearchTags(ctx, "repo:") + So(err, ShouldBeNil) + }) + }) + }) + + Convey("SearchRepos", func() { + Convey("repoMetaAttributeIterator.First errors", func() { + dynamoWrapper.RepoMetaTablename = badTablename + + _, err := dynamoWrapper.SearchRepos(ctx, "repo") + So(err, ShouldNotBeNil) + }) + + Convey("repo meta unmarshal errors", func() { + err := setRepoMeta("repo", badProtoBlob, dynamoWrapper) //nolint: contextcheck + So(err, ShouldBeNil) + + _, err = dynamoWrapper.SearchRepos(ctx, "repo") + So(err, ShouldNotBeNil) + }) + }) + + Convey("SetRepoReference", func() { + Convey("SetProtoImageMeta fails", func() { + dynamoWrapper.ImageMetaTablename = badTablename + + err := dynamoWrapper.SetRepoReference(ctx, "repo", "tag", image.AsImageMeta()) + So(err, ShouldNotBeNil) + }) + Convey("getProtoRepoMeta fails", func() { + dynamoWrapper.RepoMetaTablename = badTablename + + err := dynamoWrapper.SetRepoReference(ctx, "repo", "tag", image.AsImageMeta()) + So(err, ShouldNotBeNil) + }) + Convey("getProtoRepoBlobs fails", func() { + dynamoWrapper.RepoBlobsTablename = badTablename + + err := dynamoWrapper.SetRepoReference(ctx, "repo", "tag", image.AsImageMeta()) + So(err, ShouldNotBeNil) + }) + }) + + Convey("GetProtoImageMeta", func() { + Convey("Get request fails", func() { + dynamoWrapper.ImageMetaTablename = badTablename + + _, err := dynamoWrapper.GetProtoImageMeta(ctx, testDigest) + So(err, ShouldNotBeNil) + }) + Convey("unmarshal fails", func() { + err := setRepoMeta("repo", badProtoBlob, dynamoWrapper) //nolint: contextcheck + So(err, ShouldBeNil) + + _, err = dynamoWrapper.GetProtoImageMeta(ctx, testDigest) + So(err, ShouldNotBeNil) + }) + }) + Convey("SetUserData", func() { hashKey := "id" apiKeys := make(map[string]mTypes.APIKeyDetails) @@ -527,6 +994,38 @@ func TestWrapperErrors(t *testing.T) { _, err = mdynamodb.New(client, params, log) So(err, ShouldNotBeNil) + params = mdynamodb.DBDriverParameters{ //nolint:contextcheck + Endpoint: endpoint, + Region: region, + RepoMetaTablename: repoMetaTablename, + ImageMetaTablename: "", + RepoBlobsInfoTablename: repoBlobsTablename, + UserDataTablename: userDataTablename, + APIKeyTablename: apiKeyTablename, + VersionTablename: versionTablename, + } + client, err = mdynamodb.GetDynamoClient(params) + So(err, ShouldBeNil) + + _, err = mdynamodb.New(client, params, log) + So(err, ShouldNotBeNil) + + params = mdynamodb.DBDriverParameters{ //nolint:contextcheck + Endpoint: endpoint, + Region: region, + RepoMetaTablename: repoMetaTablename, + ImageMetaTablename: imageMetaTablename, + RepoBlobsInfoTablename: "", + UserDataTablename: userDataTablename, + APIKeyTablename: apiKeyTablename, + VersionTablename: versionTablename, + } + client, err = mdynamodb.GetDynamoClient(params) + So(err, ShouldBeNil) + + _, err = mdynamodb.New(client, params, log) + So(err, ShouldNotBeNil) + params = mdynamodb.DBDriverParameters{ //nolint:contextcheck Endpoint: endpoint, Region: region, @@ -577,6 +1076,81 @@ func TestWrapperErrors(t *testing.T) { }) } +func setRepoMeta(repo string, blob []byte, dynamoWrapper *mdynamodb.DynamoDB) error { //nolint: unparam + userAttributeValue, err := attributevalue.Marshal(blob) + if err != nil { + return err + } + + _, err = dynamoWrapper.Client.UpdateItem(context.Background(), &dynamodb.UpdateItemInput{ + ExpressionAttributeNames: map[string]string{ + "#RM": "RepoMeta", + }, + ExpressionAttributeValues: map[string]types.AttributeValue{ + ":RepoMeta": userAttributeValue, + }, + Key: map[string]types.AttributeValue{ + "Key": &types.AttributeValueMemberS{ + Value: repo, + }, + }, + TableName: aws.String(dynamoWrapper.RepoMetaTablename), + UpdateExpression: aws.String("SET #RM = :RepoMeta"), + }) + + return err +} + +func setRepoBlobInfo(repo string, blob []byte, dynamoWrapper *mdynamodb.DynamoDB) error { + userAttributeValue, err := attributevalue.Marshal(blob) + if err != nil { + return err + } + + _, err = dynamoWrapper.Client.UpdateItem(context.Background(), &dynamodb.UpdateItemInput{ + ExpressionAttributeNames: map[string]string{ + "#RB": "RepoBlobsInfo", + }, + ExpressionAttributeValues: map[string]types.AttributeValue{ + ":RepoBlobsInfo": userAttributeValue, + }, + Key: map[string]types.AttributeValue{ + "Key": &types.AttributeValueMemberS{ + Value: repo, + }, + }, + TableName: aws.String(dynamoWrapper.RepoBlobsTablename), + UpdateExpression: aws.String("SET #RB = :RepoBlobsInfo"), + }) + + return err +} + +func setImageMeta(digest godigest.Digest, blob []byte, dynamoWrapper *mdynamodb.DynamoDB) error { + userAttributeValue, err := attributevalue.Marshal(blob) + if err != nil { + return err + } + + _, err = dynamoWrapper.Client.UpdateItem(context.Background(), &dynamodb.UpdateItemInput{ + ExpressionAttributeNames: map[string]string{ + "#IM": "ImageMeta", + }, + ExpressionAttributeValues: map[string]types.AttributeValue{ + ":ImageMeta": userAttributeValue, + }, + Key: map[string]types.AttributeValue{ + "Key": &types.AttributeValueMemberS{ + Value: digest.String(), + }, + }, + TableName: aws.String(dynamoWrapper.ImageMetaTablename), + UpdateExpression: aws.String("SET #IM = :ImageMeta"), + }) + + return err +} + func setBadUserData(client *dynamodb.Client, userDataTablename, userID string) error { userAttributeValue, err := attributevalue.Marshal("string") if err != nil { diff --git a/pkg/meta/meta_test.go b/pkg/meta/meta_test.go index 14fb30fcac..4c34327431 100644 --- a/pkg/meta/meta_test.go +++ b/pkg/meta/meta_test.go @@ -43,7 +43,7 @@ const ( ARM = "arm64" ) -func getManifestDigest(md mTypes.ManifestData) string { return md.Digest.String() } +func getManifestDigest(md mTypes.ManifestMeta) string { return md.Digest.String() } func TestBoltDB(t *testing.T) { Convey("BoltDB creation", t, func() { @@ -509,6 +509,20 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func // set subject on multiarch }) + Convey("GetFullImageMeta", func() { + img1 := CreateRandomImage() + multi := CreateMultiarchWith().Images([]Image{img1}).Build() + + err := metaDB.SetRepoReference(ctx, "repo", img1.Digest().String(), img1.AsImageMeta()) + So(err, ShouldBeNil) + err = metaDB.SetRepoReference(ctx, "repo", "tag", multi.AsImageMeta()) + So(err, ShouldBeNil) + + fullImageMeta, err := metaDB.GetFullImageMeta(ctx, "repo", "tag") + So(err, ShouldBeNil) + So(fullImageMeta.Digest.String(), ShouldResemble, multi.DigestStr()) + }) + Convey("Set/Get RepoMeta", func() { err := metaDB.SetRepoMeta("repo", mTypes.RepoMeta{ Name: "repo", @@ -663,6 +677,14 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func So(repoMeta.Platforms, ShouldContain, ispec.Platform{OS: "os2", Architecture: "arch2"}) So(repoMeta.Size, ShouldEqual, repoSize) }) + + Convey("Set with a bad reference", func() { + err := metaDB.SetRepoReference(ctx, "repo", "", imgData1) + So(err, ShouldNotBeNil) + + err = metaDB.SetRepoReference(ctx, "", "tag", imgData1) + So(err, ShouldNotBeNil) + }) }) Convey("Test RemoveRepoReference", func() { @@ -816,7 +838,10 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func imageMeta = CreateDefaultImage().AsImageMeta() ) - err := metaDB.SetRepoReference(ctx, repo1, tag1, imageMeta) + err := metaDB.IncrementRepoStars("missing-repo") + So(err, ShouldNotBeNil) + + err = metaDB.SetRepoReference(ctx, repo1, tag1, imageMeta) So(err, ShouldBeNil) err = metaDB.IncrementRepoStars(repo1) @@ -848,7 +873,10 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func imageMeta = CreateDefaultImage().AsImageMeta() ) - err := metaDB.SetRepoReference(ctx, repo1, tag1, imageMeta) + err := metaDB.IncrementRepoStars("missing-repo") + So(err, ShouldNotBeNil) + + err = metaDB.SetRepoReference(ctx, repo1, tag1, imageMeta) So(err, ShouldBeNil) err = metaDB.IncrementRepoStars(repo1) diff --git a/pkg/meta/parse_test.go b/pkg/meta/parse_test.go index 195949556a..c8207d6814 100644 --- a/pkg/meta/parse_test.go +++ b/pkg/meta/parse_test.go @@ -34,6 +34,8 @@ import ( const repo = "repo" func TestParseStorageErrors(t *testing.T) { + ctx := context.Background() + Convey("ParseStorage", t, func() { imageStore := mocks.MockedImageStore{ GetIndexContentFn: func(repo string) ([]byte, error) { @@ -97,6 +99,54 @@ func TestParseStorageErrors(t *testing.T) { So(err, ShouldNotBeNil) }) + Convey("resetRepoReferences errors", func() { + imageStore.GetIndexContentFn = func(repo string) ([]byte, error) { + return []byte("{}"), nil + } + metaDB.ResetRepoReferencesFn = func(repo string) error { return ErrTestError } + err := meta.ParseRepo("repo", metaDB, storeController, log) + So(err, ShouldNotBeNil) + }) + + Convey("zcommon.IsReferrersTag", func() { + imageStore.GetIndexContentFn = func(repo string) ([]byte, error) { + return getIndexBlob(ispec.Index{ + Manifests: []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageManifest, + Digest: godigest.FromString("digest"), + Annotations: map[string]string{ + ispec.AnnotationRefName: "sha256-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + }, + }, + }, + }), nil + } + err := meta.ParseRepo("repo", metaDB, storeController, log) + So(err, ShouldBeNil) + }) + + Convey("imageStore.GetImageManifest errors", func() { + imageStore.GetIndexContentFn = func(repo string) ([]byte, error) { + return getIndexBlob(ispec.Index{ + Manifests: []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageManifest, + Digest: godigest.FromString("digest"), + Annotations: map[string]string{ + ispec.AnnotationRefName: "tag", + }, + }, + }, + }), nil + } + imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) { + return nil, "", "", ErrTestError + } + err := meta.ParseRepo("repo", metaDB, storeController, log) + So(err, ShouldNotBeNil) + }) + Convey("manifestMetaIsPresent true", func() { indexContent := ispec.Index{ Manifests: []ispec.Descriptor{ @@ -126,6 +176,75 @@ func TestParseStorageErrors(t *testing.T) { }) }) }) + + image := CreateRandomImage() + + Convey("SetImageMetaFromInput errors", t, func() { + mockImageStore := mocks.MockedImageStore{} + mockedMetaDB := mocks.MetaDBMock{} + log := log.NewLogger("debug", "") + + Convey("Image Manifest errors", func() { + Convey("Get Config blob error", func() { + mockImageStore.GetBlobContentFn = func(repo string, digest godigest.Digest) ([]byte, error) { + return []byte{}, ErrTestError + } + + err := meta.SetImageMetaFromInput(ctx, "repo", "tag", ispec.MediaTypeImageManifest, image.Digest(), + image.ManifestDescriptor.Data, mockImageStore, mockedMetaDB, log) + So(err, ShouldNotBeNil) + }) + Convey("Unmarshal config blob error", func() { + mockImageStore.GetBlobContentFn = func(repo string, digest godigest.Digest) ([]byte, error) { + return []byte("bad-blob"), nil + } + + err := meta.SetImageMetaFromInput(ctx, "repo", "tag", ispec.MediaTypeImageManifest, image.Digest(), + image.ManifestDescriptor.Data, mockImageStore, mockedMetaDB, log) + So(err, ShouldNotBeNil) + }) + Convey("Is Signature", func() { + image := CreateDefaultImage() + mediaType := ispec.MediaTypeImageManifest + // it has more than 1 layer + badNotationSignature := CreateImageWith().RandomLayers(2, 10).EmptyConfig().Subject(image.DescriptorRef()). + ArtifactType(zcommon.ArtifactTypeNotation).Build() + goodNotationSignature := CreateMockNotationSignature(image.DescriptorRef()) + + Convey("GetSignatureLayersInfo errors", func() { + err := meta.SetImageMetaFromInput(ctx, "repo", "tag", mediaType, badNotationSignature.Digest(), + badNotationSignature.ManifestDescriptor.Data, mockImageStore, mockedMetaDB, log) + So(err, ShouldNotBeNil) + }) + Convey("UpdateSignaturesValidity errors", func() { + mockedMetaDB.UpdateSignaturesValidityFn = func(repo string, manifestDigest godigest.Digest) error { + return ErrTestError + } + err := meta.SetImageMetaFromInput(ctx, "repo", "tag", mediaType, goodNotationSignature.Digest(), + goodNotationSignature.ManifestDescriptor.Data, mockImageStore, mockedMetaDB, log) + So(err, ShouldNotBeNil) + }) + }) + }) + Convey("Image Index errors", func() { + Convey("Unmarshal error", func() { + err := meta.SetImageMetaFromInput(ctx, "repo", "tag", ispec.MediaTypeImageIndex, "", + []byte("bad-json"), mockImageStore, mockedMetaDB, log) + So(err, ShouldNotBeNil) + }) + }) + }) +} + +func getIndexBlob(index ispec.Index) []byte { + index.MediaType = ispec.MediaTypeImageIndex + + blob, err := json.Marshal(index) + if err != nil { + panic("image index should always be marshable") + } + + return blob } func TestParseStorageWithBoltDB(t *testing.T) { @@ -384,6 +503,42 @@ func TestGetSignatureLayersInfo(t *testing.T) { So(layers, ShouldBeEmpty) }) + Convey("GetBlobContent errors", t, func() { + mockImageStore := mocks.MockedImageStore{} + mockImageStore.GetBlobContentFn = func(repo string, digest godigest.Digest) ([]byte, error) { + return nil, ErrTestError + } + image := CreateRandomImage() + + layers, err := meta.GetSignatureLayersInfo("repo", "tag", "123", zcommon.CosignSignature, + image.ManifestDescriptor.Data, mockImageStore, log.NewLogger("debug", "")) + So(err, ShouldNotBeNil) + So(layers, ShouldBeEmpty) + }) + + Convey("notation len(manifestContent.Layers) != 1", t, func() { + mockImageStore := mocks.MockedImageStore{} + image := CreateImageWith().RandomLayers(3, 10).RandomConfig().Build() + + layers, err := meta.GetSignatureLayersInfo("repo", "tag", "123", zcommon.NotationSignature, + image.ManifestDescriptor.Data, mockImageStore, log.NewLogger("debug", "")) + So(err, ShouldNotBeNil) + So(layers, ShouldBeEmpty) + }) + + Convey("notation GetBlobContent errors", t, func() { + mockImageStore := mocks.MockedImageStore{} + mockImageStore.GetBlobContentFn = func(repo string, digest godigest.Digest) ([]byte, error) { + return nil, ErrTestError + } + image := CreateImageWith().RandomLayers(1, 10).RandomConfig().Build() + + layers, err := meta.GetSignatureLayersInfo("repo", "tag", "123", zcommon.NotationSignature, + image.ManifestDescriptor.Data, mockImageStore, log.NewLogger("debug", "")) + So(err, ShouldNotBeNil) + So(layers, ShouldBeEmpty) + }) + Convey("error while unmarshaling manifest content", t, func() { _, err := meta.GetSignatureLayersInfo("repo", "tag", "123", zcommon.CosignSignature, []byte("bad manifest"), nil, log.NewLogger("debug", "")) diff --git a/pkg/meta/types/types.go b/pkg/meta/types/types.go index c7be530734..0216b494c6 100644 --- a/pkg/meta/types/types.go +++ b/pkg/meta/types/types.go @@ -196,11 +196,11 @@ type ImageMeta struct { Digest godigest.Digest // Digest refers to the image descriptor, a manifest or a index (if multiarch) Size int64 // Size refers to the image descriptor, a manifest or a index (if multiarch) Index *ispec.Index // If the image is multiarch the Index will be non-nil - Manifests []ManifestData // All manifests under the image, 1 for simple images and many for multiarch + Manifests []ManifestMeta // All manifests under the image, 1 for simple images and many for multiarch } -// ManifestData represents all data related to an image manifests (found from the image contents itself). -type ManifestData struct { +// ManifestMeta represents all data related to an image manifests (found from the image contents itself). +type ManifestMeta struct { Size int64 Digest godigest.Digest Manifest ispec.Manifest @@ -246,7 +246,7 @@ type FullImageMeta struct { } type FullManifestMeta struct { - ManifestData + ManifestMeta Referrers []ReferrerInfo Statistics DescriptorStatistics diff --git a/pkg/test/image-utils/images.go b/pkg/test/image-utils/images.go index 3e46ba2d6a..8bafd0eb3a 100644 --- a/pkg/test/image-utils/images.go +++ b/pkg/test/image-utils/images.go @@ -11,6 +11,7 @@ import ( "github.com/opencontainers/image-spec/specs-go" ispec "github.com/opencontainers/image-spec/specs-go/v1" + "zotregistry.io/zot/pkg/common" mTypes "zotregistry.io/zot/pkg/meta/types" storageConstants "zotregistry.io/zot/pkg/storage/constants" ) @@ -145,7 +146,7 @@ func (img Image) AsImageMeta() mTypes.ImageMeta { MediaType: img.Manifest.MediaType, Digest: img.ManifestDescriptor.Digest, Size: img.ManifestDescriptor.Size, - Manifests: []mTypes.ManifestData{ + Manifests: []mTypes.ManifestMeta{ { Size: img.ManifestDescriptor.Size, Digest: img.ManifestDescriptor.Digest, @@ -203,6 +204,16 @@ func CreateRandomVulnerableImageWith() ManifestBuilder { return CreateImageWith().VulnerableLayers().RandomVulnConfig() } +func CreateMockNotationSignature(subject *ispec.Descriptor) Image { + return CreateImageWith().RandomLayers(1, 10).EmptyConfig().Subject(subject). + ArtifactType(common.ArtifactTypeNotation).Build() +} + +func CreateMockCosignSignature(subject *ispec.Descriptor) Image { + return CreateImageWith().RandomLayers(1, 10).EmptyConfig().Subject(subject). + ArtifactType(common.ArtifactTypeCosign).Build() +} + // CreateFakeTestSignature returns a test signature that is used to mark a image as signed // when creating a test Repo. It won't be recognized as a signature if uploaded to the repository directly. func CreateFakeTestSignature(subject *ispec.Descriptor) Image { diff --git a/pkg/test/image-utils/images_test.go b/pkg/test/image-utils/images_test.go index 769497a9ad..689f0d25f7 100644 --- a/pkg/test/image-utils/images_test.go +++ b/pkg/test/image-utils/images_test.go @@ -8,6 +8,7 @@ import ( ispec "github.com/opencontainers/image-spec/specs-go/v1" . "github.com/smartystreets/goconvey/convey" + "zotregistry.io/zot/pkg/common" . "zotregistry.io/zot/pkg/test/image-utils" ) @@ -17,6 +18,15 @@ func TestImageBuilder(t *testing.T) { t.FailNow() } + Convey("Signature images", t, func() { + image := CreateDefaultImage() + cosign := CreateMockCosignSignature(image.DescriptorRef()) + So(cosign.Manifest.ArtifactType, ShouldResemble, common.ArtifactTypeCosign) + + notation := CreateMockNotationSignature(image.DescriptorRef()) + So(notation.Manifest.ArtifactType, ShouldResemble, common.ArtifactTypeNotation) + }) + Convey("Test Layer Builders", t, func() { layerBuilder := CreateImageWith() diff --git a/pkg/test/image-utils/multiarch.go b/pkg/test/image-utils/multiarch.go index a5926ba7d7..ffc28f00e3 100644 --- a/pkg/test/image-utils/multiarch.go +++ b/pkg/test/image-utils/multiarch.go @@ -33,7 +33,7 @@ func (mi *MultiarchImage) DigestStr() string { func (mi MultiarchImage) AsImageMeta() mTypes.ImageMeta { index := mi.Index - manifests := make([]mTypes.ManifestData, 0, len(index.Manifests)) + manifests := make([]mTypes.ManifestMeta, 0, len(index.Manifests)) for _, image := range mi.Images { manifests = append(manifests, image.AsImageMeta().Manifests...)