From 81bb57be6216c6c70b3e5d1ed1cbc3c498c25a98 Mon Sep 17 00:00:00 2001 From: Laurentiu Niculae Date: Mon, 25 Sep 2023 15:41:38 +0300 Subject: [PATCH] w --- pkg/extensions/search/convert/metadb.go | 89 ++++---- pkg/meta/boltdb/boltdb.go | 258 +++++++++++++++++++++++- pkg/meta/dynamodb/dynamodb.go | 2 +- pkg/meta/types/types.go | 33 ++- pkg/test/mocks/repo_db_mock.go | 2 +- 5 files changed, 331 insertions(+), 53 deletions(-) diff --git a/pkg/extensions/search/convert/metadb.go b/pkg/extensions/search/convert/metadb.go index ce62f8e4a1..75a9a9b22d 100644 --- a/pkg/extensions/search/convert/metadb.go +++ b/pkg/extensions/search/convert/metadb.go @@ -867,8 +867,8 @@ func GetSignaturesInfo(isSigned bool, repoMeta mTypes.RepoMetadata, indexDigest return signaturesInfo } -func PaginatedProtoRepoMeta2RepoSummaries(ctx context.Context, repoMetaList []*proto_go.ProtoRepoMeta, - imageDataMap map[string]*proto_go.ImageData, filter mTypes.Filter, pageInput pagination.PageInput, cveInfo cveinfo.CveInfo, +func PaginatedProtoRepoMeta2RepoSummaries(ctx context.Context, repoMetaList []mTypes.RepoMetadata2, + imageDataMap map[string]mTypes.ImageData2, filter mTypes.Filter, pageInput pagination.PageInput, cveInfo cveinfo.CveInfo, skip SkipQGLField, ) ([]*gql_generated.RepoSummary, zcommon.PageInfo, error) { reposPageFinder, err := pagination.NewRepoSumPageFinder(pageInput.Limit, pageInput.Offset, pageInput.SortBy) @@ -894,7 +894,7 @@ func PaginatedProtoRepoMeta2RepoSummaries(ctx context.Context, repoMetaList []*p return page, pageInfo, nil } -func ProtoRepoMeta2RepoSummary(ctx context.Context, repoMeta *proto_go.ProtoRepoMeta, imageDataMap map[string]*proto_go.ImageData, +func ProtoRepoMeta2RepoSummary(ctx context.Context, repoMeta mTypes.RepoMetadata2, imageDataMap map[string]mTypes.ImageData2, ) *gql_generated.RepoSummary { var ( repoName = repoMeta.Name @@ -985,8 +985,8 @@ type ( BlobDigest = string ) -func ProtoDescriptor2ImageSummary(ctx context.Context, descriptor *proto_go.Descriptor, repo, tag string, - repoMeta *proto_go.ProtoRepoMeta, imageDataMap map[ManifestDigest]*proto_go.ImageData, +func ProtoDescriptor2ImageSummary(ctx context.Context, descriptor mTypes.Descriptor, repo, tag string, + repoMeta mTypes.RepoMetadata2, imageDataMap map[ManifestDigest]mTypes.ImageData2, ) (*gql_generated.ImageSummary, map[BlobDigest]int64, error) { switch descriptor.MediaType { case ispec.MediaTypeImageManifest: @@ -999,8 +999,8 @@ func ProtoDescriptor2ImageSummary(ctx context.Context, descriptor *proto_go.Desc } } -func ProtoImageIndex2ImageSummary(ctx context.Context, repo, tag, digest string, repoMeta *proto_go.ProtoRepoMeta, - imageDataMap map[string]*proto_go.ImageData, +func ProtoImageIndex2ImageSummary(ctx context.Context, repo, tag, digest string, repoMeta mTypes.RepoMetadata2, + imageDataMap map[ManifestDigest]mTypes.ImageData2, ) (*gql_generated.ImageSummary, map[BlobDigest]int64, error) { imageIndex := imageDataMap[digest] @@ -1033,7 +1033,7 @@ func ProtoImageIndex2ImageSummary(ctx context.Context, repo, tag, digest string, indexLastUpdated = *imageManifestSummary.LastUpdated } - annotations := GetAnnotations(imageManifest.Annotations, imageManifest.Config.Config.Labels) + annotations := GetAnnotations(imageManifest.Annotations, imageManifest.ConfigContent.Config.Labels) if manifestAnnotations == nil { manifestAnnotations = &annotations @@ -1044,14 +1044,14 @@ func ProtoImageIndex2ImageSummary(ctx context.Context, repo, tag, digest string, manifestSummaries = append(manifestSummaries, imageManifestSummary.Manifests[0]) } - signaturesInfo := GetProtoSignaturesInfo(isSigned, repoMeta, digest) + signaturesInfo := GetSignaturesInfo(isSigned, mTypes.RepoMetadata(repoMeta), godigest.Digest(digest)) if manifestAnnotations == nil { // The index doesn't have manifests manifestAnnotations = &ImageAnnotations{} } - annotations := GetIndexAnnotations(imageIndex.Annotations, manifestAnnotations) + annotations := GetIndexAnnotations(imageIndex.Index.Annotations, manifestAnnotations) indexSummary := gql_generated.ImageSummary{ RepoName: &repo, @@ -1072,68 +1072,73 @@ func ProtoImageIndex2ImageSummary(ctx context.Context, repo, tag, digest string, Source: &annotations.Source, Vendor: &annotations.Vendor, Authors: &annotations.Authors, - Referrers: getProtoReferrers(repoMeta.Referrers[digest]), + Referrers: getReferrers(repoMeta.Referrers[digest]), } return &indexSummary, indexBlobs, nil } -func ProtoImageManifest2ImageSummary(ctx context.Context, repo, tag string, repoMeta *proto_go.ProtoRepoMeta, - imageData *proto_go.ImageData, +func ProtoImageManifest2ImageSummary(ctx context.Context, repo, tag string, repoMeta mTypes.RepoMetadata2, + imageData mTypes.ImageData2, ) (*gql_generated.ImageSummary, map[BlobDigest]int64, error) { manifest := imageData.Manifests[0] var ( repoName = repo - configDigest = manifest.Config.Digest - artifactType = manifest.ArtifactType - platform = getPlatform(manifest.Config.Platform) - imageLastUpdated = zcommon.GetProtoImageLastUpdated(manifest.Config) // TODO: we can cache this - downloadCount = int(repoMeta.Statistics[manifest.Digest].DownloadCount) - isSigned = isImageSigned(repoMeta.Signatures[manifest.Digest]) + configDigest = manifest.Config.Digest.String() + configSize = manifest.Config.Size + manifestDigest = manifest.Digest.String() + manifestSize = int64(manifest.Size) + mediaType = manifest.MediaType + artifactType = zcommon.GetManifestArtifactType(imageData.Manifests[0].Manifest) + platform = getPlatform(manifest.ConfigContent.Platform) + imageLastUpdated = zcommon.GetImageLastUpdated(manifest.ConfigContent) // TODO: we can cache this + downloadCount = int(repoMeta.Statistics[manifest.Digest.String()].DownloadCount) + isSigned = isImageSigned(repoMeta.Signatures[manifest.Digest.String()]) ) - imageSize, imageBlobsMap := getProtoImageBlobsInfo(manifest) - annotations := GetAnnotations(manifest.Annotations, manifest.Config.Config.Labels) // TODO: This can be cached + imageSize, imageBlobsMap := getImageBlobsInfo(manifestDigest, manifestSize, configDigest, configSize, manifest.Layers) + imageSizeStr := strconv.FormatInt(imageSize, 10) + annotations := GetAnnotations(manifest.Annotations, manifest.ConfigContent.Config.Labels) // TODO: This can be cached authors := annotations.Authors if authors == "" { - authors = manifest.Config.GetAuthor() + authors = manifest.ConfigContent.Author } - historyEntries, err := getAllProtoHistory(manifest) // TODO: this can be cached + historyEntries, err := getAllHistory(manifest.Manifest, manifest.ConfigContent) // TODO: this can be cached if err != nil { graphql.AddError(ctx, gqlerror.Errorf("error generating history on tag %s in repo %s: "+ "manifest digest: %s, error: %s", tag, repo, manifest.Digest, err.Error())) } - signaturesInfo := GetProtoSignaturesInfo(isSigned, repoMeta, manifest.Digest) + signaturesInfo := GetSignaturesInfo(isSigned, mTypes.RepoMetadata(repoMeta), manifest.Digest) manifestSummary := gql_generated.ManifestSummary{ - Digest: &manifest.Digest, + Digest: &manifestDigest, ConfigDigest: &configDigest, LastUpdated: &imageLastUpdated, - Size: &imageSize, + Size: &imageSizeStr, IsSigned: &isSigned, SignatureInfo: signaturesInfo, Platform: &platform, DownloadCount: &downloadCount, - Layers: getProtoLayersSummaries(manifest), + Layers: getLayersSummaries(manifest.Manifest), History: historyEntries, - Referrers: getProtoReferrers(repoMeta.Referrers[manifest.Digest]), - ArtifactType: artifactType, + Referrers: getReferrers(repoMeta.Referrers[manifestDigest]), + ArtifactType: &artifactType, } imageSummary := gql_generated.ImageSummary{ RepoName: &repoName, Tag: &tag, - Digest: &manifest.Digest, - MediaType: manifest.MediaType, + Digest: &manifestDigest, + MediaType: &mediaType, Manifests: []*gql_generated.ManifestSummary{&manifestSummary}, LastUpdated: &imageLastUpdated, IsSigned: &isSigned, SignatureInfo: signaturesInfo, - Size: &imageSize, + Size: &imageSizeStr, DownloadCount: &downloadCount, Description: &annotations.Description, Title: &annotations.Title, @@ -1252,9 +1257,9 @@ func getProtoImageBlobsInfo(manifest *proto_go.ManifestData) (string, map[string return strconv.FormatInt(imageSize, 10), imageBlobsMap } -func isImageSigned(manifestSignatures *proto_go.ManifestSignatures) bool { - for _, signatures := range manifestSignatures.Map { - if len(signatures.List) > 0 { +func isImageSigned(manifestSignatures mTypes.ManifestSignatures) bool { + for _, signatures := range manifestSignatures { + if len(signatures) > 0 { return true } } @@ -1262,20 +1267,16 @@ func isImageSigned(manifestSignatures *proto_go.ManifestSignatures) bool { return false } -func getPlatform(platform *proto_go.Platform) gql_generated.Platform { - if platform == nil { - return gql_generated.Platform{} - } - +func getPlatform(platform ispec.Platform) gql_generated.Platform { return gql_generated.Platform{ - Os: &platform.Os, + Os: ref(platform.OS), Arch: ref(getArch(platform.Architecture, platform.Variant)), } } -func getArch(arch string, variant *string) string { - if variant != nil && *variant != "" { - arch = arch + "/" + *variant +func getArch(arch string, variant string) string { + if variant != "" { + arch = arch + "/" + variant } return arch diff --git a/pkg/meta/boltdb/boltdb.go b/pkg/meta/boltdb/boltdb.go index 9e816703ee..98494530ac 100644 --- a/pkg/meta/boltdb/boltdb.go +++ b/pkg/meta/boltdb/boltdb.go @@ -9,6 +9,7 @@ import ( "time" godigest "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go" ispec "github.com/opencontainers/image-spec/specs-go/v1" "go.etcd.io/bbolt" "google.golang.org/protobuf/proto" @@ -2159,12 +2160,10 @@ func (bdw *BoltDB) SetProtoRepoReference(repo string, reference string, manifest return err } -type ManifestDigest = string - // TODO func (bdw *BoltDB) ProtoSearchRepos(ctx context.Context, searchText string, -) ([]*proto_go.ProtoRepoMeta, map[ManifestDigest]*proto_go.ImageData, error) { - images := map[ManifestDigest]*proto_go.ImageData{} +) ([]mTypes.RepoMetadata2, map[string]mTypes.ImageData2, error) { + images := map[string]*proto_go.ImageData{} repos := []*proto_go.ProtoRepoMeta{} // TODO: Q: We need to return info about RepoMeta like signatures, statistics and @@ -2259,7 +2258,256 @@ func (bdw *BoltDB) ProtoSearchRepos(ctx context.Context, searchText string, return nil }) - return repos, images, err + repos2 := []mTypes.RepoMetadata2{} + + for _, repo := range repos { + repos2 = append(repos2, mTypes.RepoMetadata2{ + Name: repo.Name, + Tags: getTags(repo.Tags), + Statistics: getStatistics(repo.Statistics), + Signatures: getSignatures(repo.Signatures), + Referrers: getReferrers(repo.Referrers), + }) + } + + images2 := map[string]mTypes.ImageData2{} + for digest, imageData := range images { + images2[digest] = getImageData(imageData) + } + + return repos2, images2, err +} + +func getImageData(dbImageData *proto_go.ImageData) mTypes.ImageData2 { + imageData := mTypes.ImageData2{ + MediaType: dbImageData.MediaType, + } + + if dbImageData.MediaType == ispec.MediaTypeImageIndex { + manifests := make([]ispec.Descriptor, 0, len(dbImageData.Manifests)) + var subject *ispec.Descriptor + + for _, manifest := range dbImageData.Manifests { + manifests = append(manifests, ispec.Descriptor{ + MediaType: deref(manifest.MediaType, ""), + Digest: godigest.Digest(manifest.Digest), + Size: manifest.Size, + }) + } + + if dbImageData.Subject != nil { + subject = &ispec.Descriptor{ + MediaType: dbImageData.Subject.MediaType, + Digest: godigest.Digest(dbImageData.Subject.Digest), + Size: dbImageData.Subject.Size, + } + } + + imageData.Index = &ispec.Index{ + Versioned: specs.Versioned{SchemaVersion: int(dbImageData.Versioned.GetSchemaversion())}, + MediaType: ispec.MediaTypeImageIndex, + ArtifactType: dbImageData.ArtifacType, + Manifests: manifests, + Subject: subject, + Annotations: dbImageData.Annotations, + } + } + + manifestDataList := make([]mTypes.ManifestData2, 0, len(dbImageData.Manifests)) + + for _, manifest := range dbImageData.Manifests { + manifestDataList = append(manifestDataList, mTypes.ManifestData2{ + Size: int(manifest.Size), + Digest: godigest.Digest(manifest.Digest), + Manifest: ispec.Manifest{ + Versioned: specs.Versioned{SchemaVersion: int(manifest.Versioned.GetSchemaversion())}, + MediaType: deref(manifest.MediaType, ""), + ArtifactType: deref(manifest.ArtifactType, ""), + Config: ispec.Descriptor{ + MediaType: ispec.MediaTypeImageConfig, // TODO add config media type to manifest data in DB + Size: manifest.Config.Size, + Digest: godigest.Digest(manifest.Config.Digest), + }, + Layers: getLayers(manifest.Layers), + Subject: getSubject(manifest.Subject), + Annotations: manifest.Annotations, + }, + ConfigContent: ispec.Image{ + Created: ref(manifest.Config.Created.AsTime()), + Author: deref(manifest.Config.Author, ""), + Platform: getPlatform(manifest.Config.Platform), + Config: ispec.ImageConfig{}, + RootFS: ispec.RootFS{}, + History: getHistory(manifest.Config.History), + }, + }) + } + + imageData.Manifests = manifestDataList + + return imageData +} + +func getHistory(history []*proto_go.History) []ispec.History { + results := make([]ispec.History, 0, len(history)) + + for _, his := range history { + results = append(results, ispec.History{ + Created: ref(his.Created.AsTime()), + CreatedBy: deref(his.Createdby, ""), + Author: deref(his.Author, ""), + Comment: deref(his.Comment, ""), + EmptyLayer: deref(his.Emptylayer, false), + }) + } + + return results +} + +func ref[T any](input T) *T { + ref := input + + return &ref +} + +func deref[T any](pointer *T, defaultVal T) T { + if pointer != nil { + return *pointer + } + + return defaultVal +} + +func getPlatform(platform *proto_go.Platform) ispec.Platform { + if platform == nil { + return ispec.Platform{} + } + + return ispec.Platform{ + Architecture: platform.Architecture, + OS: platform.Os, + OSVersion: deref(platform.Osversion, ""), + OSFeatures: platform.Osfeatures, + Variant: deref(platform.Variant, ""), + } +} + +func getLayers(descriptors []*proto_go.Descriptor) []ispec.Descriptor { + results := make([]ispec.Descriptor, 0, len(descriptors)) + + for _, desc := range descriptors { + results = append(results, ispec.Descriptor{ + MediaType: desc.MediaType, + Digest: godigest.Digest(desc.Digest), + Size: desc.Size, + }) + } + + return results +} + +func getSubject(subj *proto_go.Descriptor) *ispec.Descriptor { + if subj == nil { + return nil + } + + return &ispec.Descriptor{ + MediaType: subj.MediaType, + Digest: godigest.Digest(subj.Digest), + Size: subj.Size, + } +} + +func getStatistics(stats map[string]*proto_go.DescriptorStatistics) map[string]mTypes.DescriptorStatistics { + results := map[string]mTypes.DescriptorStatistics{} + + for digest, stat := range stats { + results[digest] = mTypes.DescriptorStatistics{ + DownloadCount: int(stat.DownloadCount), + } + } + + return results +} + +func getReferrers(refs map[string]*proto_go.ReferrersInfo) map[string][]mTypes.ReferrerInfo { + results := map[string][]mTypes.ReferrerInfo{} + + for digest, ref := range refs { + referrers := []mTypes.ReferrerInfo{} + + for _, dbRef := range ref.List { + referrers = append(referrers, mTypes.ReferrerInfo{ + Digest: dbRef.Digest, + MediaType: dbRef.MediaType, + ArtifactType: dbRef.ArtifactType, + Size: int(dbRef.Size), + Annotations: dbRef.Annotations, + }) + } + + results[digest] = referrers + } + + return results +} + +func getSignatures(sigs map[string]*proto_go.ManifestSignatures) map[string]mTypes.ManifestSignatures { + results := map[string]mTypes.ManifestSignatures{} + + for digest, dbSignatures := range sigs { + imageSignatures := mTypes.ManifestSignatures{} + + for signatureName, signatureInfo := range dbSignatures.Map { + imageSignatures[signatureName] = getSignaturesInfo(signatureInfo.List) + } + + results[digest] = imageSignatures + } + + return results +} + +func getSignaturesInfo(sigInfo []*proto_go.SignatureInfo) []mTypes.SignatureInfo { + results := []mTypes.SignatureInfo{} + + for _, info := range sigInfo { + results = append(results, mTypes.SignatureInfo{ + SignatureManifestDigest: info.SignatureManifestDigest, + LayersInfo: getLayersInfo(info.LayersInfo), + }) + } + + return results +} + +func getLayersInfo(layersInfo []*proto_go.LayersInfo) []mTypes.LayerInfo { + results := []mTypes.LayerInfo{} + + for _, layerInfo := range layersInfo { + results = append(results, mTypes.LayerInfo{ + LayerDigest: layerInfo.LayerDigest, + LayerContent: layerInfo.LayerContent, + SignatureKey: layerInfo.SignatureKey, + Signer: layerInfo.Signer, + Date: layerInfo.Date.AsTime(), + }) + } + + return results +} + +func getTags(tags map[string]*proto_go.Descriptor) map[string]mTypes.Descriptor { + resultMap := map[string]mTypes.Descriptor{} + + for tag, tagDescriptor := range tags { + resultMap[tag] = mTypes.Descriptor{ + Digest: tagDescriptor.Digest, + MediaType: tagDescriptor.MediaType, + } + } + + return resultMap } func fetchProtoImageData(imageBuck *bbolt.Bucket, digest string) (*proto_go.ImageData, error) { diff --git a/pkg/meta/dynamodb/dynamodb.go b/pkg/meta/dynamodb/dynamodb.go index 53b97a734f..7740f70393 100644 --- a/pkg/meta/dynamodb/dynamodb.go +++ b/pkg/meta/dynamodb/dynamodb.go @@ -2128,6 +2128,6 @@ func (dwr *DynamoDB) SetProtoRepoReference(repo string, reference string, manife } // TODO -func (dwr *DynamoDB) ProtoSearchRepos(ctx context.Context, searchText string) ([]*proto_go.ProtoRepoMeta, map[string]*proto_go.ImageData, error) { +func (dwr *DynamoDB) ProtoSearchRepos(ctx context.Context, searchText string) ([]mTypes.RepoMetadata2, map[string]mTypes.ImageData2, error) { panic("not implemented") } diff --git a/pkg/meta/types/types.go b/pkg/meta/types/types.go index 192d1ee37e..ffb2c656db 100644 --- a/pkg/meta/types/types.go +++ b/pkg/meta/types/types.go @@ -5,6 +5,7 @@ import ( "time" godigest "github.com/opencontainers/go-digest" + ispec "github.com/opencontainers/image-spec/specs-go/v1" "zotregistry.io/zot/pkg/meta/proto_go" ) @@ -33,7 +34,7 @@ type ( ) type ( - ManifestDigest = string + ImageDigest = string ) type MetaDB interface { //nolint:interfacebloat @@ -137,7 +138,7 @@ type MetaDB interface { //nolint:interfacebloat SetProtoRepoReference(repo string, reference string, manifestDigest godigest.Digest, mediaType string) error - ProtoSearchRepos(ctx context.Context, searchText string) ([]*proto_go.ProtoRepoMeta, map[ManifestDigest]*proto_go.ImageData, error) + ProtoSearchRepos(ctx context.Context, searchText string) ([]RepoMetadata2, map[ImageDigest]ImageData2, error) } type UserDB interface { //nolint:interfacebloat @@ -184,6 +185,34 @@ type ImageTrustStore interface { ) (string, time.Time, bool, error) } +type ImageData2 struct { + MediaType string + Index *ispec.Index + Manifests []ManifestData2 +} + +type ManifestData2 struct { + Size int + Digest godigest.Digest + ispec.Manifest + ConfigContent ispec.Image +} + +type RepoMetadata2 struct { + Name string + Tags map[string]Descriptor + + Statistics map[string]DescriptorStatistics + Signatures map[string]ManifestSignatures + Referrers map[string][]ReferrerInfo + + IsStarred bool + IsBookmarked bool + Rank int + + Stars int +} + type ManifestMetadata struct { ManifestBlob []byte ConfigBlob []byte diff --git a/pkg/test/mocks/repo_db_mock.go b/pkg/test/mocks/repo_db_mock.go index db4996a0ec..89982c7f85 100644 --- a/pkg/test/mocks/repo_db_mock.go +++ b/pkg/test/mocks/repo_db_mock.go @@ -512,6 +512,6 @@ func (bdw MetaDBMock) SetProtoRepoReference(repo string, reference string, manif } // TODO -func (bdw MetaDBMock) ProtoSearchRepos(ctx context.Context, searchText string) ([]*proto_go.ProtoRepoMeta, map[string]*proto_go.ImageData, error) { +func (bdw MetaDBMock) ProtoSearchRepos(ctx context.Context, searchText string) ([]mTypes.RepoMetadata2, map[string]mTypes.ImageData2, error) { panic("not implemented") }