Skip to content

Commit

Permalink
w
Browse files Browse the repository at this point in the history
  • Loading branch information
@laurentiuNiculae
laurentiuNiculae committed Sep 25, 2023
1 parent 81bb57b commit 0834860
Show file tree
Hide file tree
Showing 6 changed files with 326 additions and 12 deletions.
35 changes: 35 additions & 0 deletions pkg/extensions/search/convert/metadb.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1328,3 +1328,38 @@ func GetProtoSignaturesInfo(isSigned bool, repoMeta *proto_go.ProtoRepoMeta, dig

return signaturesInfo
}

func PaginatedProtoRepoMeta2ImageSummaries(ctx context.Context, reposMeta []mTypes.RepoMetadata2,
imageData map[string]mTypes.ImageData2, skip SkipQGLField, cveInfo cveinfo.CveInfo, filter mTypes.Filter,
pageInput pagination.PageInput,
) ([]*gql_generated.ImageSummary, zcommon.PageInfo, error) {
imagePageFinder, err := pagination.NewImgSumPageFinder(pageInput.Limit, pageInput.Offset, pageInput.SortBy)
if err != nil {
return []*gql_generated.ImageSummary{}, zcommon.PageInfo{}, err
}

for _, repoMeta := range reposMeta {
for tag := range repoMeta.Tags {
descriptor := repoMeta.Tags[tag]

imageSummary, _, err := ProtoDescriptor2ImageSummary(ctx, descriptor, repoMeta.Name, tag,
repoMeta, imageData)
if err != nil {
continue
}

if ImgSumAcceptedByFilter(imageSummary, filter) {
imagePageFinder.Add(imageSummary)
}
}
}

page, pageInfo := imagePageFinder.Page()

for _, imageSummary := range page {
// CVE scanning is expensive, only scan for this page
updateImageSummaryVulnerabilities(ctx, imageSummary, skip, cveInfo)
}

return page, pageInfo, nil
}
17 changes: 14 additions & 3 deletions pkg/extensions/search/resolver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -763,17 +763,28 @@ func globalSearch(ctx context.Context, query string, metaDB mTypes.MetaDB, filte
),
}

reposMeta, manifestMetaMap, indexDataMap, err := metaDB.SearchTags(ctx, query)
reposMeta, imageData, err := metaDB.ProtoSearchTags(ctx, query)
if err != nil {
return &gql_generated.PaginatedReposResult{}, []*gql_generated.ImageSummary{}, []*gql_generated.LayerSummary{}, err
}

imageSummaries, pageInfo, err := convert.PaginatedRepoMeta2ImageSummaries(ctx, reposMeta, manifestMetaMap,
indexDataMap, skip, cveInfo, localFilter, pageInput)
imageSummaries, pageInfo, err := convert.PaginatedProtoRepoMeta2ImageSummaries(ctx, reposMeta, imageData, skip, cveInfo,
localFilter, pageInput)
if err != nil {
return &gql_generated.PaginatedReposResult{}, []*gql_generated.ImageSummary{}, []*gql_generated.LayerSummary{}, err
}

// reposMeta, manifestMetaMap, indexDataMap, err := metaDB.SearchTags(ctx, query)
// if err != nil {
// return &gql_generated.PaginatedReposResult{}, []*gql_generated.ImageSummary{}, []*gql_generated.LayerSummary{}, err
// }

// imageSummaries, pageInfo, err := convert.PaginatedRepoMeta2ImageSummaries(ctx, reposMeta, manifestMetaMap,
// indexDataMap, skip, cveInfo, localFilter, pageInput)
// if err != nil {
// return &gql_generated.PaginatedReposResult{}, []*gql_generated.ImageSummary{}, []*gql_generated.LayerSummary{}, err
// }

images = imageSummaries

paginatedRepos.Page = &gql_generated.PageInfo{
Expand Down
255 changes: 248 additions & 7 deletions pkg/meta/boltdb/boltdb.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2261,13 +2261,7 @@ func (bdw *BoltDB) ProtoSearchRepos(ctx context.Context, searchText string,
repos2 := []mTypes.RepoMetadata2{}

for _, repo := range repos {
repos2 = append(repos2, mTypes.RepoMetadata2{
Name: repo.Name,
Tags: getTags(repo.Tags),
Statistics: getStatistics(repo.Statistics),
Signatures: getSignatures(repo.Signatures),
Referrers: getReferrers(repo.Referrers),
})
repos2 = append(repos2, getRepoMeta(repo))
}

images2 := map[string]mTypes.ImageData2{}
Expand All @@ -2278,6 +2272,16 @@ func (bdw *BoltDB) ProtoSearchRepos(ctx context.Context, searchText string,
return repos2, images2, err
}

func getRepoMeta(repo *proto_go.ProtoRepoMeta) mTypes.RepoMetadata2 {
return mTypes.RepoMetadata2{
Name: repo.Name,
Tags: getTags(repo.Tags),
Statistics: getStatistics(repo.Statistics),
Signatures: getSignatures(repo.Signatures),
Referrers: getReferrers(repo.Referrers),
}
}

func getImageData(dbImageData *proto_go.ImageData) mTypes.ImageData2 {
imageData := mTypes.ImageData2{
MediaType: dbImageData.MediaType,
Expand Down Expand Up @@ -2523,3 +2527,240 @@ func fetchProtoImageData(imageBuck *bbolt.Bucket, digest string) (*proto_go.Imag

return &imageData, nil
}

func (bdw *BoltDB) ProtoSearchTags(ctx context.Context, searchText string,
) ([]mTypes.RepoMetadata2, map[string]mTypes.ImageData2, error) {
repos := []*proto_go.ProtoRepoMeta{}
images := map[string]*proto_go.ImageData{}

searchedRepo, searchedTag, err := common.GetRepoTag(searchText)
if err != nil {
return []mTypes.RepoMetadata2{}, map[string]mTypes.ImageData2{},
fmt.Errorf("metadb: error while parsing search text, invalid format %w", err)
}

err = bdw.DB.View(func(transaction *bbolt.Tx) error {
var (
repoBuck = transaction.Bucket([]byte(ProtoRepoMetaBuck))
imageBuck = transaction.Bucket([]byte(ProtoImageDataBuck))
userBookmarks = getUserBookmarks(ctx, transaction)
userStars = getUserStars(ctx, transaction)
)

repoName, repoMetaBlob := repoBuck.Cursor().Seek([]byte(searchedRepo))

if string(repoName) != searchedRepo {
return nil
}

if ok, err := reqCtx.RepoIsUserAvailable(ctx, string(repoName)); !ok || err != nil {
return err
}

repoMeta := proto_go.ProtoRepoMeta{}

err := proto.Unmarshal(repoMetaBlob, &repoMeta)
if err != nil {
return err
}

repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name)
repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name)

matchedTags := make(map[string]*proto_go.Descriptor)

for tag, descriptor := range repoMeta.Tags {
if !strings.HasPrefix(tag, searchedTag) || tag == "" {
continue
}

matchedTags[tag] = descriptor

switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
manifestDigest := descriptor.Digest

imageManifestData, err := fetchProtoImageData(imageBuck, manifestDigest)
if err != nil {
return fmt.Errorf("metadb: error fetching manifest meta for manifest with digest %s %w",
manifestDigest, err)
}

imageManifestData.Repo = ref(string(repoName))
imageManifestData.Tag = &tag

images[manifestDigest] = imageManifestData
case ispec.MediaTypeImageIndex:
indexDigest := descriptor.Digest

imageIndexData, err := fetchProtoImageData(imageBuck, indexDigest)
if err != nil {
return fmt.Errorf("metadb: error fetching manifest meta for manifest with digest %s %w",
indexDigest, err)
}

for i, manifest := range imageIndexData.Manifests {
manifestDigest := manifest.Digest

imageManifestData, err := fetchProtoImageData(imageBuck, manifestDigest)
if err != nil {
return err
}

imageIndexData.Manifests[i] = imageManifestData.Manifests[0]
}

imageIndexData.Repo = ref(string(repoName))
imageIndexData.Tag = &tag

images[indexDigest] = imageIndexData
default:
bdw.Log.Error().Str("mediaType", descriptor.MediaType).Msg("Unsupported media type")

continue
}
}

if len(matchedTags) == 0 {
return nil
}

repoMeta.Tags = matchedTags

repos = append(repos, &repoMeta)

return nil
})

resultRepos := []mTypes.RepoMetadata2{}
resultImages := map[string]mTypes.ImageData2{}

for _, repo := range repos {
resultRepos = append(resultRepos, mTypes.RepoMetadata2{
Name: repo.Name,
Tags: getTags(repo.Tags),
Statistics: getStatistics(repo.Statistics),
Signatures: getSignatures(repo.Signatures),
Referrers: getReferrers(repo.Referrers),
})
}

for digest, imageData := range images {
resultImages[digest] = getImageData(imageData)
}

return resultRepos, resultImages, err
}

func (bdw *BoltDB) ProtoFilterTag(ctx context.Context, filterFunc mTypes.FilterProtoFunc,
) ([]mTypes.RepoMetadata2, map[string]mTypes.ImageData2, error) {
repos := []mTypes.RepoMetadata2{}
images := map[string]mTypes.ImageData2{}

err := bdw.DB.View(func(transaction *bbolt.Tx) error {
var (
repoBuck = transaction.Bucket([]byte(ProtoRepoMetaBuck))
imageDataBuck = transaction.Bucket([]byte(ProtoImageDataBuck))
userBookmarks = getUserBookmarks(ctx, transaction)
userStars = getUserStars(ctx, transaction)
)

cursor := repoBuck.Cursor()
repoName, repoMetaBlob := cursor.First()

for ; repoName != nil; repoName, repoMetaBlob = cursor.Next() {
if ok, err := reqCtx.RepoIsUserAvailable(ctx, string(repoName)); !ok || err != nil {
continue
}

repoMeta := proto_go.ProtoRepoMeta{}

err := json.Unmarshal(repoMetaBlob, &repoMeta)
if err != nil {
return err
}

repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name)
repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name)

repoMeta2 := getRepoMeta(&repoMeta)
matchedTags := make(map[string]mTypes.Descriptor)
// take all manifestsMeta
for tag, descriptor := range repoMeta.Tags {
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
manifestDigest := descriptor.Digest

imageManifestData, err := fetchProtoImageData(imageDataBuck, manifestDigest)
if err != nil {
return fmt.Errorf("metadb: error while unmashaling manifest metadata for digest %s %w", manifestDigest, err)
}

imageData := getImageData(imageManifestData)

if filterFunc(repoMeta2, imageData) {
matchedTags[tag] = mTypes.Descriptor{
Digest: descriptor.Digest,
MediaType: descriptor.MediaType,
}
images[manifestDigest] = imageData
}
case ispec.MediaTypeImageIndex:
indexDigest := descriptor.Digest

imageIndexData, err := fetchProtoImageData(imageDataBuck, indexDigest)
if err != nil {
return fmt.Errorf("metadb: error while getting index data for digest %s %w", indexDigest, err)
}

matchedManifests := []*proto_go.ManifestData{}

for _, manifest := range imageIndexData.Manifests {
manifestDigest := manifest.Digest

imageManifestData, err := fetchProtoImageData(imageDataBuck, indexDigest)
if err != nil {
return fmt.Errorf("metadb: error while getting manifest data for digest %s %w", manifestDigest, err)
}

imageData := getImageData(imageManifestData)

if filterFunc(repoMeta2, imageData) {
matchedManifests = append(matchedManifests, manifest)
}
}

if len(matchedManifests) > 0 {
imageIndexData.Manifests = matchedManifests

images[indexDigest] = getImageData(imageIndexData)
matchedTags[tag] = mTypes.Descriptor{
Digest: descriptor.Digest,
MediaType: descriptor.MediaType,
}
}
default:
bdw.Log.Error().Str("mediaType", descriptor.MediaType).Msg("Unsupported media type")

continue
}
}

if len(matchedTags) == 0 {
continue
}

repoMeta2.Tags = matchedTags

repos = append(repos, repoMeta2)
}

return nil
})

return repos, images, err
}

func filterProtoFunc(repoMeta mTypes.RepoMetadata, imageManifestData *proto_go.ImageData) {
panic("unimplemented")
}
11 changes: 11 additions & 0 deletions pkg/meta/dynamodb/dynamodb.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2131,3 +2131,14 @@ func (dwr *DynamoDB) SetProtoRepoReference(repo string, reference string, manife
func (dwr *DynamoDB) ProtoSearchRepos(ctx context.Context, searchText string) ([]mTypes.RepoMetadata2, map[string]mTypes.ImageData2, error) {
panic("not implemented")
}

// TODO
func (dwr *DynamoDB) ProtoSearchTags(ctx context.Context, searchText string) ([]mTypes.RepoMetadata2, map[string]mTypes.ImageData2, error) {
panic("not implemented")
}

// TODO
func (bdw *DynamoDB) ProtoFilterTag(ctx context.Context, filterFunc mTypes.FilterProtoFunc,
) ([]mTypes.RepoMetadata2, map[string]mTypes.ImageData2, error) {
panic("not implemented")
}
10 changes: 8 additions & 2 deletions pkg/meta/types/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,9 @@ const (
)

type (
FilterFunc func(repoMeta RepoMetadata, manifestMeta ManifestMetadata) bool
FilterRepoFunc func(repoMeta RepoMetadata) bool
FilterFunc func(repoMeta RepoMetadata, manifestMeta ManifestMetadata) bool
FilterProtoFunc func(repoMeta RepoMetadata2, imageData ImageData2) bool
FilterRepoFunc func(repoMeta RepoMetadata) bool
)

type (
Expand Down Expand Up @@ -139,6 +140,11 @@ type MetaDB interface { //nolint:interfacebloat
SetProtoRepoReference(repo string, reference string, manifestDigest godigest.Digest, mediaType string) error

ProtoSearchRepos(ctx context.Context, searchText string) ([]RepoMetadata2, map[ImageDigest]ImageData2, error)

ProtoSearchTags(ctx context.Context, searchText string) (
[]RepoMetadata2, map[string]ImageData2, error)

ProtoFilterTag(ctx context.Context, filterFunc FilterProtoFunc) ([]RepoMetadata2, map[string]ImageData2, error)
}

type UserDB interface { //nolint:interfacebloat
Expand Down
10 changes: 10 additions & 0 deletions pkg/test/mocks/repo_db_mock.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -515,3 +515,13 @@ func (bdw MetaDBMock) SetProtoRepoReference(repo string, reference string, manif
func (bdw MetaDBMock) ProtoSearchRepos(ctx context.Context, searchText string) ([]mTypes.RepoMetadata2, map[string]mTypes.ImageData2, error) {
panic("not implemented")
}

// TODO
func (bdw MetaDBMock) ProtoSearchTags(ctx context.Context, searchText string) ([]mTypes.RepoMetadata2, map[string]mTypes.ImageData2, error) {
panic("not implemented")
}

func (bdw MetaDBMock) ProtoFilterTag(ctx context.Context, filterFunc mTypes.FilterProtoFunc,
) ([]mTypes.RepoMetadata2, map[string]mTypes.ImageData2, error) {
panic("not implemented")
}

0 comments on commit 0834860

Please sign in to comment.