Skip to content

Commit

Permalink
feat(clean): clean part of the old metadb functions
Browse files Browse the repository at this point in the history 
Signed-off-by: Laurentiu Niculae <[email protected]>
  • Loading branch information
@laurentiuNiculae
laurentiuNiculae committed Oct 13, 2023
1 parent 9f88846 commit 414ce16
Show file tree
Hide file tree
Showing 59 changed files with 4,745 additions and 11,144 deletions.
3 changes: 2 additions & 1 deletion examples/config-all-remote.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@
"region": "us-east-2",
"cacheTablename": "ZotBlobTable",
"repoMetaTablename": "ZotRepoMetadataTable",
"manifestDataTablename": "ZotManifestDataTable",
"imageDataTablename": "ZotImageDataTable",
"repoBlobsInfoTablename": "ZotRepoBlobsInfoTable",
"versionTablename": "ZotVersion"
}
},
Expand Down
3 changes: 2 additions & 1 deletion examples/config-dynamodb.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@
"region": "us-east-2",
"cacheTablename": "ZotBlobTable",
"repoMetaTablename": "ZotRepoMetadataTable",
"manifestDataTablename": "ZotManifestDataTable",
"imageDataTablename": "ZotImageDataTable",
"repoBlobsInfoTablename": "ZotRepoBlobsInfoTable",
"userDataTablename": "ZotUserDataTable",
"versionTablename": "ZotVersion"
}
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -500,7 +500,7 @@ require (
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb // indirect
google.golang.org/grpc v1.58.2 // indirect
google.golang.org/protobuf v1.31.0 // indirect
google.golang.org/protobuf v1.31.0
gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
Expand Down
8 changes: 4 additions & 4 deletions pkg/api/controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -277,10 +277,10 @@ func (c *Controller) InitMetaDB(reloadCtx context.Context) error {
return err
}

// err = meta.ParseStorage(driver, c.StoreController, c.Log)
// if err != nil {
// return err
// }
err = meta.ParseStorage(driver, c.StoreController, c.Log)
if err != nil {
return err
}

c.MetaDB = driver
}
Expand Down
123 changes: 65 additions & 58 deletions pkg/api/controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,14 +153,15 @@ func TestCreateCacheDatabaseDriver(t *testing.T) {

endpoint := os.Getenv("DYNAMODBMOCK_ENDPOINT")
conf.Storage.CacheDriver = map[string]interface{}{
"name": "dynamodb",
"endpoint": endpoint,
"region": "us-east-2",
"cacheTablename": "BlobTable",
"repoMetaTablename": "RepoMetadataTable",
"manifestDataTablename": "ManifestDataTable",
"userDataTablename": "ZotUserDataTable",
"versionTablename": "Version",
"name": "dynamodb",
"endpoint": endpoint,
"region": "us-east-2",
"cacheTablename": "BlobTable",
"repoMetaTablename": "RepoMetadataTable",
"imageDataTablename": "ZotImageDataTable",
"repoBlobsInfoTablename": "ZotRepoBlobsInfoTable",
"userDataTablename": "ZotUserDataTable",
"versionTablename": "Version",
}

driver := storage.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log)
Expand All @@ -169,27 +170,29 @@ func TestCreateCacheDatabaseDriver(t *testing.T) {
// negative test cases

conf.Storage.CacheDriver = map[string]interface{}{
"endpoint": endpoint,
"region": "us-east-2",
"cacheTablename": "BlobTable",
"repoMetaTablename": "RepoMetadataTable",
"manifestDataTablename": "ManifestDataTable",
"userDataTablename": "ZotUserDataTable",
"versionTablename": "Version",
"endpoint": endpoint,
"region": "us-east-2",
"cacheTablename": "BlobTable",
"repoMetaTablename": "RepoMetadataTable",
"imageDataTablename": "ZotImageDataTable",
"repoBlobsInfoTablename": "ZotRepoBlobsInfoTable",
"userDataTablename": "ZotUserDataTable",
"versionTablename": "Version",
}

driver = storage.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log)
So(driver, ShouldBeNil)

conf.Storage.CacheDriver = map[string]interface{}{
"name": "dummy",
"endpoint": endpoint,
"region": "us-east-2",
"cacheTablename": "BlobTable",
"repoMetaTablename": "RepoMetadataTable",
"manifestDataTablename": "ManifestDataTable",
"userDataTablename": "ZotUserDataTable",
"versionTablename": "Version",
"name": "dummy",
"endpoint": endpoint,
"region": "us-east-2",
"cacheTablename": "BlobTable",
"repoMetaTablename": "RepoMetadataTable",
"imageDataTablename": "ZotImageDataTable",
"repoBlobsInfoTablename": "ZotRepoBlobsInfoTable",
"userDataTablename": "ZotUserDataTable",
"versionTablename": "Version",
}

driver = storage.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log)
Expand All @@ -215,43 +218,46 @@ func TestCreateMetaDBDriver(t *testing.T) {
}

conf.Storage.CacheDriver = map[string]interface{}{
"name": "dummy",
"endpoint": "http://localhost:4566",
"region": "us-east-2",
"cachetablename": "BlobTable",
"repometatablename": "RepoMetadataTable",
"manifestdatatablename": "ManifestDataTable",
"userdatatablename": "UserDatatable",
"name": "dummy",
"endpoint": "http://localhost:4566",
"region": "us-east-2",
"cachetablename": "BlobTable",
"repometatablename": "RepoMetadataTable",
"imageDataTablename": "ZotImageDataTable",
"repoBlobsInfoTablename": "ZotRepoBlobsInfoTable",
"userdatatablename": "UserDatatable",
}

testFunc := func() { _, _ = meta.New(conf.Storage.StorageConfig, log) }
So(testFunc, ShouldPanic)

conf.Storage.CacheDriver = map[string]interface{}{
"name": "dummy",
"endpoint": "http://localhost:4566",
"region": "us-east-2",
"cachetablename": "",
"repometatablename": "RepoMetadataTable",
"manifestdatatablename": "ManifestDataTable",
"userDataTablename": "ZotUserDataTable",
"versiontablename": 1,
"name": "dummy",
"endpoint": "http://localhost:4566",
"region": "us-east-2",
"cachetablename": "",
"repometatablename": "RepoMetadataTable",
"imageDataTablename": "ZotImageDataTable",
"repoBlobsInfoTablename": "ZotRepoBlobsInfoTable",
"userDataTablename": "ZotUserDataTable",
"versiontablename": 1,
}

testFunc = func() { _, _ = meta.New(conf.Storage.StorageConfig, log) }
So(testFunc, ShouldPanic)

conf.Storage.CacheDriver = map[string]interface{}{
"name": "dummy",
"endpoint": "http://localhost:4566",
"region": "us-east-2",
"cachetablename": "test",
"repometatablename": "RepoMetadataTable",
"manifestdatatablename": "ManifestDataTable",
"indexdatatablename": "IndexDataTable",
"userdatatablename": "ZotUserDataTable",
"apikeytablename": "APIKeyTable",
"versiontablename": "1",
"name": "dummy",
"endpoint": "http://localhost:4566",
"region": "us-east-2",
"cachetablename": "test",
"repometatablename": "RepoMetadataTable",
"imagedatatablename": "ZotImageDataTable",
"repoblobsinfotablename": "ZotRepoBlobsInfoTable",
"indexdatatablename": "IndexDataTable",
"userdatatablename": "ZotUserDataTable",
"apikeytablename": "APIKeyTable",
"versiontablename": "1",
}

testFunc = func() { _, _ = meta.New(conf.Storage.StorageConfig, log) }
Expand Down Expand Up @@ -424,16 +430,17 @@ func TestObjectStorageController(t *testing.T) {
conf.Storage.StorageDriver = storageDriverParams

conf.Storage.CacheDriver = map[string]interface{}{
"name": "dynamodb",
"endpoint": os.Getenv("DYNAMODBMOCK_ENDPOINT"),
"region": "us-east-2",
"cachetablename": "test",
"repometatablename": "RepoMetadataTable",
"manifestdatatablename": "ManifestDataTable",
"indexdatatablename": "IndexDataTable",
"userdatatablename": "ZotUserDataTable",
"apikeytablename": "APIKeyTable1",
"versiontablename": "Version",
"name": "dynamodb",
"endpoint": os.Getenv("DYNAMODBMOCK_ENDPOINT"),
"region": "us-east-2",
"cachetablename": "test",
"repometatablename": "RepoMetadataTable",
"imagedatatablename": "ZotImageDataTable",
"repoblobsinfotablename": "ZotRepoBlobsInfoTable",
"indexdatatablename": "IndexDataTable",
"userdatatablename": "ZotUserDataTable",
"apikeytablename": "APIKeyTable1",
"versiontablename": "Version",
}

mockOIDCServer, err := authutils.MockOIDCRun()
Expand Down
2 changes: 1 addition & 1 deletion pkg/api/routes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -516,7 +516,7 @@ func (rh *RouteHandler) GetManifest(response http.ResponseWriter, request *http.
}

if rh.c.MetaDB != nil {
err := meta.OnGetManifest(name, reference, content, rh.c.StoreController, rh.c.MetaDB, rh.c.Log)
err := meta.OnGetManifest(name, reference, mediaType, content, rh.c.StoreController, rh.c.MetaDB, rh.c.Log)
if err != nil {
response.WriteHeader(http.StatusInternalServerError)

Expand Down
12 changes: 2 additions & 10 deletions pkg/cli/client/cve_cmd_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ package client_test
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
Expand Down Expand Up @@ -756,19 +755,12 @@ func getMockCveScanner(metaDB mTypes.MetaDB) cveinfo.Scanner {
return false, err
}

manifestData, err := metaDB.GetManifestData(manifestDigest)
manifestData, err := metaDB.GetImageData(manifestDigest)
if err != nil {
return false, err
}

var manifestContent ispec.Manifest

err = json.Unmarshal(manifestData.ManifestBlob, &manifestContent)
if err != nil {
return false, zerr.ErrScanNotSupported
}

for _, imageLayer := range manifestContent.Layers {
for _, imageLayer := range manifestData.Manifests[0].Layers {
switch imageLayer.MediaType {
case ispec.MediaTypeImageLayerGzip, ispec.MediaTypeImageLayer, string(regTypes.DockerLayer):

Expand Down
20 changes: 0 additions & 20 deletions pkg/common/oci.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,8 @@ import (

"github.com/opencontainers/go-digest"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
"google.golang.org/protobuf/types/known/timestamppb"

zerr "zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/meta/proto_go"
)

func GetImageDirAndTag(imageName string) (string, string) {
Expand Down Expand Up @@ -86,24 +84,6 @@ func GetImageLastUpdated(imageInfo ispec.Image) time.Time {
return *timeStamp
}

func GetProtoImageLastUpdated(configData *proto_go.ConfigData) time.Time {
timeStamp := configData.Created

if timeStamp != nil && !timeStamp.AsTime().IsZero() {
return timeStamp.AsTime()
}

if len(configData.History) > 0 {
timeStamp = configData.History[len(configData.History)-1].Created
}

if timeStamp == nil {
timeStamp = timestamppb.New(time.Time{})
}

return timeStamp.AsTime()
}

// GetRepoReference returns the components of a repoName:tag or repoName@digest string. If the format is wrong
// an error is returned.
// The returned values have the following meaning:
Expand Down
24 changes: 12 additions & 12 deletions pkg/extensions/extension_image_trust_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,23 +131,23 @@ func TestSignatureUploadAndVerificationAWS(t *testing.T) {

cacheTablename := "BlobTable" + uuid.String()
repoMetaTablename := "RepoMetadataTable" + uuid.String()
manifestDataTablename := "ManifestDataTable" + uuid.String()
versionTablename := "Version" + uuid.String()
indexDataTablename := "IndexDataTable" + uuid.String()
userDataTablename := "UserDataTable" + uuid.String()
apiKeyTablename := "ApiKeyTable" + uuid.String()
imageDataTablename := "imageDataTable" + uuid.String()
repoBlobsInfoTablename := "repoBlobsInfoTable" + uuid.String()

cacheDriverParams := map[string]interface{}{
"name": "dynamoDB",
"endpoint": os.Getenv("DYNAMODBMOCK_ENDPOINT"),
"region": "us-east-2",
"cacheTablename": cacheTablename,
"repoMetaTablename": repoMetaTablename,
"manifestDataTablename": manifestDataTablename,
"indexDataTablename": indexDataTablename,
"userDataTablename": userDataTablename,
"apiKeyTablename": apiKeyTablename,
"versionTablename": versionTablename,
"name": "dynamoDB",
"endpoint": os.Getenv("DYNAMODBMOCK_ENDPOINT"),
"region": "us-east-2",
"cacheTablename": cacheTablename,
"repoMetaTablename": repoMetaTablename,
"imageDataTablename": imageDataTablename,
"repoBlobsInfoTablename": repoBlobsInfoTablename,
"userDataTablename": userDataTablename,
"apiKeyTablename": apiKeyTablename,
"versionTablename": versionTablename,
}

t.Logf("using dynamo driver options: %v", cacheDriverParams)
Expand Down
16 changes: 8 additions & 8 deletions pkg/extensions/imagetrust/image_trust.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ func GetSecretsManagerRetrieval(region, endpoint string) *secretcache.Cache {
}

func (imgTrustStore *ImageTrustStore) ProtoVerifySignature(
signatureType string, rawSignature []byte, sigKey string, manifestDigest godigest.Digest, imageData mTypes.ImageData2,
signatureType string, rawSignature []byte, sigKey string, manifestDigest godigest.Digest, imageData mTypes.ImageData,
repo string,
) (string, time.Time, bool, error) {
desc := ispec.Descriptor{
Expand Down Expand Up @@ -201,15 +201,15 @@ func (imgTrustStore *ImageTrustStore) VerifySignature(

func NewTaskGenerator(metaDB mTypes.MetaDB, log log.Logger) scheduler.TaskGenerator {
return &sigValidityTaskGenerator{
repos: []mTypes.RepoMetadata2{},
repos: []mTypes.RepoMetadata{},
metaDB: metaDB,
repoIndex: -1,
log: log,
}
}

type sigValidityTaskGenerator struct {
repos []mTypes.RepoMetadata2
repos []mTypes.RepoMetadata
metaDB mTypes.MetaDB
repoIndex int
done bool
Expand All @@ -220,7 +220,7 @@ func (gen *sigValidityTaskGenerator) Next() (scheduler.Task, error) {
if len(gen.repos) == 0 {
ctx := context.Background()

repos, err := gen.metaDB.ProtoGetMultipleRepoMeta(ctx, func(repoMeta mTypes.RepoMetadata2) bool {
repos, err := gen.metaDB.GetMultipleRepoMeta(ctx, func(repoMeta mTypes.RepoMetadata) bool {
return true
})
if err != nil {
Expand Down Expand Up @@ -254,18 +254,18 @@ func (gen *sigValidityTaskGenerator) IsReady() bool {
func (gen *sigValidityTaskGenerator) Reset() {
gen.done = false
gen.repoIndex = -1
gen.repos = []mTypes.RepoMetadata2{}
gen.repos = []mTypes.RepoMetadata{}

gen.log.Info().Msg("finished resetting task generator for updating signatures validity")
}

type validityTask struct {
metaDB mTypes.MetaDB
repo mTypes.RepoMetadata2
repo mTypes.RepoMetadata
log log.Logger
}

func NewValidityTask(metaDB mTypes.MetaDB, repo mTypes.RepoMetadata2, log log.Logger) *validityTask {
func NewValidityTask(metaDB mTypes.MetaDB, repo mTypes.RepoMetadata, log log.Logger) *validityTask {
return &validityTask{metaDB, repo, log}
}

Expand All @@ -274,7 +274,7 @@ func (validityT *validityTask) DoWork(ctx context.Context) error {

for signedManifest, sigs := range validityT.repo.Signatures {
if len(sigs[zcommon.CosignSignature]) != 0 || len(sigs[zcommon.NotationSignature]) != 0 {
err := validityT.metaDB.ProtoUpdateSignaturesValidity(validityT.repo.Name, godigest.Digest(signedManifest))
err := validityT.metaDB.UpdateSignaturesValidity(validityT.repo.Name, godigest.Digest(signedManifest))
if err != nil {
validityT.log.Info().Msg("error while verifying signatures")

Expand Down
2 changes: 1 addition & 1 deletion pkg/extensions/imagetrust/image_trust_disabled.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ func (imgTrustStore *imageTrustDisabled) VerifySignature(
}

func (imgTrustStore *imageTrustDisabled) ProtoVerifySignature(
signatureType string, rawSignature []byte, sigKey string, manifestDigest godigest.Digest, imageData mTypes.ImageData2,
signatureType string, rawSignature []byte, sigKey string, manifestDigest godigest.Digest, imageData mTypes.ImageData,
repo string,
) (string, time.Time, bool, error) {
return "", time.Time{}, false, nil
Expand Down
Loading

0 comments on commit 414ce16

Please sign in to comment.