feat(meta): code clean-up

change cve logic to use new db remove part of the old metadb functions Signed-off-by: Laurentiu Niculae <[email protected]>
project-zot · Oct 17, 2023 · 292ac05 · 292ac05
1 parent b0da7a7
commit 292ac05
Show file tree

Hide file tree

Showing 79 changed files with 7,101 additions and 14,669 deletions.
diff --git a/Makefile b/Makefile
@@ -104,41 +104,14 @@ gen-protobuf: $(PROTOC)
 	$(PROTOC) --experimental_allow_proto3_optional \
 		--proto_path=$(TOP_LEVEL)/pkg/meta/proto \
 		--go_out=$(TOP_LEVEL)/pkg/meta/ \
-		--go_opt='Mdescriptor.proto=./proto_go' \
-		$(TOP_LEVEL)/pkg/meta/proto/descriptor.proto
+		--go_opt='Moci.proto=./proto_go' \
+		--go_opt='Mmeta.proto=./proto_go' \
+		$(TOP_LEVEL)/pkg/meta/proto/meta.proto
 	$(PROTOC) --experimental_allow_proto3_optional \
 		--proto_path=$(TOP_LEVEL)/pkg/meta/proto \
 		--go_out=$(TOP_LEVEL)/pkg/meta/ \
-		--go_opt='Mconfig.proto=./proto_go' \
-		--go_opt='Mdescriptor.proto=./proto_go' \
-		$(TOP_LEVEL)/pkg/meta/proto/config.proto
-	$(PROTOC) --experimental_allow_proto3_optional \
-		--proto_path=$(TOP_LEVEL)/pkg/meta/proto \
-		--go_out=$(TOP_LEVEL)/pkg/meta/ \
-		--go_opt='Mversioned.proto=./proto_go' \
-		$(TOP_LEVEL)/pkg/meta/proto/versioned.proto
-	$(PROTOC) --experimental_allow_proto3_optional \
-		--proto_path=$(TOP_LEVEL)/pkg/meta/proto \
-		--go_out=$(TOP_LEVEL)/pkg/meta/ \
-		--go_opt='Mmanifest.proto=./proto_go' \
-		--go_opt='Mdescriptor.proto=./proto_go' \
-		--go_opt='Mversioned.proto=./proto_go' \
-		$(TOP_LEVEL)/pkg/meta/proto/manifest.proto
-	$(PROTOC) --experimental_allow_proto3_optional \
-		--proto_path=$(TOP_LEVEL)/pkg/meta/proto \
-		--go_out=$(TOP_LEVEL)/pkg/meta/ \
-		--go_opt='Mindex.proto=./proto_go' \
-		--go_opt='Mdescriptor.proto=./proto_go' \
-		--go_opt='Mversioned.proto=./proto_go' \
-		$(TOP_LEVEL)/pkg/meta/proto/index.proto
-	$(PROTOC) --experimental_allow_proto3_optional \
-		--proto_path=$(TOP_LEVEL)/pkg/meta/proto \
-		--go_out=$(TOP_LEVEL)/pkg/meta/ \
-		--go_opt='MimageData.proto=./proto_go' \
-		--go_opt='Mdescriptor.proto=./proto_go' \
-		--go_opt='Mversioned.proto=./proto_go' \
-		--go_opt='Mconfig.proto=./proto_go' \
-		$(TOP_LEVEL)/pkg/meta/proto/imageData.proto
+		--go_opt='Moci.proto=./proto_go' \
+		$(TOP_LEVEL)/pkg/meta/proto/oci.proto
 
 .PHONY: binary-minimal
 binary-minimal: EXTENSIONS=

diff --git a/errors/errors.go b/errors/errors.go
@@ -108,6 +108,8 @@ var (
 	ErrManifestConflict               = errors.New("manifest: multiple manifests found")
 	ErrManifestMetaNotFound           = errors.New("metadb: image metadata not found for given manifest reference")
 	ErrManifestDataNotFound           = errors.New("metadb: image data not found for given manifest digest")
+	ErrImageDataNotFound              = errors.New("metadb: image data not found for")
+	ErrWrongMediaType                 = errors.New("metadb: got unexpected media type")
 	ErrIndexDataNotFount              = errors.New("metadb: index data not found for given digest")
 	ErrRepoMetaNotFound               = errors.New("metadb: repo metadata not found for given repo name")
 	ErrTagMetaNotFound                = errors.New("metadb: tag metadata not found for given repo and tag names")

diff --git a/examples/config-all-remote.json b/examples/config-all-remote.json
@@ -18,7 +18,8 @@
             "region": "us-east-2",
             "cacheTablename": "ZotBlobTable",
             "repoMetaTablename": "ZotRepoMetadataTable",
-            "manifestDataTablename": "ZotManifestDataTable",
+            "imageDataTablename": "ZotImageDataTable",
+            "repoBlobsInfoTablename": "ZotRepoBlobsInfoTable",
             "versionTablename": "ZotVersion"
         }
     },

diff --git a/examples/config-dynamodb.json b/examples/config-dynamodb.json
@@ -19,7 +19,8 @@
             "region": "us-east-2",
             "cacheTablename": "ZotBlobTable",
             "repoMetaTablename": "ZotRepoMetadataTable",
-            "manifestDataTablename": "ZotManifestDataTable",
+            "imageDataTablename": "ZotImageDataTable",
+            "repoBlobsInfoTablename": "ZotRepoBlobsInfoTable",
             "userDataTablename": "ZotUserDataTable",
             "versionTablename": "ZotVersion"
         }

diff --git a/go.mod b/go.mod
@@ -500,7 +500,7 @@ require (
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb // indirect
 	google.golang.org/grpc v1.58.2 // indirect
-	google.golang.org/protobuf v1.31.0 // indirect
+	google.golang.org/protobuf v1.31.0
 	gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect

diff --git a/pkg/api/controller.go b/pkg/api/controller.go
@@ -293,7 +293,6 @@ func (c *Controller) InitMetaDB(reloadCtx context.Context) error {
 			return err
 		}
 
-		// TODO: undo, commented for testing
 		err = meta.ParseStorage(driver, c.StoreController, c.Log)
 		if err != nil {
 			return err

diff --git a/pkg/api/controller_test.go b/pkg/api/controller_test.go
@@ -153,14 +153,15 @@ func TestCreateCacheDatabaseDriver(t *testing.T) {
 
 		endpoint := os.Getenv("DYNAMODBMOCK_ENDPOINT")
 		conf.Storage.CacheDriver = map[string]interface{}{
-			"name":                  "dynamodb",
-			"endpoint":              endpoint,
-			"region":                "us-east-2",
-			"cacheTablename":        "BlobTable",
-			"repoMetaTablename":     "RepoMetadataTable",
-			"manifestDataTablename": "ManifestDataTable",
-			"userDataTablename":     "ZotUserDataTable",
-			"versionTablename":      "Version",
+			"name":                   "dynamodb",
+			"endpoint":               endpoint,
+			"region":                 "us-east-2",
+			"cacheTablename":         "BlobTable",
+			"repoMetaTablename":      "RepoMetadataTable",
+			"imageDataTablename":     "ZotImageDataTable",
+			"repoBlobsInfoTablename": "ZotRepoBlobsInfoTable",
+			"userDataTablename":      "ZotUserDataTable",
+			"versionTablename":       "Version",
 		}
 
 		driver := storage.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log)
@@ -169,27 +170,29 @@ func TestCreateCacheDatabaseDriver(t *testing.T) {
 		// negative test cases
 
 		conf.Storage.CacheDriver = map[string]interface{}{
-			"endpoint":              endpoint,
-			"region":                "us-east-2",
-			"cacheTablename":        "BlobTable",
-			"repoMetaTablename":     "RepoMetadataTable",
-			"manifestDataTablename": "ManifestDataTable",
-			"userDataTablename":     "ZotUserDataTable",
-			"versionTablename":      "Version",
+			"endpoint":               endpoint,
+			"region":                 "us-east-2",
+			"cacheTablename":         "BlobTable",
+			"repoMetaTablename":      "RepoMetadataTable",
+			"imageDataTablename":     "ZotImageDataTable",
+			"repoBlobsInfoTablename": "ZotRepoBlobsInfoTable",
+			"userDataTablename":      "ZotUserDataTable",
+			"versionTablename":       "Version",
 		}
 
 		driver = storage.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log)
 		So(driver, ShouldBeNil)
 
 		conf.Storage.CacheDriver = map[string]interface{}{
-			"name":                  "dummy",
-			"endpoint":              endpoint,
-			"region":                "us-east-2",
-			"cacheTablename":        "BlobTable",
-			"repoMetaTablename":     "RepoMetadataTable",
-			"manifestDataTablename": "ManifestDataTable",
-			"userDataTablename":     "ZotUserDataTable",
-			"versionTablename":      "Version",
+			"name":                   "dummy",
+			"endpoint":               endpoint,
+			"region":                 "us-east-2",
+			"cacheTablename":         "BlobTable",
+			"repoMetaTablename":      "RepoMetadataTable",
+			"imageDataTablename":     "ZotImageDataTable",
+			"repoBlobsInfoTablename": "ZotRepoBlobsInfoTable",
+			"userDataTablename":      "ZotUserDataTable",
+			"versionTablename":       "Version",
 		}
 
 		driver = storage.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log)
@@ -215,43 +218,46 @@ func TestCreateMetaDBDriver(t *testing.T) {
 		}
 
 		conf.Storage.CacheDriver = map[string]interface{}{
-			"name":                  "dummy",
-			"endpoint":              "http://localhost:4566",
-			"region":                "us-east-2",
-			"cachetablename":        "BlobTable",
-			"repometatablename":     "RepoMetadataTable",
-			"manifestdatatablename": "ManifestDataTable",
-			"userdatatablename":     "UserDatatable",
+			"name":                   "dummy",
+			"endpoint":               "http://localhost:4566",
+			"region":                 "us-east-2",
+			"cachetablename":         "BlobTable",
+			"repometatablename":      "RepoMetadataTable",
+			"imageDataTablename":     "ZotImageDataTable",
+			"repoBlobsInfoTablename": "ZotRepoBlobsInfoTable",
+			"userdatatablename":      "UserDatatable",
 		}
 
 		testFunc := func() { _, _ = meta.New(conf.Storage.StorageConfig, log) }
 		So(testFunc, ShouldPanic)
 
 		conf.Storage.CacheDriver = map[string]interface{}{
-			"name":                  "dummy",
-			"endpoint":              "http://localhost:4566",
-			"region":                "us-east-2",
-			"cachetablename":        "",
-			"repometatablename":     "RepoMetadataTable",
-			"manifestdatatablename": "ManifestDataTable",
-			"userDataTablename":     "ZotUserDataTable",
-			"versiontablename":      1,
+			"name":                   "dummy",
+			"endpoint":               "http://localhost:4566",
+			"region":                 "us-east-2",
+			"cachetablename":         "",
+			"repometatablename":      "RepoMetadataTable",
+			"imageDataTablename":     "ZotImageDataTable",
+			"repoBlobsInfoTablename": "ZotRepoBlobsInfoTable",
+			"userDataTablename":      "ZotUserDataTable",
+			"versiontablename":       1,
 		}
 
 		testFunc = func() { _, _ = meta.New(conf.Storage.StorageConfig, log) }
 		So(testFunc, ShouldPanic)
 
 		conf.Storage.CacheDriver = map[string]interface{}{
-			"name":                  "dummy",
-			"endpoint":              "http://localhost:4566",
-			"region":                "us-east-2",
-			"cachetablename":        "test",
-			"repometatablename":     "RepoMetadataTable",
-			"manifestdatatablename": "ManifestDataTable",
-			"indexdatatablename":    "IndexDataTable",
-			"userdatatablename":     "ZotUserDataTable",
-			"apikeytablename":       "APIKeyTable",
-			"versiontablename":      "1",
+			"name":                   "dummy",
+			"endpoint":               "http://localhost:4566",
+			"region":                 "us-east-2",
+			"cachetablename":         "test",
+			"repometatablename":      "RepoMetadataTable",
+			"imagedatatablename":     "ZotImageDataTable",
+			"repoblobsinfotablename": "ZotRepoBlobsInfoTable",
+			"indexdatatablename":     "IndexDataTable",
+			"userdatatablename":      "ZotUserDataTable",
+			"apikeytablename":        "APIKeyTable",
+			"versiontablename":       "1",
 		}
 
 		testFunc = func() { _, _ = meta.New(conf.Storage.StorageConfig, log) }
@@ -424,16 +430,17 @@ func TestObjectStorageController(t *testing.T) {
 		conf.Storage.StorageDriver = storageDriverParams
 
 		conf.Storage.CacheDriver = map[string]interface{}{
-			"name":                  "dynamodb",
-			"endpoint":              os.Getenv("DYNAMODBMOCK_ENDPOINT"),
-			"region":                "us-east-2",
-			"cachetablename":        "test",
-			"repometatablename":     "RepoMetadataTable",
-			"manifestdatatablename": "ManifestDataTable",
-			"indexdatatablename":    "IndexDataTable",
-			"userdatatablename":     "ZotUserDataTable",
-			"apikeytablename":       "APIKeyTable1",
-			"versiontablename":      "Version",
+			"name":                   "dynamodb",
+			"endpoint":               os.Getenv("DYNAMODBMOCK_ENDPOINT"),
+			"region":                 "us-east-2",
+			"cachetablename":         "test",
+			"repometatablename":      "RepoMetadataTable",
+			"imagedatatablename":     "ZotImageDataTable",
+			"repoblobsinfotablename": "ZotRepoBlobsInfoTable",
+			"indexdatatablename":     "IndexDataTable",
+			"userdatatablename":      "ZotUserDataTable",
+			"apikeytablename":        "APIKeyTable1",
+			"versiontablename":       "Version",
 		}
 
 		mockOIDCServer, err := authutils.MockOIDCRun()

diff --git a/pkg/api/routes.go b/pkg/api/routes.go
@@ -516,7 +516,7 @@ func (rh *RouteHandler) GetManifest(response http.ResponseWriter, request *http.
 	}
 
 	if rh.c.MetaDB != nil {
-		err := meta.OnGetManifest(name, reference, content, rh.c.StoreController, rh.c.MetaDB, rh.c.Log)
+		err := meta.OnGetManifest(name, reference, mediaType, content, rh.c.StoreController, rh.c.MetaDB, rh.c.Log)
 		if err != nil {
 			response.WriteHeader(http.StatusInternalServerError)
 

diff --git a/pkg/cli/client/cve_cmd_test.go b/pkg/cli/client/cve_cmd_test.go
@@ -6,7 +6,6 @@ package client_test
 import (
 	"bytes"
 	"context"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -756,19 +755,12 @@ func getMockCveScanner(metaDB mTypes.MetaDB) cveinfo.Scanner {
 				return false, err
 			}
 
-			manifestData, err := metaDB.GetManifestData(manifestDigest)
+			manifestData, err := metaDB.GetImageData(manifestDigest)
 			if err != nil {
 				return false, err
 			}
 
-			var manifestContent ispec.Manifest
-
-			err = json.Unmarshal(manifestData.ManifestBlob, &manifestContent)
-			if err != nil {
-				return false, zerr.ErrScanNotSupported
-			}
-
-			for _, imageLayer := range manifestContent.Layers {
+			for _, imageLayer := range manifestData.Manifests[0].Layers {
 				switch imageLayer.MediaType {
 				case ispec.MediaTypeImageLayerGzip, ispec.MediaTypeImageLayer, string(regTypes.DockerLayer):
 

diff --git a/pkg/common/oci.go b/pkg/common/oci.go
@@ -6,10 +6,8 @@ import (
 
 	"github.com/opencontainers/go-digest"
 	ispec "github.com/opencontainers/image-spec/specs-go/v1"
-	"google.golang.org/protobuf/types/known/timestamppb"
 
 	zerr "zotregistry.io/zot/errors"
-	"zotregistry.io/zot/pkg/meta/proto_go"
 )
 
 func GetImageDirAndTag(imageName string) (string, string) {
@@ -86,24 +84,6 @@ func GetImageLastUpdated(imageInfo ispec.Image) time.Time {
 	return *timeStamp
 }
 
-func GetProtoImageLastUpdated(configData *proto_go.ConfigData) time.Time {
-	timeStamp := configData.Created
-
-	if timeStamp != nil && !timeStamp.AsTime().IsZero() {
-		return timeStamp.AsTime()
-	}
-
-	if len(configData.History) > 0 {
-		timeStamp = configData.History[len(configData.History)-1].Created
-	}
-
-	if timeStamp == nil {
-		timeStamp = timestamppb.New(time.Time{})
-	}
-
-	return timeStamp.AsTime()
-}
-
 // GetRepoReference returns the components of a repoName:tag or repoName@digest string. If the format is wrong
 // an error is returned.
 // The returned values have the following meaning:

diff --git a/pkg/extensions/extension_image_trust_test.go b/pkg/extensions/extension_image_trust_test.go
@@ -131,23 +131,23 @@ func TestSignatureUploadAndVerificationAWS(t *testing.T) {
 
 		cacheTablename := "BlobTable" + uuid.String()
 		repoMetaTablename := "RepoMetadataTable" + uuid.String()
-		manifestDataTablename := "ManifestDataTable" + uuid.String()
 		versionTablename := "Version" + uuid.String()
-		indexDataTablename := "IndexDataTable" + uuid.String()
 		userDataTablename := "UserDataTable" + uuid.String()
 		apiKeyTablename := "ApiKeyTable" + uuid.String()
+		imageDataTablename := "imageDataTable" + uuid.String()
+		repoBlobsInfoTablename := "repoBlobsInfoTable" + uuid.String()
 
 		cacheDriverParams := map[string]interface{}{
-			"name":                  "dynamoDB",
-			"endpoint":              os.Getenv("DYNAMODBMOCK_ENDPOINT"),
-			"region":                "us-east-2",
-			"cacheTablename":        cacheTablename,
-			"repoMetaTablename":     repoMetaTablename,
-			"manifestDataTablename": manifestDataTablename,
-			"indexDataTablename":    indexDataTablename,
-			"userDataTablename":     userDataTablename,
-			"apiKeyTablename":       apiKeyTablename,
-			"versionTablename":      versionTablename,
+			"name":                   "dynamoDB",
+			"endpoint":               os.Getenv("DYNAMODBMOCK_ENDPOINT"),
+			"region":                 "us-east-2",
+			"cacheTablename":         cacheTablename,
+			"repoMetaTablename":      repoMetaTablename,
+			"imageDataTablename":     imageDataTablename,
+			"repoBlobsInfoTablename": repoBlobsInfoTablename,
+			"userDataTablename":      userDataTablename,
+			"apiKeyTablename":        apiKeyTablename,
+			"versionTablename":       versionTablename,
 		}
 
 		t.Logf("using dynamo driver options: %v", cacheDriverParams)

diff --git a/pkg/extensions/imagetrust/image_trust.go b/pkg/extensions/imagetrust/image_trust.go
@@ -141,7 +141,7 @@ func GetSecretsManagerRetrieval(region, endpoint string) *secretcache.Cache {
 }
 
 func (imgTrustStore *ImageTrustStore) ProtoVerifySignature(
-	signatureType string, rawSignature []byte, sigKey string, manifestDigest godigest.Digest, imageData mTypes.ImageData2,
+	signatureType string, rawSignature []byte, sigKey string, manifestDigest godigest.Digest, imageData mTypes.ImageData,
 	repo string,
 ) (string, time.Time, bool, error) {
 	desc := ispec.Descriptor{

diff --git a/pkg/extensions/imagetrust/image_trust_disabled.go b/pkg/extensions/imagetrust/image_trust_disabled.go
@@ -29,7 +29,7 @@ func (imgTrustStore *imageTrustDisabled) VerifySignature(
 }
 
 func (imgTrustStore *imageTrustDisabled) ProtoVerifySignature(
-	signatureType string, rawSignature []byte, sigKey string, manifestDigest godigest.Digest, imageData mTypes.ImageData2,
+	signatureType string, rawSignature []byte, sigKey string, manifestDigest godigest.Digest, imageData mTypes.ImageData,
 	repo string,
 ) (string, time.Time, bool, error) {
 	return "", time.Time{}, false, nil