feat(meta): code clean-up

change cve logic to use new db remove part of the old metadb functions refactor(meta): aesthethic changes feat(test): remove all old meta methods refactor(proto): change proto structs reactor(meta): cleanup Signed-off-by: Laurentiu Niculae <[email protected]>
project-zot · Oct 24, 2023 · 32dec56 · 32dec56
1 parent 8d27fbd
commit 32dec56
Show file tree

Hide file tree

Showing 87 changed files with 8,064 additions and 18,114 deletions.
diff --git a/Makefile b/Makefile
@@ -28,12 +28,26 @@ ZUI_VERSION := commit-fad5572
 SWAGGER_VERSION := v1.8.12
 STACKER := $(TOOLSDIR)/bin/stacker
 BATS := $(TOOLSDIR)/bin/bats
-PROTOC_VERSION := 3.15.8
-PROTOC := $(TOOLSDIR)/bin/protoc
 TESTDATA := $(TOP_LEVEL)/test/data
 OS ?= $(shell go env GOOS)
 ARCH ?= $(shell go env GOARCH)
 
+GO_PROTOC_VERSION := 1.31.0
+PROTOC_VERSION := 3.15.8
+HOST_OS := $(shell go env GOOS)
+HOST_ARCH := $(shell go env GOARCH)
+ifeq ($(HOST_OS),linux)
+	PROTOC_OS = linux
+	PROTOC_ARCH = x86_64
+	ifeq ($(HOST_ARCH),arm64)
+		PROTOC_ARCH = aarch_64
+	endif
+else ifeq ($(HOST_OS),darwin)
+	PROTOC_OS = osx
+	PROTOC_ARCH = universal_binary
+endif
+PROTOC := $(TOOLSDIR)/bin/protoc
+
 BENCH_OUTPUT ?= stdout
 ALL_EXTENSIONS = debug,imagetrust,lint,metrics,mgmt,profile,scrub,search,sync,ui,userprefs
 EXTENSIONS ?= sync,search,scrub,metrics,lint,ui,mgmt,profile,userprefs,imagetrust
@@ -104,14 +118,14 @@ gen-protobuf: $(PROTOC)
 	$(PROTOC) --experimental_allow_proto3_optional \
 		--proto_path=$(TOP_LEVEL)/pkg/meta/proto \
 		--go_out=$(TOP_LEVEL)/pkg/meta/ \
-		--go_opt='Mdescriptor.proto=./proto_go' \
-		$(TOP_LEVEL)/pkg/meta/proto/descriptor.proto
-	$(PROTOC) --experimental_allow_proto3_optional \
-		--proto_path=$(TOP_LEVEL)/pkg/meta/proto \
-		--go_out=$(TOP_LEVEL)/pkg/meta/ \
+		--go_opt='Moci.proto=./proto_go' \
+		--go_opt='Mmeta.proto=./proto_go' \
 		--go_opt='Mconfig.proto=./proto_go' \
+		--go_opt='Mmanifest.proto=./proto_go' \
+		--go_opt='Mindex.proto=./proto_go' \
 		--go_opt='Mdescriptor.proto=./proto_go' \
-		$(TOP_LEVEL)/pkg/meta/proto/config.proto
+		--go_opt='Mversioned.proto=./proto_go' \
+		$(TOP_LEVEL)/pkg/meta/proto/meta.proto
 	$(PROTOC) --experimental_allow_proto3_optional \
 		--proto_path=$(TOP_LEVEL)/pkg/meta/proto \
 		--go_out=$(TOP_LEVEL)/pkg/meta/ \
@@ -120,25 +134,29 @@ gen-protobuf: $(PROTOC)
 	$(PROTOC) --experimental_allow_proto3_optional \
 		--proto_path=$(TOP_LEVEL)/pkg/meta/proto \
 		--go_out=$(TOP_LEVEL)/pkg/meta/ \
-		--go_opt='Mmanifest.proto=./proto_go' \
 		--go_opt='Mdescriptor.proto=./proto_go' \
-		--go_opt='Mversioned.proto=./proto_go' \
-		$(TOP_LEVEL)/pkg/meta/proto/manifest.proto
+		$(TOP_LEVEL)/pkg/meta/proto/descriptor.proto
 	$(PROTOC) --experimental_allow_proto3_optional \
 		--proto_path=$(TOP_LEVEL)/pkg/meta/proto \
 		--go_out=$(TOP_LEVEL)/pkg/meta/ \
-		--go_opt='Mindex.proto=./proto_go' \
 		--go_opt='Mdescriptor.proto=./proto_go' \
 		--go_opt='Mversioned.proto=./proto_go' \
+		--go_opt='Mindex.proto=./proto_go' \
 		$(TOP_LEVEL)/pkg/meta/proto/index.proto
 	$(PROTOC) --experimental_allow_proto3_optional \
 		--proto_path=$(TOP_LEVEL)/pkg/meta/proto \
 		--go_out=$(TOP_LEVEL)/pkg/meta/ \
-		--go_opt='MimageData.proto=./proto_go' \
+		--go_opt='Moci.proto=./proto_go' \
 		--go_opt='Mdescriptor.proto=./proto_go' \
-		--go_opt='Mversioned.proto=./proto_go' \
 		--go_opt='Mconfig.proto=./proto_go' \
-		$(TOP_LEVEL)/pkg/meta/proto/imageData.proto
+		$(TOP_LEVEL)/pkg/meta/proto/config.proto
+	$(PROTOC) --experimental_allow_proto3_optional \
+		--proto_path=$(TOP_LEVEL)/pkg/meta/proto \
+		--go_out=$(TOP_LEVEL)/pkg/meta/ \
+		--go_opt='Mversioned.proto=./proto_go' \
+		--go_opt='Mdescriptor.proto=./proto_go' \
+		--go_opt='Mmanifest.proto=./proto_go' \
+		$(TOP_LEVEL)/pkg/meta/proto/manifest.proto
 
 .PHONY: binary-minimal
 binary-minimal: EXTENSIONS=
@@ -263,10 +281,10 @@ $(CRICTL):
 
 $(PROTOC):
 	mkdir -p $(TOOLSDIR)/bin
-	curl -Lo protoc.zip https://github.com/protocolbuffers/protobuf/releases/download/v$(PROTOC_VERSION)/protoc-$(PROTOC_VERSION)-linux-x86_64.zip
-	unzip -d $(TOOLSDIR) protoc.zip bin/protoc
+	curl -Lo protoc.zip https://github.com/protocolbuffers/protobuf/releases/download/v$(PROTOC_VERSION)/protoc-$(PROTOC_VERSION)-$(PROTOC_OS)-$(PROTOC_ARCH).zip
+	unzip -o -d $(TOOLSDIR) protoc.zip bin/protoc
 	chmod +x $(PROTOC)
-	go install google.golang.org/protobuf/cmd/protoc-gen-go@latest
+	go install google.golang.org/protobuf/cmd/protoc-gen-go@v$(GO_PROTOC_VERSION)
 
 $(ACTION_VALIDATOR):
 	mkdir -p $(TOOLSDIR)/bin

diff --git a/codecov.yml b/codecov.yml
@@ -6,3 +6,4 @@ ignore:
     - "./pkg/test/mocks/*.go"
     - "./swagger/*.go"
     - "./pkg/test/test_http_server.go"
+    - "./pkg/meta/proto_go/*.go"
diff --git a/errors/errors.go b/errors/errors.go
@@ -108,6 +108,8 @@ var (
 	ErrManifestConflict               = errors.New("manifest: multiple manifests found")
 	ErrManifestMetaNotFound           = errors.New("metadb: image metadata not found for given manifest reference")
 	ErrManifestDataNotFound           = errors.New("metadb: image data not found for given manifest digest")
+	ErrImageDataNotFound              = errors.New("metadb: image data not found")
+	ErrUnexpectedMediaType            = errors.New("metadb: got unexpected media type")
 	ErrIndexDataNotFount              = errors.New("metadb: index data not found for given digest")
 	ErrRepoMetaNotFound               = errors.New("metadb: repo metadata not found for given repo name")
 	ErrTagMetaNotFound                = errors.New("metadb: tag metadata not found for given repo and tag names")
@@ -163,4 +165,5 @@ var (
 	ErrInvalidOutputFormat            = errors.New("cli: invalid output format")
 	ErrFlagValueUnsupported           = errors.New("supported values ")
 	ErrUnknownSubcommand              = errors.New("cli: unknown subcommand")
+	ErrMultipleReposSameName          = errors.New("test: can't have multiple repos with the same name")
 )
diff --git a/examples/config-all-remote.json b/examples/config-all-remote.json
@@ -18,7 +18,8 @@
             "region": "us-east-2",
             "cacheTablename": "ZotBlobTable",
             "repoMetaTablename": "ZotRepoMetadataTable",
-            "manifestDataTablename": "ZotManifestDataTable",
+            "imageDataTablename": "ZotImageDataTable",
+            "repoBlobsInfoTablename": "ZotRepoBlobsInfoTable",
             "versionTablename": "ZotVersion"
         }
     },

diff --git a/examples/config-dynamodb.json b/examples/config-dynamodb.json
@@ -19,7 +19,8 @@
             "region": "us-east-2",
             "cacheTablename": "ZotBlobTable",
             "repoMetaTablename": "ZotRepoMetadataTable",
-            "manifestDataTablename": "ZotManifestDataTable",
+            "imageDataTablename": "ZotImageDataTable",
+            "repoBlobsInfoTablename": "ZotRepoBlobsInfoTable",
             "userDataTablename": "ZotUserDataTable",
             "versionTablename": "ZotVersion"
         }

diff --git a/go.mod b/go.mod
@@ -500,7 +500,7 @@ require (
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb // indirect
 	google.golang.org/grpc v1.58.2 // indirect
-	google.golang.org/protobuf v1.31.0 // indirect
+	google.golang.org/protobuf v1.31.0
 	gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect

diff --git a/pkg/api/controller.go b/pkg/api/controller.go
@@ -293,7 +293,6 @@ func (c *Controller) InitMetaDB(reloadCtx context.Context) error {
 			return err
 		}
 
-		// TODO: undo, commented for testing
 		err = meta.ParseStorage(driver, c.StoreController, c.Log)
 		if err != nil {
 			return err

diff --git a/pkg/api/controller_test.go b/pkg/api/controller_test.go
@@ -153,14 +153,15 @@ func TestCreateCacheDatabaseDriver(t *testing.T) {
 
 		endpoint := os.Getenv("DYNAMODBMOCK_ENDPOINT")
 		conf.Storage.CacheDriver = map[string]interface{}{
-			"name":                  "dynamodb",
-			"endpoint":              endpoint,
-			"region":                "us-east-2",
-			"cacheTablename":        "BlobTable",
-			"repoMetaTablename":     "RepoMetadataTable",
-			"manifestDataTablename": "ManifestDataTable",
-			"userDataTablename":     "ZotUserDataTable",
-			"versionTablename":      "Version",
+			"name":                   "dynamodb",
+			"endpoint":               endpoint,
+			"region":                 "us-east-2",
+			"cacheTablename":         "BlobTable",
+			"repoMetaTablename":      "RepoMetadataTable",
+			"imageDataTablename":     "ZotImageDataTable",
+			"repoBlobsInfoTablename": "ZotRepoBlobsInfoTable",
+			"userDataTablename":      "ZotUserDataTable",
+			"versionTablename":       "Version",
 		}
 
 		driver := storage.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log)
@@ -169,27 +170,29 @@ func TestCreateCacheDatabaseDriver(t *testing.T) {
 		// negative test cases
 
 		conf.Storage.CacheDriver = map[string]interface{}{
-			"endpoint":              endpoint,
-			"region":                "us-east-2",
-			"cacheTablename":        "BlobTable",
-			"repoMetaTablename":     "RepoMetadataTable",
-			"manifestDataTablename": "ManifestDataTable",
-			"userDataTablename":     "ZotUserDataTable",
-			"versionTablename":      "Version",
+			"endpoint":               endpoint,
+			"region":                 "us-east-2",
+			"cacheTablename":         "BlobTable",
+			"repoMetaTablename":      "RepoMetadataTable",
+			"imageDataTablename":     "ZotImageDataTable",
+			"repoBlobsInfoTablename": "ZotRepoBlobsInfoTable",
+			"userDataTablename":      "ZotUserDataTable",
+			"versionTablename":       "Version",
 		}
 
 		driver = storage.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log)
 		So(driver, ShouldBeNil)
 
 		conf.Storage.CacheDriver = map[string]interface{}{
-			"name":                  "dummy",
-			"endpoint":              endpoint,
-			"region":                "us-east-2",
-			"cacheTablename":        "BlobTable",
-			"repoMetaTablename":     "RepoMetadataTable",
-			"manifestDataTablename": "ManifestDataTable",
-			"userDataTablename":     "ZotUserDataTable",
-			"versionTablename":      "Version",
+			"name":                   "dummy",
+			"endpoint":               endpoint,
+			"region":                 "us-east-2",
+			"cacheTablename":         "BlobTable",
+			"repoMetaTablename":      "RepoMetadataTable",
+			"imageDataTablename":     "ZotImageDataTable",
+			"repoBlobsInfoTablename": "ZotRepoBlobsInfoTable",
+			"userDataTablename":      "ZotUserDataTable",
+			"versionTablename":       "Version",
 		}
 
 		driver = storage.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log)
@@ -215,43 +218,45 @@ func TestCreateMetaDBDriver(t *testing.T) {
 		}
 
 		conf.Storage.CacheDriver = map[string]interface{}{
-			"name":                  "dummy",
-			"endpoint":              "http://localhost:4566",
-			"region":                "us-east-2",
-			"cachetablename":        "BlobTable",
-			"repometatablename":     "RepoMetadataTable",
-			"manifestdatatablename": "ManifestDataTable",
-			"userdatatablename":     "UserDatatable",
+			"name":                   "dummy",
+			"endpoint":               "http://localhost:4566",
+			"region":                 "us-east-2",
+			"cachetablename":         "BlobTable",
+			"repometatablename":      "RepoMetadataTable",
+			"imageDataTablename":     "ZotImageDataTable",
+			"repoBlobsInfoTablename": "ZotRepoBlobsInfoTable",
+			"userdatatablename":      "UserDatatable",
 		}
 
 		testFunc := func() { _, _ = meta.New(conf.Storage.StorageConfig, log) }
 		So(testFunc, ShouldPanic)
 
 		conf.Storage.CacheDriver = map[string]interface{}{
-			"name":                  "dummy",
-			"endpoint":              "http://localhost:4566",
-			"region":                "us-east-2",
-			"cachetablename":        "",
-			"repometatablename":     "RepoMetadataTable",
-			"manifestdatatablename": "ManifestDataTable",
-			"userDataTablename":     "ZotUserDataTable",
-			"versiontablename":      1,
+			"name":                   "dummy",
+			"endpoint":               "http://localhost:4566",
+			"region":                 "us-east-2",
+			"cachetablename":         "",
+			"repometatablename":      "RepoMetadataTable",
+			"imageDataTablename":     "ZotImageDataTable",
+			"repoBlobsInfoTablename": "ZotRepoBlobsInfoTable",
+			"userDataTablename":      "ZotUserDataTable",
+			"versiontablename":       1,
 		}
 
 		testFunc = func() { _, _ = meta.New(conf.Storage.StorageConfig, log) }
 		So(testFunc, ShouldPanic)
 
 		conf.Storage.CacheDriver = map[string]interface{}{
-			"name":                  "dummy",
-			"endpoint":              "http://localhost:4566",
-			"region":                "us-east-2",
-			"cachetablename":        "test",
-			"repometatablename":     "RepoMetadataTable",
-			"manifestdatatablename": "ManifestDataTable",
-			"indexdatatablename":    "IndexDataTable",
-			"userdatatablename":     "ZotUserDataTable",
-			"apikeytablename":       "APIKeyTable",
-			"versiontablename":      "1",
+			"name":                   "dummy",
+			"endpoint":               "http://localhost:4566",
+			"region":                 "us-east-2",
+			"cachetablename":         "test",
+			"repometatablename":      "RepoMetadataTable",
+			"imagedatatablename":     "ZotImageDataTable",
+			"repoblobsinfotablename": "ZotRepoBlobsInfoTable",
+			"userdatatablename":      "ZotUserDataTable",
+			"apikeytablename":        "APIKeyTable",
+			"versiontablename":       "1",
 		}
 
 		testFunc = func() { _, _ = meta.New(conf.Storage.StorageConfig, log) }
@@ -424,16 +429,16 @@ func TestObjectStorageController(t *testing.T) {
 		conf.Storage.StorageDriver = storageDriverParams
 
 		conf.Storage.CacheDriver = map[string]interface{}{
-			"name":                  "dynamodb",
-			"endpoint":              os.Getenv("DYNAMODBMOCK_ENDPOINT"),
-			"region":                "us-east-2",
-			"cachetablename":        "test",
-			"repometatablename":     "RepoMetadataTable",
-			"manifestdatatablename": "ManifestDataTable",
-			"indexdatatablename":    "IndexDataTable",
-			"userdatatablename":     "ZotUserDataTable",
-			"apikeytablename":       "APIKeyTable1",
-			"versiontablename":      "Version",
+			"name":                   "dynamodb",
+			"endpoint":               os.Getenv("DYNAMODBMOCK_ENDPOINT"),
+			"region":                 "us-east-2",
+			"cachetablename":         "test",
+			"repometatablename":      "RepoMetadataTable",
+			"imagedatatablename":     "ZotImageDataTable",
+			"repoblobsinfotablename": "ZotRepoBlobsInfoTable",
+			"userdatatablename":      "ZotUserDataTable",
+			"apikeytablename":        "APIKeyTable1",
+			"versiontablename":       "Version",
 		}
 
 		mockOIDCServer, err := authutils.MockOIDCRun()
@@ -7715,6 +7720,8 @@ func TestInjectTooManyOpenFiles(t *testing.T) {
 }
 
 func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) {
+	ctx := context.Background()
+
 	Convey("Make controller", t, func() {
 		Convey("Garbage collect signatures without subject and manifests without tags", func(c C) {
 			repoName := "testrepo" //nolint:goconst
@@ -7746,7 +7753,7 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) {
 			ctlr.Config.Storage.Dedupe = false
 
 			cm := test.NewControllerManager(ctlr)
-			cm.StartServer()
+			cm.StartServer() //nolint: contextcheck
 			cm.WaitServerToBeReady(port)
 			defer cm.StopServer()
 
@@ -7777,7 +7784,7 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) {
 
 			// generate a keypair
 			os.Setenv("COSIGN_PASSWORD", "")
-			err = generate.GenerateKeyPairCmd(context.TODO(), "", "cosign", nil)
+			err = generate.GenerateKeyPairCmd(ctx, "", "cosign", nil)
 			So(err, ShouldBeNil)
 
 			image := fmt.Sprintf("localhost:%s/%s@%s", port, repoName, digest.String())
@@ -7806,7 +7813,7 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) {
 			So(err, ShouldBeNil)
 
 			// sign the image
-			err = signature.SignWithNotation("good", image, tdir, true)
+			err = signature.SignWithNotation("good", image, tdir, true) //nolint: contextcheck
 			So(err, ShouldBeNil)
 
 			// get cosign signature manifest
@@ -7836,7 +7843,7 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) {
 			So(err, ShouldBeNil)
 
 			// make sure both signatures are stored in repodb
-			repoMeta, err := ctlr.MetaDB.GetRepoMeta(repoName)
+			repoMeta, err := ctlr.MetaDB.GetRepoMeta(ctx, repoName)
 			So(err, ShouldBeNil)
 
 			sigMeta := repoMeta.Signatures[digest.String()]
@@ -7897,7 +7904,7 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) {
 				So(err, ShouldBeNil)
 
 				// make sure repoDB reference was added
-				repoMeta, err := ctlr.MetaDB.GetRepoMeta(repoName)
+				repoMeta, err := ctlr.MetaDB.GetRepoMeta(ctx, repoName)
 				So(err, ShouldBeNil)
 
 				_, ok := repoMeta.Referrers[untaggedManifestDigest.String()]
@@ -7927,7 +7934,7 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) {
 				So(err, ShouldBeNil)
 
 				// make sure both signatures are removed from repodb and repo reference for untagged is removed
-				repoMeta, err = ctlr.MetaDB.GetRepoMeta(repoName)
+				repoMeta, err = ctlr.MetaDB.GetRepoMeta(ctx, repoName)
 				So(err, ShouldBeNil)
 
 				sigMeta := repoMeta.Signatures[digest.String()]

diff --git a/pkg/api/routes.go b/pkg/api/routes.go
@@ -516,7 +516,7 @@ func (rh *RouteHandler) GetManifest(response http.ResponseWriter, request *http.
 	}
 
 	if rh.c.MetaDB != nil {
-		err := meta.OnGetManifest(name, reference, content, rh.c.StoreController, rh.c.MetaDB, rh.c.Log)
+		err := meta.OnGetManifest(name, reference, mediaType, content, rh.c.StoreController, rh.c.MetaDB, rh.c.Log)
 		if err != nil {
 			response.WriteHeader(http.StatusInternalServerError)