feat: improve UX by speeding up metadb serialize/deserialize

Use protocol buffers and update the metadb interface to better suit our search needs Signed-off-by: Ramkumar Chinchani <[email protected]> Signed-off-by: Laurentiu Niculae <[email protected]>
project-zot · Oct 24, 2023 · 8d27fbd · 8d27fbd
1 parent a3d8202
commit 8d27fbd
Show file tree

Hide file tree

Showing 40 changed files with 9,125 additions and 213 deletions.
diff --git a/.github/workflows/golangci-lint.yaml b/.github/workflows/golangci-lint.yaml
@@ -38,6 +38,7 @@ jobs:
 
           # Optional: if set to true then the action don't cache or restore ~/go/pkg.
           # skip-pkg-cache: true
+          skip-pkg-cache: false
 
           # Optional: if set to true then the action don't cache or restore ~/.cache/go-build.
           # skip-build-cache: true

diff --git a/Makefile b/Makefile
@@ -28,6 +28,8 @@ ZUI_VERSION := commit-fad5572
 SWAGGER_VERSION := v1.8.12
 STACKER := $(TOOLSDIR)/bin/stacker
 BATS := $(TOOLSDIR)/bin/bats
+PROTOC_VERSION := 3.15.8
+PROTOC := $(TOOLSDIR)/bin/protoc
 TESTDATA := $(TOP_LEVEL)/test/data
 OS ?= $(shell go env GOOS)
 ARCH ?= $(shell go env GOARCH)
@@ -71,7 +73,7 @@ modtidy:
 	go mod tidy
 
 .PHONY: modcheck
-modcheck: modtidy
+modcheck: gen-protobuf modtidy
 	$(eval UNCOMMITED_FILES = $(shell git status --porcelain | grep -c 'go.mod\|go.sum'))
 	@if [ $(UNCOMMITED_FILES) != 0 ]; then \
 		echo "Updated go.mod and/or go.sum have uncommitted changes, commit the changes accordingly ";\
@@ -97,6 +99,47 @@ build-metadata: $(if $(findstring ui,$(BUILD_LABELS)), ui)
 	echo "\n Files: \n"
 	go list -tags $(BUILD_TAGS) -f '{{ join .GoFiles "\n" }}' ./... | sort -u
 
+.PHONY: gen-protobuf
+gen-protobuf: $(PROTOC)
+	$(PROTOC) --experimental_allow_proto3_optional \
+		--proto_path=$(TOP_LEVEL)/pkg/meta/proto \
+		--go_out=$(TOP_LEVEL)/pkg/meta/ \
+		--go_opt='Mdescriptor.proto=./proto_go' \
+		$(TOP_LEVEL)/pkg/meta/proto/descriptor.proto
+	$(PROTOC) --experimental_allow_proto3_optional \
+		--proto_path=$(TOP_LEVEL)/pkg/meta/proto \
+		--go_out=$(TOP_LEVEL)/pkg/meta/ \
+		--go_opt='Mconfig.proto=./proto_go' \
+		--go_opt='Mdescriptor.proto=./proto_go' \
+		$(TOP_LEVEL)/pkg/meta/proto/config.proto
+	$(PROTOC) --experimental_allow_proto3_optional \
+		--proto_path=$(TOP_LEVEL)/pkg/meta/proto \
+		--go_out=$(TOP_LEVEL)/pkg/meta/ \
+		--go_opt='Mversioned.proto=./proto_go' \
+		$(TOP_LEVEL)/pkg/meta/proto/versioned.proto
+	$(PROTOC) --experimental_allow_proto3_optional \
+		--proto_path=$(TOP_LEVEL)/pkg/meta/proto \
+		--go_out=$(TOP_LEVEL)/pkg/meta/ \
+		--go_opt='Mmanifest.proto=./proto_go' \
+		--go_opt='Mdescriptor.proto=./proto_go' \
+		--go_opt='Mversioned.proto=./proto_go' \
+		$(TOP_LEVEL)/pkg/meta/proto/manifest.proto
+	$(PROTOC) --experimental_allow_proto3_optional \
+		--proto_path=$(TOP_LEVEL)/pkg/meta/proto \
+		--go_out=$(TOP_LEVEL)/pkg/meta/ \
+		--go_opt='Mindex.proto=./proto_go' \
+		--go_opt='Mdescriptor.proto=./proto_go' \
+		--go_opt='Mversioned.proto=./proto_go' \
+		$(TOP_LEVEL)/pkg/meta/proto/index.proto
+	$(PROTOC) --experimental_allow_proto3_optional \
+		--proto_path=$(TOP_LEVEL)/pkg/meta/proto \
+		--go_out=$(TOP_LEVEL)/pkg/meta/ \
+		--go_opt='MimageData.proto=./proto_go' \
+		--go_opt='Mdescriptor.proto=./proto_go' \
+		--go_opt='Mversioned.proto=./proto_go' \
+		--go_opt='Mconfig.proto=./proto_go' \
+		$(TOP_LEVEL)/pkg/meta/proto/imageData.proto
+
 .PHONY: binary-minimal
 binary-minimal: EXTENSIONS=
 binary-minimal: modcheck build-metadata
@@ -218,6 +261,12 @@ $(CRICTL):
 	mv crictl $(TOOLSDIR)/bin/crictl
 	chmod +x $(TOOLSDIR)/bin/crictl
 
+$(PROTOC):
+	mkdir -p $(TOOLSDIR)/bin
+	curl -Lo protoc.zip https://github.com/protocolbuffers/protobuf/releases/download/v$(PROTOC_VERSION)/protoc-$(PROTOC_VERSION)-linux-x86_64.zip
+	unzip -d $(TOOLSDIR) protoc.zip bin/protoc
+	chmod +x $(PROTOC)
+	go install google.golang.org/protobuf/cmd/protoc-gen-go@latest
 
 $(ACTION_VALIDATOR):
 	mkdir -p $(TOOLSDIR)/bin
@@ -242,7 +291,7 @@ $(GOLINTER):
 
 .PHONY: check
 check: $(if $(findstring ui,$(BUILD_LABELS)), ui)
-check: ./golangcilint.yaml $(GOLINTER)
+check: ./golangcilint.yaml modcheck $(GOLINTER)
 	mkdir -p pkg/extensions/build; touch pkg/extensions/build/.empty
 	$(GOLINTER) --config ./golangcilint.yaml run --enable-all --out-format=colored-line-number --build-tags containers_image_openpgp ./...
 	$(GOLINTER) --config ./golangcilint.yaml run --enable-all --out-format=colored-line-number --build-tags $(BUILD_LABELS),containers_image_openpgp  ./...
@@ -339,7 +388,7 @@ verify-config-commited: _verify-config
 	fi; \
 
 .PHONY: gqlgen
-gqlgen:
+gqlgen: gen-protobuf
 	cd pkg/extensions/search;\
 	go run github.com/99designs/gqlgen version;\
 	go run github.com/99designs/gqlgen generate

diff --git a/pkg/api/controller.go b/pkg/api/controller.go
@@ -293,6 +293,7 @@ func (c *Controller) InitMetaDB(reloadCtx context.Context) error {
 			return err
 		}
 
+		// TODO: undo, commented for testing
 		err = meta.ParseStorage(driver, c.StoreController, c.Log)
 		if err != nil {
 			return err

diff --git a/pkg/common/oci.go b/pkg/common/oci.go
@@ -6,8 +6,10 @@ import (
 
 	"github.com/opencontainers/go-digest"
 	ispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"google.golang.org/protobuf/types/known/timestamppb"
 
 	zerr "zotregistry.io/zot/errors"
+	"zotregistry.io/zot/pkg/meta/proto_go"
 )
 
 func GetImageDirAndTag(imageName string) (string, string) {
@@ -84,6 +86,24 @@ func GetImageLastUpdated(imageInfo ispec.Image) time.Time {
 	return *timeStamp
 }
 
+func GetProtoImageLastUpdated(configData *proto_go.ConfigData) time.Time {
+	timeStamp := configData.Created
+
+	if timeStamp != nil && !timeStamp.AsTime().IsZero() {
+		return timeStamp.AsTime()
+	}
+
+	if len(configData.History) > 0 {
+		timeStamp = configData.History[len(configData.History)-1].Created
+	}
+
+	if timeStamp == nil {
+		timeStamp = timestamppb.New(time.Time{})
+	}
+
+	return timeStamp.AsTime()
+}
+
 // GetRepoReference returns the components of a repoName:tag or repoName@digest string. If the format is wrong
 // an error is returned.
 // The returned values have the following meaning:

diff --git a/pkg/extensions/imagetrust/image_trust.go b/pkg/extensions/imagetrust/image_trust.go
@@ -157,19 +157,47 @@ func IsResourceExistsException(err error) bool {
 	return false
 }
 
+func (imgTrustStore *ImageTrustStore) ProtoVerifySignature(
+	signatureType string, rawSignature []byte, sigKey string, manifestDigest godigest.Digest, imageData mTypes.ImageData2,
+	repo string,
+) (string, time.Time, bool, error) {
+	desc := ispec.Descriptor{
+		MediaType: imageData.MediaType,
+		Digest:    imageData.Digest,
+		Size:      imageData.Size,
+	}
+
+	if manifestDigest.String() == "" {
+		return "", time.Time{}, false, zerr.ErrBadManifestDigest
+	}
+
+	switch signatureType {
+	case zcommon.CosignSignature:
+		author, isValid, err := VerifyCosignSignature(imgTrustStore.CosignStorage, repo, manifestDigest, sigKey, rawSignature)
+
+		return author, time.Time{}, isValid, err
+	case zcommon.NotationSignature:
+		return VerifyNotationSignature(imgTrustStore.NotationStorage, desc, manifestDigest.String(), rawSignature, sigKey)
+	default:
+		return "", time.Time{}, false, zerr.ErrInvalidSignatureType
+	}
+}
+
 func (imgTrustStore *ImageTrustStore) VerifySignature(
-	signatureType string, rawSignature []byte, sigKey string, manifestDigest godigest.Digest, manifestContent []byte,
+	signatureType string, rawSignature []byte, sigKey string, manifestDigest godigest.Digest, blob []byte,
 	repo string,
 ) (string, time.Time, bool, error) {
 	var manifest ispec.Manifest
-	if err := json.Unmarshal(manifestContent, &manifest); err != nil {
+
+	err := json.Unmarshal(blob, &manifest)
+	if err != nil {
 		return "", time.Time{}, false, err
 	}
 
 	desc := ispec.Descriptor{
 		MediaType: manifest.MediaType,
 		Digest:    manifestDigest,
-		Size:      int64(len(manifestContent)),
+		Size:      int64(len(blob)),
 	}
 
 	if manifestDigest.String() == "" {

diff --git a/pkg/extensions/imagetrust/image_trust_disabled.go b/pkg/extensions/imagetrust/image_trust_disabled.go
@@ -7,6 +7,8 @@ import (
 	"time"
 
 	godigest "github.com/opencontainers/go-digest"
+
+	mTypes "zotregistry.io/zot/pkg/meta/types"
 )
 
 func NewLocalImageTrustStore(dir string) (*imageTrustDisabled, error) {
@@ -25,3 +27,10 @@ func (imgTrustStore *imageTrustDisabled) VerifySignature(
 ) (string, time.Time, bool, error) {
 	return "", time.Time{}, false, nil
 }
+
+func (imgTrustStore *imageTrustDisabled) ProtoVerifySignature(
+	signatureType string, rawSignature []byte, sigKey string, manifestDigest godigest.Digest, imageData mTypes.ImageData2,
+	repo string,
+) (string, time.Time, bool, error) {
+	return "", time.Time{}, false, nil
+}
diff --git a/pkg/extensions/imagetrust/image_trust_test.go b/pkg/extensions/imagetrust/image_trust_test.go
@@ -150,14 +150,6 @@ func TestInitCosignAndNotationDirs(t *testing.T) {
 }
 
 func TestVerifySignatures(t *testing.T) {
-	Convey("wrong manifest content", t, func() {
-		manifestContent := []byte("wrong json")
-
-		imgTrustStore := &imagetrust.ImageTrustStore{}
-		_, _, _, err := imgTrustStore.VerifySignature("", []byte(""), "", "", manifestContent, "repo")
-		So(err, ShouldNotBeNil)
-	})
-
 	Convey("empty manifest digest", t, func() {
 		image := CreateRandomImage()
 		manifestContent := image.ManifestDescriptor.Data