refactor(scrub): replace umoci logic in scrub implementation

- implement scrub also for S3 storage by replacing umoci
- change scrub implementation for ImageIndex
- take the `Subject` into consideration when running scrub
- remove test code relying on the umoci library. Since we started
relying on images in test/data, and we create our own images using
go code we can obtain digests by other means. (cherry picked from commit 489d4e2)

Signed-off-by: Andrei Aaron <[email protected]>
Signed-off-by: Andreea-Lupu <[email protected]>

errors/errors.go

@@ -50,6 +50,7 @@ var (
 	ErrCliBadConfig                   = errors.New("cli: bad config")
 	ErrRepoNotFound                   = errors.New("repository: not found")
 	ErrRepoBadVersion                 = errors.New("repository: unsupported layout version")
+	ErrRepoBadLayout                  = errors.New("repository: invalid layout")
 	ErrManifestNotFound               = errors.New("manifest: not found")
 	ErrBadManifest                    = errors.New("manifest: invalid contents")
 	ErrUploadNotFound                 = errors.New("uploads: not found")

go.mod

@@ -5,7 +5,6 @@ go 1.20
 require (
 	github.com/99designs/gqlgen v0.17.38
 	github.com/Masterminds/semver v1.5.0
-	github.com/apex/log v1.9.0 // indirect
 	github.com/aquasecurity/trivy-db v0.0.0-20230831170347-f732860d4917
 	github.com/bmatcuk/doublestar/v4 v4.6.0
 	github.com/briandowns/spinner v1.23.0
@@ -27,7 +26,6 @@ require (
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.1.0-rc5
-	github.com/opencontainers/umoci v0.4.8-0.20210922062158-e60a0cc726e6
 	github.com/oras-project/artifacts-spec v1.0.0-rc.2
 	github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5
 	github.com/prometheus/client_golang v1.16.0
@@ -439,7 +437,6 @@ require (
 	github.com/protocolbuffers/txtpbfmt v0.0.0-20230328191034-3462fbc510c0 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
-	github.com/rootless-containers/proto/go-proto v0.0.0-20210921234734-69430b6543fb // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/saracen/walker v0.1.3 // indirect
 	github.com/sassoftware/relic v7.2.1+incompatible // indirect
@@ -471,8 +468,6 @@ require (
 	github.com/transparency-dev/merkle v0.0.2 // indirect
 	github.com/twitchtv/twirp v8.1.2+incompatible // indirect
 	github.com/ulikunitz/xz v0.5.11 // indirect
-	github.com/urfave/cli v1.22.14 // indirect
-	github.com/vbatts/go-mtree v0.5.2 // indirect
 	github.com/vbatts/tar-split v0.11.5 // indirect
 	github.com/xanzy/go-gitlab v0.90.0 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect

pkg/api/controller_test.go

@@ -4350,7 +4350,7 @@ func TestInvalidCases(t *testing.T) {
 			panic(err)
 		}
 
-		digest := test.GetTestBlobDigest("zot-cve-test", "config").String()
+		digest := godigest.FromString("dummy").String()
 		name := "zot-c-test"
 
 		client := resty.New()
@@ -4449,7 +4449,8 @@ func TestCrossRepoMount(t *testing.T) {
 		ctlr.Config.Storage.RemoteCache = false
 		ctlr.Config.Storage.Dedupe = false
 
-		err := test.WriteImageToFileSystem(CreateDefaultImage(), "zot-cve-test", "test", storage.StoreController{
+		image := CreateDefaultImage()
+		err := test.WriteImageToFileSystem(image, "zot-cve-test", "test", storage.StoreController{
 			DefaultStore: test.GetDefaultImageStore(dir, ctlr.Log),
 		})
 		So(err, ShouldBeNil)
@@ -4459,8 +4460,7 @@ func TestCrossRepoMount(t *testing.T) {
 
 		params := make(map[string]string)
 
-		var manifestDigest godigest.Digest
-		manifestDigest, _, _ = test.GetOciLayoutDigests(path.Join(dir, "zot-cve-test"))
+		manifestDigest := image.ManifestDescriptor.Digest
 
 		dgst := manifestDigest
 		name := "zot-cve-test"
@@ -4487,7 +4487,7 @@ func TestCrossRepoMount(t *testing.T) {
 			baseURL, constants.RoutePrefix, constants.Blobs, constants.Uploads))
 
 		incorrectParams := make(map[string]string)
-		incorrectParams["mount"] = test.GetTestBlobDigest("zot-cve-test", "manifest").String()
+		incorrectParams["mount"] = godigest.FromString("dummy").String()
 		incorrectParams["from"] = "zot-x-test"
 
 		postResponse, err = client.R().

pkg/extensions/extension_scrub.go

@@ -28,25 +28,19 @@ func EnableScrubExtension(config *config.Config, log log.Logger, storeController
 			log.Warn().Msg("Scrub interval set to too-short interval < 2h, changing scrub duration to 2 hours and continuing.") //nolint:lll // gofumpt conflicts with lll
 		}
 
-		// is local imagestore (because of umoci dependency which works only locally)
-		if config.Storage.StorageDriver == nil {
-			generator := &taskGenerator{
-				imgStore: storeController.DefaultStore,
-				log:      log,
-			}
-			sch.SubmitGenerator(generator, config.Extensions.Scrub.Interval, scheduler.LowPriority)
+		generator := &taskGenerator{
+			imgStore: storeController.DefaultStore,
+			log:      log,
 		}
+		sch.SubmitGenerator(generator, config.Extensions.Scrub.Interval, scheduler.LowPriority)
 
 		if config.Storage.SubPaths != nil {
 			for route := range config.Storage.SubPaths {
-				// is local imagestore (because of umoci dependency which works only locally)
-				if config.Storage.SubPaths[route].StorageDriver == nil {
-					generator := &taskGenerator{
-						imgStore: storeController.SubStore[route],
-						log:      log,
-					}
-					sch.SubmitGenerator(generator, config.Extensions.Scrub.Interval, scheduler.LowPriority)
+				generator := &taskGenerator{
+					imgStore: storeController.SubStore[route],
+					log:      log,
 				}
+				sch.SubmitGenerator(generator, config.Extensions.Scrub.Interval, scheduler.LowPriority)
 			}
 		}
 	} else {

pkg/extensions/scrub/scrub.go

@@ -39,6 +39,7 @@ func RunScrubRepo(ctx context.Context, imgStore storageTypes.ImageStore, repo st
 				Str("image", result.ImageName).
 				Str("tag", result.Tag).
 				Str("status", result.Status).
+				Str("affected blob", result.AffectedBlob).
 				Str("error", result.Error).
 				Msg("scrub: blobs/manifest affected")
 		}

pkg/extensions/search/search_test.go

@@ -381,19 +381,21 @@ func TestRepoListWithNewestImage(t *testing.T) {
 		ctlrManager.StartAndWait(port)
 		defer ctlrManager.StopServer()
 
-		config, layers, manifest, err := GetImageComponents(100)
+		config, layers, _, err := GetImageComponents(100)
 		So(err, ShouldBeNil)
 
-		err = UploadImage(Image{Manifest: manifest, Config: config, Layers: layers}, baseURL, "zot-cve-test", "0.0.1")
+		uploadedImage := CreateImageWith().LayerBlobs(layers).ImageConfig(config).Build()
+
+		err = UploadImage(uploadedImage, baseURL, "zot-cve-test", "0.0.1")
 		So(err, ShouldBeNil)
 
-		err = UploadImage(Image{Manifest: manifest, Config: config, Layers: layers}, baseURL, "a/zot-cve-test", "0.0.1")
+		err = UploadImage(uploadedImage, baseURL, "a/zot-cve-test", "0.0.1")
 		So(err, ShouldBeNil)
 
-		err = UploadImage(Image{Manifest: manifest, Config: config, Layers: layers}, baseURL, "zot-test", "0.0.1")
+		err = UploadImage(uploadedImage, baseURL, "zot-test", "0.0.1")
 		So(err, ShouldBeNil)
 
-		err = UploadImage(Image{Manifest: manifest, Config: config, Layers: layers}, baseURL, "a/zot-test", "0.0.1")
+		err = UploadImage(uploadedImage, baseURL, "a/zot-test", "0.0.1")
 		So(err, ShouldBeNil)
 
 		resp, err := resty.R().Get(baseURL + "/v2/")
@@ -590,9 +592,8 @@ func TestRepoListWithNewestImage(t *testing.T) {
 				panic(err)
 			}
 
-			var manifestDigest godigest.Digest
-			var configDigest godigest.Digest
-			manifestDigest, configDigest, _ = GetOciLayoutDigests(path.Join(subRootDir, "a/zot-test"))
+			manifestDigest := uploadedImage.ManifestDescriptor.Digest
+			configDigest := uploadedImage.ConfigDescriptor.Digest
 
 			// Delete config blob and try.
 			err = os.Remove(path.Join(subRootDir, "a/zot-test/blobs/sha256", configDigest.Encoded()))
@@ -1269,22 +1270,25 @@ func TestExpandedRepoInfo(t *testing.T) {
 		ctlrManager.StartAndWait(port)
 		defer ctlrManager.StopServer()
 
-		config, layers, manifest, err := GetImageComponents(100)
+		config, layers, _, err := GetImageComponents(100)
 		So(err, ShouldBeNil)
 
-		manifest.Annotations = make(map[string]string)
-		manifest.Annotations["org.opencontainers.image.vendor"] = "zot"
+		annotations := make(map[string]string)
+		annotations["org.opencontainers.image.vendor"] = "zot"
 
-		err = UploadImage(Image{Manifest: manifest, Config: config, Layers: layers}, baseURL, "zot-cve-test", "0.0.1")
+		uploadedImage := CreateImageWith().LayerBlobs(layers).ImageConfig(config).
+			Annotations(annotations).Build()
+
+		err = UploadImage(uploadedImage, baseURL, "zot-cve-test", "0.0.1")
 		So(err, ShouldBeNil)
 
-		err = UploadImage(Image{Manifest: manifest, Config: config, Layers: layers}, baseURL, "a/zot-cve-test", "0.0.1")
+		err = UploadImage(uploadedImage, baseURL, "a/zot-cve-test", "0.0.1")
 		So(err, ShouldBeNil)
 
-		err = UploadImage(Image{Manifest: manifest, Config: config, Layers: layers}, baseURL, "zot-test", "0.0.1")
+		err = UploadImage(uploadedImage, baseURL, "zot-test", "0.0.1")
 		So(err, ShouldBeNil)
 
-		err = UploadImage(Image{Manifest: manifest, Config: config, Layers: layers}, baseURL, "a/zot-test", "0.0.1")
+		err = UploadImage(uploadedImage, baseURL, "a/zot-test", "0.0.1")
 		So(err, ShouldBeNil)
 
 		log := log.NewLogger("debug", "")
@@ -1455,8 +1459,7 @@ func TestExpandedRepoInfo(t *testing.T) {
 		}
 		So(found, ShouldEqual, true)
 
-		var manifestDigest godigest.Digest
-		manifestDigest, _, _ = GetOciLayoutDigests(path.Join(rootDir, "zot-test"))
+		manifestDigest := uploadedImage.ManifestDescriptor.Digest
 
 		err = os.Remove(path.Join(rootDir, "zot-test/blobs/sha256", manifestDigest.Encoded()))
 		So(err, ShouldBeNil)