feat: support copying referrers for multi-arch images (#1122)

Signed-off-by: Billy Zha <[email protected]>

cmd/oras/root/cp.go

@@ -17,6 +17,7 @@ package root
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"strings"
 	"sync"
@@ -28,6 +29,7 @@ import (
 	"oras.land/oras-go/v2/content"
 	"oras.land/oras/cmd/oras/internal/display"
 	"oras.land/oras/cmd/oras/internal/option"
+	"oras.land/oras/internal/docker"
 	"oras.land/oras/internal/graph"
 )
 
@@ -137,36 +139,27 @@ func runCopy(ctx context.Context, opts copyOptions) error {
 	var desc ocispec.Descriptor
 	rOpts := oras.DefaultResolveOptions
 	rOpts.TargetPlatform = opts.Platform.Platform
-	if dstRef := opts.To.Reference; dstRef == "" {
+	if opts.recursive {
 		desc, err = oras.Resolve(ctx, src, opts.From.Reference, rOpts)
 		if err != nil {
 			return fmt.Errorf("failed to resolve %s: %w", opts.From.Reference, err)
 		}
-		if opts.recursive {
-			err = oras.ExtendedCopyGraph(ctx, src, dst, desc, extendedCopyOptions.ExtendedCopyGraphOptions)
-		} else {
-			err = oras.CopyGraph(ctx, src, dst, desc, extendedCopyOptions.CopyGraphOptions)
-		}
+		err = recursiveCopy(ctx, src, dst, opts.To.Reference, desc, extendedCopyOptions)
 	} else {
-		if opts.recursive {
-			srcRef := opts.From.Reference
-			if rOpts.TargetPlatform != nil {
-				// resolve source reference to specified platform
-				desc, err := oras.Resolve(ctx, src, opts.From.Reference, rOpts)
-				if err != nil {
-					return fmt.Errorf("failed to resolve %s: %w", opts.From.Reference, err)
-				}
-				srcRef = desc.Digest.String()
+		if opts.To.Reference == "" {
+			desc, err = oras.Resolve(ctx, src, opts.From.Reference, rOpts)
+			if err != nil {
+				return fmt.Errorf("failed to resolve %s: %w", opts.From.Reference, err)
 			}
-			desc, err = oras.ExtendedCopy(ctx, src, srcRef, dst, dstRef, extendedCopyOptions)
+			err = oras.CopyGraph(ctx, src, dst, desc, extendedCopyOptions.CopyGraphOptions)
 		} else {
 			copyOptions := oras.CopyOptions{
 				CopyGraphOptions: extendedCopyOptions.CopyGraphOptions,
 			}
 			if opts.Platform.Platform != nil {
 				copyOptions.WithTargetPlatform(opts.Platform.Platform)
 			}
-			desc, err = oras.Copy(ctx, src, opts.From.Reference, dst, dstRef, copyOptions)
+			desc, err = oras.Copy(ctx, src, opts.From.Reference, dst, opts.To.Reference, copyOptions)
 		}
 	}
 	if err != nil {
@@ -191,3 +184,44 @@ func runCopy(ctx context.Context, opts copyOptions) error {
 
 	return nil
 }
+
+// recursiveCopy copies an artifact and its referrers from one target to another.
+// If the artifact is a manifest list or index, referrers of its manifests are copied as well.
+func recursiveCopy(ctx context.Context, src oras.ReadOnlyGraphTarget, dst oras.Target, dstRef string, root ocispec.Descriptor, opts oras.ExtendedCopyOptions) error {
+	if root.MediaType == ocispec.MediaTypeImageIndex || root.MediaType == docker.MediaTypeManifestList {
+		fetched, err := content.FetchAll(ctx, src, root)
+		if err != nil {
+			return err
+		}
+		var index ocispec.Index
+		if err = json.Unmarshal(fetched, &index); err != nil {
+			return nil
+		}
+
+		referrers, err := graph.FindPredecessors(ctx, src, index.Manifests, opts)
+		if err != nil {
+			return err
+		}
+
+		findPredecessor := opts.FindPredecessors
+		opts.FindPredecessors = func(ctx context.Context, src content.ReadOnlyGraphStorage, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
+			descs, err := findPredecessor(ctx, src, desc)
+			if err != nil {
+				return nil, err
+			}
+			if content.Equal(desc, root) {
+				// make sure referrers of child manifests are copied by pointing them to root
+				descs = append(descs, referrers...)
+			}
+			return descs, nil
+		}
+	}
+
+	var err error
+	if dstRef == "" || dstRef == root.Digest.String() {
+		err = oras.ExtendedCopyGraph(ctx, src, dst, root, opts.ExtendedCopyGraphOptions)
+	} else {
+		_, err = oras.ExtendedCopy(ctx, src, root.Digest.String(), dst, dstRef, opts)
+	}
+	return err
+}

internal/docker/mediatype.go

@@ -17,5 +17,6 @@ package docker
 
 // docker media types
 const (
-	MediaTypeManifest = "application/vnd.docker.distribution.manifest.v2+json"
+	MediaTypeManifest     = "application/vnd.docker.distribution.manifest.v2+json"
+	MediaTypeManifestList = "application/vnd.docker.distribution.manifest.list.v2+json"
 )

internal/graph/graph.go

@@ -18,8 +18,11 @@ package graph
 import (
 	"context"
 	"encoding/json"
+	"sync"
 
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"golang.org/x/sync/errgroup"
+	"oras.land/oras-go/v2"
 	"oras.land/oras-go/v2/content"
 	"oras.land/oras-go/v2/registry"
 	"oras.land/oras/internal/docker"
@@ -188,3 +191,31 @@ func fetchBytes(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descr
 	defer rc.Close()
 	return content.ReadAll(rc, desc)
 }
+
+// FindPredecessors returns all predecessors of descs in src concurrently.
+func FindPredecessors(ctx context.Context, src oras.ReadOnlyGraphTarget, descs []ocispec.Descriptor, opts oras.ExtendedCopyOptions) ([]ocispec.Descriptor, error) {
+	var referrers []ocispec.Descriptor
+	g, ctx := errgroup.WithContext(ctx)
+	var m sync.Mutex
+	if opts.Concurrency != 0 {
+		g.SetLimit(opts.Concurrency)
+	}
+	for _, desc := range descs {
+		g.Go(func(node ocispec.Descriptor) func() error {
+			return func() error {
+				descs, err := opts.FindPredecessors(ctx, src, node)
+				if err != nil {
+					return err
+				}
+				m.Lock()
+				defer m.Unlock()
+				referrers = append(referrers, descs...)
+				return nil
+			}
+		}(desc))
+	}
+	if err := g.Wait(); err != nil {
+		return nil, err
+	}
+	return referrers, nil
+}

test/e2e/internal/testdata/multi_arch/const.go

@@ -23,6 +23,7 @@ import (
 
 var (
 	Tag                         = "multi"
+	EmptyTag                    = "empty_index"
 	Digest                      = "sha256:e2bfc9cc6a84ec2d7365b5a28c6bc5806b7fa581c9ad7883be955a64e3cc034f"
 	Manifest                    = `{"mediaType":"application/vnd.oci.image.index.v1+json","schemaVersion":2,"manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:9d84a5716c66a1d1b9c13f8ed157ba7d1edfe7f9b8766728b8a1f25c0d9c14c1","size":458,"platform":{"architecture":"amd64","os":"linux"}},{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:4f93460061882467e6fb3b772dc6ab72130d9ac1906aed2fc7589a5cd145433c","size":458,"platform":{"architecture":"arm64","os":"linux"}},{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:58efe73e78fe043ca31b89007a025c594ce12aa7e6da27d21c7b14b50112e255","size":458,"platform":{"architecture":"arm","os":"linux","variant":"v7"}}]}`
 	Descriptor                  = `{"mediaType":"application/vnd.oci.image.index.v1+json","digest":"sha256:e2bfc9cc6a84ec2d7365b5a28c6bc5806b7fa581c9ad7883be955a64e3cc034f","size":706}`

test/e2e/suite/command/cp.go

@@ -115,15 +115,15 @@ var _ = Describe("1.1 registry users:", func() {
 		})
 
 		It("should copy an image and its referrers to a new repository", func() {
-			stateKeys := append(append(foobarStates, foobar.ImageReferrersStateKeys...), foobar.ImageReferrerConfigStateKeys...)
+			stateKeys := append(append(foobar.ImageLayerStateKeys, foobar.ManifestStateKey, foobar.ImageReferrerConfigStateKeys[0]), foobar.ImageReferrersStateKeys...)
 			src := RegistryRef(ZOTHost, ArtifactRepo, foobar.Tag)
 			dst := RegistryRef(ZOTHost, cpTestRepo("referrers"), foobar.Digest)
 			ORAS("cp", "-r", src, dst, "-v").MatchStatus(stateKeys, true, len(stateKeys)).Exec()
 			CompareRef(src, dst)
 		})
 
 		It("should copy a multi-arch image and its referrers to a new repository via tag", func() {
-			stateKeys := append(ma.IndexStateKeys, ma.IndexZOTReferrerStateKey, ma.IndexReferrerConfigStateKey)
+			stateKeys := append(ma.IndexStateKeys, ma.IndexZOTReferrerStateKey, ma.LinuxAMD64ReferrerConfigStateKey)
 			src := RegistryRef(ZOTHost, ArtifactRepo, ma.Tag)
 			dstRepo := cpTestRepo("index-referrers")
 			dst := RegistryRef(ZOTHost, dstRepo, "copiedTag")
@@ -142,13 +142,47 @@ var _ = Describe("1.1 registry users:", func() {
 			Expect(len(index.Manifests)).To(Equal(1))
 			Expect(index.Manifests[0].Digest.String()).To(Equal(ma.IndexReferrerDigest))
 			ORAS("manifest", "fetch", RegistryRef(ZOTHost, dstRepo, ma.LinuxAMD64Referrer.Digest.String())).
-				WithDescription("not copy referrer of successor").
-				ExpectFailure().
+				WithDescription("copy referrer of successor").
+				Exec()
+		})
+
+		It("should copy a multi-arch image and its referrers without concurrency limitation", func() {
+			stateKeys := append(ma.IndexStateKeys, ma.IndexZOTReferrerStateKey, ma.LinuxAMD64ReferrerConfigStateKey)
+			src := RegistryRef(ZOTHost, ArtifactRepo, ma.Tag)
+			dstRepo := cpTestRepo("index-referrers-concurrent")
+			dst := RegistryRef(ZOTHost, dstRepo, "copiedTag")
+			// test
+			ORAS("cp", src, dst, "-r", "-v", "--concurrency", "0").
+				MatchStatus(stateKeys, true, len(stateKeys)).
+				MatchKeyWords("Digest: " + ma.Digest).
+				Exec()
+			// validate
+			CompareRef(RegistryRef(ZOTHost, ImageRepo, ma.Digest), dst)
+			var index ocispec.Index
+			bytes := ORAS("discover", dst, "-o", "json", "--artifact-type", ma.IndexReferrerConfigStateKey.Name).
+				MatchKeyWords(ma.IndexReferrerDigest).
+				WithDescription("copy image referrer").
+				Exec().Out.Contents()
+			Expect(json.Unmarshal(bytes, &index)).ShouldNot(HaveOccurred())
+			Expect(len(index.Manifests)).To(Equal(1))
+			Expect(index.Manifests[0].Digest.String()).To(Equal(ma.IndexReferrerDigest))
+			ORAS("manifest", "fetch", RegistryRef(ZOTHost, dstRepo, ma.LinuxAMD64Referrer.Digest.String())).
+				WithDescription("copy referrer of successor").
 				Exec()
 		})
 
+		It("should copy an empty index", func() {
+			src := RegistryRef(ZOTHost, ImageRepo, ma.EmptyTag)
+			dstRepo := cpTestRepo("empty-index")
+			dst := RegistryRef(ZOTHost, dstRepo, "copiedTag")
+			// test
+			ORAS("cp", src, dst, "-r", "-v", "--concurrency", "0").Exec()
+			// validate
+			CompareRef(RegistryRef(ZOTHost, ImageRepo, ma.EmptyTag), dst)
+		})
+
 		It("should copy a multi-arch image and its referrers to a new repository via digest", func() {
-			stateKeys := append(ma.IndexStateKeys, ma.IndexZOTReferrerStateKey, ma.IndexReferrerConfigStateKey)
+			stateKeys := append(ma.IndexStateKeys, ma.IndexZOTReferrerStateKey, ma.LinuxAMD64ReferrerConfigStateKey)
 			src := RegistryRef(ZOTHost, ArtifactRepo, ma.Tag)
 			dstRepo := cpTestRepo("index-referrers-digest")
 			dst := RegistryRef(ZOTHost, dstRepo, ma.Digest)
@@ -168,7 +202,6 @@ var _ = Describe("1.1 registry users:", func() {
 			Expect(index.Manifests[0].Digest.String()).To(Equal(ma.IndexReferrerDigest))
 			ORAS("manifest", "fetch", RegistryRef(ZOTHost, dstRepo, ma.LinuxAMD64Referrer.Digest.String())).
 				WithDescription("not copy referrer of successor").
-				ExpectFailure().
 				Exec()
 		})
 
@@ -270,7 +303,7 @@ var _ = Describe("OCI spec 1.0 registry users:", func() {
 	When("running `cp`", func() {
 		It("should copy an image artifact and its referrers from a registry to a fallback registry", func() {
 			repo := cpTestRepo("to-fallback")
-			stateKeys := append(append(foobarStates, foobar.ImageReferrersStateKeys...), foobar.ImageReferrerConfigStateKeys...)
+			stateKeys := append(append(foobar.ImageLayerStateKeys, foobar.ManifestStateKey, foobar.ImageReferrerConfigStateKeys[0]), foobar.ImageReferrersStateKeys...)
 			src := RegistryRef(ZOTHost, ArtifactRepo, foobar.SignatureImageReferrer.Digest.String())
 			dst := RegistryRef(FallbackHost, repo, "")
 			ORAS("cp", "-r", src, dst, "-v").MatchStatus(stateKeys, true, len(stateKeys)).Exec()
@@ -280,7 +313,7 @@ var _ = Describe("OCI spec 1.0 registry users:", func() {
 		})
 		It("should copy an image artifact and its referrers from a fallback registry to a registry", func() {
 			repo := cpTestRepo("from-fallback")
-			stateKeys := append(append(foobarStates, foobar.ImageReferrersStateKeys...), foobar.ImageReferrerConfigStateKeys...)
+			stateKeys := append(append(foobar.ImageLayerStateKeys, foobar.ManifestStateKey, foobar.ImageReferrerConfigStateKeys[0]), foobar.ImageReferrersStateKeys...)
 			src := RegistryRef(FallbackHost, ArtifactRepo, foobar.SBOMImageReferrer.Digest.String())
 			dst := RegistryRef(ZOTHost, repo, "")
 			ORAS("cp", "-r", src, dst, "-v").MatchStatus(stateKeys, true, len(stateKeys)).Exec()
@@ -439,7 +472,7 @@ var _ = Describe("OCI layout users:", func() {
 		})
 
 		It("should copy a tagged image and its referrers from a registry to an OCI image layout", func() {
-			stateKeys := append(append(foobarStates, foobar.ImageReferrersStateKeys...), foobar.ImageReferrerConfigStateKeys...)
+			stateKeys := append(append(foobar.ImageLayerStateKeys, foobar.ManifestStateKey, foobar.ImageReferrerConfigStateKeys[0]), foobar.ImageReferrersStateKeys...)
 			dst := LayoutRef(GinkgoT().TempDir(), "copied")
 			src := RegistryRef(ZOTHost, ArtifactRepo, foobar.Tag)
 			// test
@@ -451,7 +484,7 @@ var _ = Describe("OCI layout users:", func() {
 		})
 
 		It("should copy a image and its referrers from a registry to an OCI image layout via digest", func() {
-			stateKeys := append(append(foobarStates, foobar.ImageReferrersStateKeys...), foobar.ImageReferrerConfigStateKeys...)
+			stateKeys := append(append(foobar.ImageLayerStateKeys, foobar.ManifestStateKey, foobar.ImageReferrerConfigStateKeys[0]), foobar.ImageReferrersStateKeys...)
 			toDir := GinkgoT().TempDir()
 			src := RegistryRef(ZOTHost, ArtifactRepo, foobar.Digest)
 			// test
@@ -463,7 +496,7 @@ var _ = Describe("OCI layout users:", func() {
 		})
 
 		It("should copy a multi-arch image and its referrers from a registry to an OCI image layout a via tag", func() {
-			stateKeys := append(ma.IndexStateKeys, ma.IndexZOTReferrerStateKey, ma.IndexReferrerConfigStateKey)
+			stateKeys := append(ma.IndexStateKeys, ma.IndexZOTReferrerStateKey, ma.LinuxAMD64ReferrerConfigStateKey)
 			src := RegistryRef(ZOTHost, ArtifactRepo, ma.Tag)
 			toDir := GinkgoT().TempDir()
 			dst := LayoutRef(toDir, "copied")
@@ -485,13 +518,12 @@ var _ = Describe("OCI layout users:", func() {
 			Expect(len(index.Manifests)).To(Equal(1))
 			Expect(index.Manifests[0].Digest.String()).To(Equal(ma.IndexReferrerDigest))
 			ORAS("manifest", "fetch", Flags.Layout, LayoutRef(toDir, ma.LinuxAMD64Referrer.Digest.String())).
-				WithDescription("not copy referrer of successor").
-				ExpectFailure().
+				WithDescription("copy referrer of successor").
 				Exec()
 		})
 
 		It("should copy a multi-arch image and its referrers from an OCI image layout to a registry via digest", func() {
-			stateKeys := append(ma.IndexStateKeys, ma.IndexZOTReferrerStateKey, ma.IndexReferrerConfigStateKey)
+			stateKeys := append(ma.IndexStateKeys, ma.IndexZOTReferrerStateKey, ma.LinuxAMD64ReferrerConfigStateKey)
 			fromDir := GinkgoT().TempDir()
 			src := LayoutRef(fromDir, ma.Tag)
 			dst := RegistryRef(ZOTHost, cpTestRepo("recursive-from-layout"), "copied")
@@ -514,9 +546,8 @@ var _ = Describe("OCI layout users:", func() {
 			Expect(json.Unmarshal(bytes, &index)).ShouldNot(HaveOccurred())
 			Expect(len(index.Manifests)).To(Equal(1))
 			Expect(index.Manifests[0].Digest.String()).To(Equal(ma.IndexReferrerDigest))
-			ORAS("manifest", "fetch", LayoutRef(fromDir, ma.LinuxAMD64Referrer.Digest.String())).
-				WithDescription("not copy referrer of successor").
-				ExpectFailure().
+			ORAS("manifest", "fetch", dst).
+				WithDescription("copy referrer of successor").
 				Exec()
 		})
 

test/e2e/suite/command/pull.go

@@ -122,7 +122,7 @@ var _ = Describe("OCI spec 1.1 registry users:", func() {
 
 		It("should copy an artifact with blob", func() {
 			repo := cpTestRepo("artifact-with-blob")
-			stateKeys := append(append(foobarStates, foobar.ImageReferrersStateKeys...), foobar.ImageReferrerConfigStateKeys...)
+			stateKeys := append(append(foobar.ImageLayerStateKeys, foobar.ManifestStateKey, foobar.ImageReferrerConfigStateKeys[0]), foobar.ImageReferrersStateKeys...)
 			src := RegistryRef(ZOTHost, ArtifactRepo, foobar.SignatureImageReferrer.Digest.String())
 			dst := RegistryRef(FallbackHost, repo, "")
 			ORAS("cp", "-r", src, dst, "-v").MatchStatus(stateKeys, true, len(stateKeys)).Exec()

test/e2e/testdata/zot/command/images/blobs/sha256/b2a5fcfb112ccde647a5a3dc0215c2c9e7d0ce598924a5ec48aa85beca048286

@@ -0,0 +1 @@
+{"mediaType":"application/vnd.oci.image.index.v1+json","schemaVersion":2,"manifests":[]}

test/e2e/testdata/zot/command/images/index.json

@@ -44,6 +44,14 @@
 				"architecture": "amd64",
 				"os": "linux"
 			}
+		},
+		{
+			"mediaType": "application/vnd.oci.image.index.v1+json",
+			"digest": "sha256:b2a5fcfb112ccde647a5a3dc0215c2c9e7d0ce598924a5ec48aa85beca048286",
+			"size": 89,
+			"annotations": {
+				"org.opencontainers.image.ref.name": "empty_index"
+			}
 		}
 	]
 }