Add oci/layout.PutBlobFromLocalFile

Try to reflink the file and restort to copying it in case of failure.
Also add an Options struct to be future proof.

Signed-off-by: Miloslav Trmač <[email protected]>
Signed-off-by: Valentin Rothberg <[email protected]>

internal/reflink/reflink_linux.go

@@ -0,0 +1,30 @@
+//go:build cgo
+
+package reflink
+
+/*
+#include <linux/fs.h>
+
+#ifndef FICLONE
+#define FICLONE		_IOW(0x94, 9, int)
+#endif
+*/
+import "C"
+import (
+	"io"
+	"os"
+
+	"golang.org/x/sys/unix"
+)
+
+// Copy attemts to reflink the source to the destination fd.
+// If reflinking fails or is unsupported, it falls back to io.Copy().
+func Copy(src, dst *os.File) error {
+	_, _, errno := unix.Syscall(unix.SYS_IOCTL, dst.Fd(), C.FICLONE, src.Fd())
+	if errno == 0 {
+		return nil
+	}
+
+	_, err := io.Copy(dst, src)
+	return err
+}

internal/reflink/reflink_unsupported.go

@@ -0,0 +1,15 @@
+//go:build !linux || !cgo
+
+package reflink
+
+import (
+	"io"
+	"os"
+)
+
+// Copy attemts to reflink the source to the destination fd.
+// If reflinking fails or is unsupported, it falls back to io.Copy().
+func Copy(src, dst *os.File) error {
+	_, err := io.Copy(dst, src)
+	return err
+}

oci/layout/fixtures/files/a.txt

@@ -0,0 +1 @@
+aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa

oci/layout/fixtures/files/b.txt

@@ -0,0 +1 @@
+bbbbbbbbbbbbbbbbbbbbbbbb

oci/layout/oci_dest.go

@@ -17,6 +17,7 @@ import (
 	"github.com/containers/image/v5/internal/manifest"
 	"github.com/containers/image/v5/internal/private"
 	"github.com/containers/image/v5/internal/putblobdigest"
+	"github.com/containers/image/v5/internal/reflink"
 	"github.com/containers/image/v5/types"
 	"github.com/containers/storage/pkg/fileutils"
 	digest "github.com/opencontainers/go-digest"
@@ -141,36 +142,46 @@ func (d *ociImageDestination) PutBlobWithOptions(ctx context.Context, stream io.
 	if inputInfo.Size != -1 && size != inputInfo.Size {
 		return private.UploadedBlob{}, fmt.Errorf("Size mismatch when copying %s, expected %d, got %d", blobDigest, inputInfo.Size, size)
 	}
-	if err := blobFile.Sync(); err != nil {
+
+	if err := d.blobFileSyncAndRename(blobFile, blobDigest, &explicitClosed); err != nil {
 		return private.UploadedBlob{}, err
 	}
+	succeeded = true
+	return private.UploadedBlob{Digest: blobDigest, Size: size}, nil
+}
+
+func (d *ociImageDestination) blobFileSyncAndRename(blobFile *os.File, blobDigest digest.Digest, closed *bool) error {
+	if err := blobFile.Sync(); err != nil {
+		return err
+	}
 
 	// On POSIX systems, blobFile was created with mode 0600, so we need to make it readable.
 	// On Windows, the “permissions of newly created files” argument to syscall.Open is
 	// ignored and the file is already readable; besides, blobFile.Chmod, i.e. syscall.Fchmod,
 	// always fails on Windows.
 	if runtime.GOOS != "windows" {
 		if err := blobFile.Chmod(0644); err != nil {
-			return private.UploadedBlob{}, err
+			return err
 		}
 	}
 
 	blobPath, err := d.ref.blobPath(blobDigest, d.sharedBlobDir)
 	if err != nil {
-		return private.UploadedBlob{}, err
+		return err
 	}
 	if err := ensureParentDirectoryExists(blobPath); err != nil {
-		return private.UploadedBlob{}, err
+		return err
 	}
 
-	// need to explicitly close the file, since a rename won't otherwise not work on Windows
+	// need to explicitly close the file, since a rename won't otherwise work on Windows
 	blobFile.Close()
-	explicitClosed = true
+
+	*closed = true
 	if err := os.Rename(blobFile.Name(), blobPath); err != nil {
-		return private.UploadedBlob{}, err
+		return err
 	}
-	succeeded = true
-	return private.UploadedBlob{Digest: blobDigest, Size: size}, nil
+
+	return nil
 }
 
 // TryReusingBlobWithOptions checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination
@@ -303,6 +314,63 @@ func (d *ociImageDestination) CommitWithOptions(ctx context.Context, options pri
 	return os.WriteFile(d.ref.indexPath(), indexJSON, 0644)
 }
 
+// PutBlobFromLocalFileOptions is unused but may receive functionality in the future.
+type PutBlobFromLocalFileOptions struct{}
+
+// PutBlobFromLocalFile arranges the data from path to be used as blob with digest.
+// It computes, and returns, the digest and size of the used file.
+//
+// This function can be used instead of dest.PutBlob() where the ImageDestination requires PutBlob() to be called.
+func PutBlobFromLocalFile(ctx context.Context, dest types.ImageDestination, file string, options ...PutBlobFromLocalFileOptions) (digest.Digest, int64, error) {
+	d, ok := dest.(*ociImageDestination)
+	if !ok {
+		return "", -1, errors.New("internal error: PutBlobFromLocalFile called with a non-oci: destination")
+	}
+
+	blobFileClosed := false
+	blobFile, err := os.CreateTemp(d.ref.dir, "oci-put-blob")
+	if err != nil {
+		return "", -1, err
+	}
+	defer func() {
+		if !blobFileClosed {
+			blobFile.Close()
+		}
+	}()
+
+	srcFile, err := os.Open(file)
+	if err != nil {
+		return "", -1, err
+	}
+	defer srcFile.Close()
+
+	// reflink.Copy will io.Copy() in case reflinking fails
+	err = reflink.Copy(srcFile, blobFile)
+	if err != nil {
+		return "", -1, err
+	}
+
+	_, err = blobFile.Seek(0, 0)
+	if err != nil {
+		return "", -1, err
+	}
+	blobDigest, err := digest.FromReader(blobFile)
+	if err != nil {
+		return "", -1, err
+	}
+
+	fileInfo, err := blobFile.Stat()
+	if err != nil {
+		return "", -1, err
+	}
+
+	if err := d.blobFileSyncAndRename(blobFile, blobDigest, &blobFileClosed); err != nil {
+		return "", -1, err
+	}
+
+	return blobDigest, fileInfo.Size(), nil
+}
+
 func ensureDirectoryExists(path string) error {
 	if err := fileutils.Exists(path); err != nil && errors.Is(err, fs.ErrNotExist) {
 		if err := os.MkdirAll(path, 0755); err != nil {

oci/layout/oci_dest_test.go

@@ -178,3 +178,41 @@ func putTestManifest(t *testing.T, ociRef ociReference, tmpDir string) {
 	digest := digest.FromBytes(data).Encoded()
 	assert.Contains(t, paths, filepath.Join(tmpDir, "blobs", "sha256", digest), "The OCI directory does not contain the new manifest data")
 }
+
+func TestPutblobFromLocalFile(t *testing.T) {
+	ref, _ := refToTempOCI(t, false)
+	dest, err := ref.NewImageDestination(context.Background(), nil)
+	require.NoError(t, err)
+	defer dest.Close()
+	ociDest, ok := dest.(*ociImageDestination)
+	require.True(t, ok)
+
+	for _, test := range []struct {
+		path   string
+		size   int64
+		digest string
+	}{
+		{path: "fixtures/files/a.txt", size: 31, digest: "sha256:c8a3f498ce6aaa13c803fa3a6a0d5fd6b5d75be5781f98f56c0f960efcc53174"},
+		{path: "fixtures/files/b.txt", size: 25, digest: "sha256:8c1e9b03116b95e6dfac68c588190d56bfc82b9cc550ada726e882e138a3b93b"},
+		{path: "fixtures/files/b.txt", size: 25, digest: "sha256:8c1e9b03116b95e6dfac68c588190d56bfc82b9cc550ada726e882e138a3b93b"}, // Must not fail
+	} {
+		digest, size, err := PutBlobFromLocalFile(context.Background(), dest, test.path)
+		require.NoError(t, err)
+		require.Equal(t, test.size, size)
+		require.Equal(t, test.digest, digest.String())
+
+		blobPath, err := ociDest.ref.blobPath(digest, ociDest.sharedBlobDir)
+		require.NoError(t, err)
+		require.FileExists(t, blobPath)
+
+		expectedContent, err := os.ReadFile(test.path)
+		require.NoError(t, err)
+		require.NotEmpty(t, expectedContent)
+		blobContent, err := os.ReadFile(blobPath)
+		require.NoError(t, err)
+		require.Equal(t, expectedContent, blobContent)
+	}
+
+	err = ociDest.CommitWithOptions(context.Background(), private.CommitOptions{})
+	require.NoError(t, err)
+}