Add support for idmapped layers

Signed-off-by: David Son <[email protected]>

config/config.toml

@@ -16,6 +16,7 @@ filesystem_cache_type=""
 resolve_result_entry=0
 debug=false
 allow_no_verification=true
+allow_idmap=true
 # disable_verification=false
 # Causes TestRunWithDefaultConfig to break, but
 # fine to use in /etc/soci-snapshotter-grpc-config.toml

config/fs.go

@@ -56,6 +56,7 @@ type FSConfig struct {
 	NoPrometheus                   bool   `toml:"no_prometheus"`
 	MountTimeoutSec                int64  `toml:"mount_timeout_sec"`
 	FuseMetricsEmitWaitDurationSec int64  `toml:"fuse_metrics_emit_wait_duration_sec"`
+	AllowIDMap                     bool   `toml:"allow_idmap" default:"true"`
 
 	RetryableHTTPClientConfig `toml:"http"`
 	BlobConfig                `toml:"blob"`

fs/fs.go

@@ -47,7 +47,9 @@ import (
 	"fmt"
 	golog "log"
 	"net/http"
+	"os"
 	"os/exec"
+	"path/filepath"
 	"sync"
 	"syscall"
 	"time"
@@ -59,6 +61,7 @@ import (
 	layermetrics "github.com/awslabs/soci-snapshotter/fs/metrics/layer"
 	"github.com/awslabs/soci-snapshotter/fs/remote"
 	"github.com/awslabs/soci-snapshotter/fs/source"
+	"github.com/awslabs/soci-snapshotter/idtools"
 	"github.com/awslabs/soci-snapshotter/metadata"
 	"github.com/awslabs/soci-snapshotter/snapshot"
 	"github.com/awslabs/soci-snapshotter/soci"
@@ -67,6 +70,7 @@ import (
 	ctdsnapshotters "github.com/containerd/containerd/pkg/snapshotters"
 	"github.com/containerd/containerd/reference"
 	"github.com/containerd/containerd/remotes/docker"
+	"github.com/containerd/errdefs"
 	"github.com/containerd/log"
 	metrics "github.com/docker/go-metrics"
 	fusefs "github.com/hanwen/go-fuse/v2/fs"
@@ -455,6 +459,79 @@ func (fs *filesystem) getSociContext(ctx context.Context, imageRef, indexDigest,
 	return c, err
 }
 
+func (fs *filesystem) getIDMappedMountpoint(mountpoint, activeLayerKey string) string {
+	d := filepath.Dir(mountpoint)
+	return filepath.Join(fmt.Sprintf("%s_%s", d, activeLayerKey), "fs")
+}
+
+func (fs *filesystem) IDMapMount(ctx context.Context, mountpoint, activeLayerKey string, idmapper idtools.IDMap) (string, error) {
+	newMountpoint := fs.getIDMappedMountpoint(mountpoint, activeLayerKey)
+	log.G(ctx).WithField("mountpoint", newMountpoint).Info("creating remote id-mapped mount")
+	if err := os.Mkdir(filepath.Dir(newMountpoint), 0700); err != nil {
+		return "", err
+	}
+	if err := os.Mkdir(newMountpoint, 0755); err != nil {
+		return "", err
+	}
+
+	fs.layerMu.Lock()
+	l := fs.layer[mountpoint]
+	if l == nil {
+		fs.layerMu.Unlock()
+		log.G(ctx).WithField("mountpoint", newMountpoint).Info("failed to create remote id-mapped mount")
+		return "", errdefs.ErrNotFound
+	}
+	fs.layer[newMountpoint] = l
+	fs.layerMu.Unlock()
+	node, err := l.RootNode(0, idmapper)
+	if err != nil {
+		return "", err
+	}
+
+	rawFS := fusefs.NewNodeFS(node, &fusefs.Options{
+		AttrTimeout:     &fs.attrTimeout,
+		EntryTimeout:    &fs.entryTimeout,
+		NegativeTimeout: &fs.negativeTimeout,
+		NullPermissions: true,
+	})
+	mountOpts := &fuse.MountOptions{
+		AllowOther:    true,   // allow users other than root&mounter to access fs
+		FsName:        "soci", // name this filesystem as "soci"
+		Debug:         fs.debug,
+		DisableXAttrs: l.DisableXAttrs(),
+		Options:       []string{"default_permissions", "ro"},
+	}
+	if _, err := exec.LookPath(fusermountBin); err == nil {
+		mountOpts.Options = []string{"suid"} // option for fusermount; allow setuid inside container
+	} else {
+		log.G(ctx).WithError(err).Infof("%s not installed; trying direct mount", fusermountBin)
+		mountOpts.DirectMount = true
+	}
+	server, err := fuse.NewServer(rawFS, newMountpoint, mountOpts)
+	if err != nil {
+		log.G(ctx).WithError(err).Debug("failed to make filesystem server")
+		return "", err
+	}
+
+	go server.Serve()
+
+	log.G(ctx).WithField("mountpoint", newMountpoint).Info("successfully created remote mountpoint")
+	return newMountpoint, server.WaitMount()
+}
+
+func (fs *filesystem) IDMapMountLocal(ctx context.Context, mountpoint, activeLayerKey string, idmapper idtools.IDMap) (string, error) {
+	newMountpoint := fs.getIDMappedMountpoint(mountpoint, activeLayerKey)
+	log.G(ctx).WithField("mountpoint", newMountpoint).Info("creating local id-mapped mount")
+
+	if err := idtools.RemapDir(ctx, mountpoint, newMountpoint, idmapper); err != nil {
+		log.G(ctx).WithField("mountpoint", newMountpoint).Errorf("failed to create local mount: %v", err)
+		return "", err
+	}
+
+	log.G(ctx).WithField("mountpoint", newMountpoint).Info("successfully created local mountpoint")
+	return newMountpoint, nil
+}
+
 func (fs *filesystem) Mount(ctx context.Context, mountpoint string, labels map[string]string) (retErr error) {
 	// Setting the start time to measure the Mount operation duration.
 	start := time.Now()
@@ -566,7 +643,7 @@ func (fs *filesystem) Mount(ctx context.Context, mountpoint string, labels map[s
 	// Maybe we should reword the log here or remove it entirely,
 	// since the old Verify() function no longer serves any purpose.
 
-	node, err := l.RootNode(0)
+	node, err := l.RootNode(0, idtools.IDMap{})
 	if err != nil {
 		log.G(ctx).WithError(err).Warnf("Failed to get root node")
 		retErr = fmt.Errorf("failed to get root node: %w", err)
@@ -602,6 +679,7 @@ func (fs *filesystem) Mount(ctx context.Context, mountpoint string, labels map[s
 		Debug:         fs.debug,
 		Logger:        golog.New(logger, "", 0),
 		DisableXAttrs: l.DisableXAttrs(),
+		Options:       []string{"default_permissions", "ro"},
 	}
 	if _, err := exec.LookPath(fusermountBin); err == nil {
 		mountOpts.Options = []string{"suid"} // option for fusermount; allow setuid inside container
@@ -692,7 +770,8 @@ func (fs *filesystem) Unmount(ctx context.Context, mountpoint string) error {
 	l, ok := fs.layer[mountpoint]
 	if !ok {
 		fs.layerMu.Unlock()
-		return fmt.Errorf("specified path %q isn't a mountpoint", mountpoint)
+		log.G(ctx).Errorf("specified path %q isn't a remote mount", mountpoint)
+		return errdefs.ErrNotFound
 	}
 	delete(fs.layer, mountpoint) // unregisters the corresponding layer
 	l.Done()

fs/fs_test.go

@@ -46,6 +46,7 @@ import (
 	"github.com/awslabs/soci-snapshotter/fs/layer"
 	"github.com/awslabs/soci-snapshotter/fs/remote"
 	"github.com/awslabs/soci-snapshotter/fs/source"
+	"github.com/awslabs/soci-snapshotter/idtools"
 	"github.com/containerd/containerd/reference"
 	"github.com/containerd/containerd/remotes/docker"
 	fusefs "github.com/hanwen/go-fuse/v2/fs"
@@ -83,10 +84,12 @@ func (l *breakableLayer) Info() layer.Info {
 		Size: 1,
 	}
 }
-func (l *breakableLayer) DisableXAttrs() bool                           { return false }
-func (l *breakableLayer) RootNode(uint32) (fusefs.InodeEmbedder, error) { return nil, nil }
-func (l *breakableLayer) Verify(tocDigest digest.Digest) error          { return nil }
-func (l *breakableLayer) SkipVerify()                                   {}
+func (l *breakableLayer) DisableXAttrs() bool { return false }
+func (l *breakableLayer) RootNode(uint32, idtools.IDMap) (fusefs.InodeEmbedder, error) {
+	return nil, nil
+}
+func (l *breakableLayer) Verify(tocDigest digest.Digest) error { return nil }
+func (l *breakableLayer) SkipVerify()                          {}
 func (l *breakableLayer) ReadAt([]byte, int64, ...remote.Option) (int, error) {
 	return 0, fmt.Errorf("fail")
 }

fs/layer/layer.go

@@ -58,6 +58,7 @@ import (
 	"github.com/awslabs/soci-snapshotter/fs/remote"
 
 	spanmanager "github.com/awslabs/soci-snapshotter/fs/span-manager"
+	"github.com/awslabs/soci-snapshotter/idtools"
 	"github.com/awslabs/soci-snapshotter/metadata"
 	"github.com/awslabs/soci-snapshotter/soci"
 	"github.com/awslabs/soci-snapshotter/util/lrucache"
@@ -86,7 +87,7 @@ type Layer interface {
 	Info() Info
 
 	// RootNode returns the root node of this layer.
-	RootNode(baseInode uint32) (fusefs.InodeEmbedder, error)
+	RootNode(baseInode uint32, idMapper idtools.IDMap) (fusefs.InodeEmbedder, error)
 
 	// Check checks if the layer is still connectable.
 	Check() error
@@ -489,14 +490,14 @@ func (l *layerRef) Done() {
 	l.done()
 }
 
-func (l *layer) RootNode(baseInode uint32) (fusefs.InodeEmbedder, error) {
+func (l *layer) RootNode(baseInode uint32, idMapper idtools.IDMap) (fusefs.InodeEmbedder, error) {
 	if l.isClosed() {
 		return nil, fmt.Errorf("layer is already closed")
 	}
 	if l.r == nil {
 		return nil, fmt.Errorf("layer hasn't been verified yet")
 	}
-	return newNode(l.desc.Digest, l.r, l.blob, baseInode, l.resolver.overlayOpaqueType, l.resolver.config.LogFuseOperations, l.fuseOperationCounter)
+	return newNode(l.desc.Digest, l.r, l.blob, baseInode, l.resolver.overlayOpaqueType, l.resolver.config.LogFuseOperations, l.fuseOperationCounter, idMapper)
 }
 
 func (l *layer) ReadAt(p []byte, offset int64, opts ...remote.Option) (int, error) {

fs/layer/node.go

@@ -56,6 +56,7 @@ import (
 	commonmetrics "github.com/awslabs/soci-snapshotter/fs/metrics/common"
 	"github.com/awslabs/soci-snapshotter/fs/reader"
 	"github.com/awslabs/soci-snapshotter/fs/remote"
+	"github.com/awslabs/soci-snapshotter/idtools"
 	"github.com/awslabs/soci-snapshotter/metadata"
 	"github.com/containerd/log"
 	fusefs "github.com/hanwen/go-fuse/v2/fs"
@@ -189,7 +190,7 @@ func (f *FuseOperationCounter) Run(ctx context.Context) {
 
 // logFSOperations may cause sensitive information to be emitted to logs
 // e.g. filenames and paths within an image
-func newNode(layerDgst digest.Digest, r reader.Reader, blob remote.Blob, baseInode uint32, opaque OverlayOpaqueType, logFSOperations bool, opCounter *FuseOperationCounter) (fusefs.InodeEmbedder, error) {
+func newNode(layerDgst digest.Digest, r reader.Reader, blob remote.Blob, baseInode uint32, opaque OverlayOpaqueType, logFSOperations bool, opCounter *FuseOperationCounter, idMapper idtools.IDMap) (fusefs.InodeEmbedder, error) {
 	rootID := r.Metadata().RootID()
 	rootAttr, err := r.Metadata().GetAttr(rootID)
 	if err != nil {
@@ -210,9 +211,10 @@ func newNode(layerDgst digest.Digest, r reader.Reader, blob remote.Blob, baseIno
 	}
 	ffs.s = ffs.newState(layerDgst, blob)
 	return &node{
-		id:   rootID,
-		attr: rootAttr,
-		fs:   ffs,
+		id:       rootID,
+		attr:     rootAttr,
+		fs:       ffs,
+		idMapper: idMapper,
 	}, nil
 }
 
@@ -272,9 +274,10 @@ func (fs *fs) inodeOfID(id uint32) (uint64, error) {
 // node is a filesystem inode abstraction.
 type node struct {
 	fusefs.Inode
-	fs   *fs
-	id   uint32
-	attr metadata.Attr
+	fs       *fs
+	id       uint32
+	attr     metadata.Attr
+	idMapper idtools.IDMap
 
 	ents       []fuse.DirEntry
 	entsCached bool
@@ -407,14 +410,14 @@ func (n *node) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fu
 				n.fs.reportFailure(fuseOpLookup, fmt.Errorf("%s: %v", fuseOpLookup, err))
 				return nil, syscall.EIO
 			}
-			entryToAttr(ino, tn.attr, &out.Attr)
+			n.entryToAttr(ino, tn.attr, &out.Attr)
 		case *whiteout:
 			ino, err := n.fs.inodeOfID(tn.id)
 			if err != nil {
 				n.fs.reportFailure(fuseOpLookup, fmt.Errorf("%s: %v", fuseOpLookup, err))
 				return nil, syscall.EIO
 			}
-			entryToAttr(ino, tn.attr, &out.Attr)
+			n.entryToAttr(ino, tn.attr, &out.Attr)
 		default:
 			n.fs.reportFailure(fuseOpLookup, fmt.Errorf("%s: unknown node type detected", fuseOpLookup))
 			return nil, syscall.EIO
@@ -463,10 +466,11 @@ func (n *node) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fu
 		return nil, syscall.EIO
 	}
 	return n.NewInode(ctx, &node{
-		id:   id,
-		fs:   n.fs,
-		attr: ce,
-	}, entryToAttr(ino, ce, &out.Attr)), 0
+		id:       id,
+		fs:       n.fs,
+		attr:     ce,
+		idMapper: n.idMapper,
+	}, n.entryToAttr(ino, ce, &out.Attr)), 0
 }
 
 var _ = (fusefs.NodeOpener)((*node)(nil))
@@ -495,7 +499,7 @@ func (n *node) Getattr(ctx context.Context, f fusefs.FileHandle, out *fuse.AttrO
 		n.fs.reportFailure(fuseOpGetattr, fmt.Errorf("%s: %v", fuseOpGetattr, err))
 		return syscall.EIO
 	}
-	entryToAttr(ino, n.attr, &out.Attr)
+	n.entryToAttr(ino, n.attr, &out.Attr)
 	return 0
 }
 
@@ -594,7 +598,7 @@ func (f *file) Getattr(ctx context.Context, out *fuse.AttrOut) syscall.Errno {
 		f.n.fs.reportFailure(fuseOpFileGetattr, fmt.Errorf("%s: %v", fuseOpFileGetattr, err))
 		return syscall.EIO
 	}
-	entryToAttr(ino, f.n.attr, &out.Attr)
+	f.n.entryToAttr(ino, f.n.attr, &out.Attr)
 	return 0
 }
 
@@ -797,7 +801,7 @@ func (sf *statFile) updateStatUnlocked() ([]byte, error) {
 }
 
 // entryToAttr converts metadata.Attr to go-fuse's Attr.
-func entryToAttr(ino uint64, e metadata.Attr, out *fuse.Attr) fusefs.StableAttr {
+func (n *node) entryToAttr(ino uint64, e metadata.Attr, out *fuse.Attr) fusefs.StableAttr {
 	out.Ino = ino
 	out.Size = uint64(e.Size)
 	if e.Mode&os.ModeSymlink != 0 {
@@ -808,7 +812,9 @@ func entryToAttr(ino uint64, e metadata.Attr, out *fuse.Attr) fusefs.StableAttr
 	mtime := e.ModTime
 	out.SetTimes(nil, &mtime, nil)
 	out.Mode = fileModeToSystemMode(e.Mode)
-	out.Owner = fuse.Owner{Uid: uint32(e.UID), Gid: uint32(e.GID)}
+	// Potentially dangerous casting int -> uint32? But probably fine.
+	mappedID, _ := n.idMapper.ToHost(idtools.User{Uid: uint32(e.UID), Gid: uint32(e.GID)})
+	out.Owner = fuse.Owner{Uid: mappedID.Uid, Gid: mappedID.Gid}
 	out.Rdev = uint32(unix.Mkdev(uint32(e.DevMajor), uint32(e.DevMinor)))
 	out.Nlink = uint32(e.NumLink)
 	if out.Nlink == 0 {

fs/layer/node_test.go

@@ -50,7 +50,8 @@ func TestEntryToAttr(t *testing.T) {
 		tc := tc
 		t.Run(tc.name, func(t *testing.T) {
 			var actual fuse.Attr
-			entryToAttr(0, tc.attr, &actual)
+			var n node
+			n.entryToAttr(0, tc.attr, &actual)
 			tc.expected.Mtime = actual.Mtime
 			if actual != tc.expected {
 				t.Fatalf("unexpected fuse attr. actual %v expected %v", actual, tc.expected)

fs/layer/util_test.go

@@ -56,6 +56,7 @@ import (
 	"github.com/awslabs/soci-snapshotter/fs/reader"
 	"github.com/awslabs/soci-snapshotter/fs/remote"
 	spanmanager "github.com/awslabs/soci-snapshotter/fs/span-manager"
+	"github.com/awslabs/soci-snapshotter/idtools"
 	"github.com/awslabs/soci-snapshotter/metadata"
 	"github.com/awslabs/soci-snapshotter/util/testutil"
 	"github.com/awslabs/soci-snapshotter/ztoc"
@@ -362,7 +363,7 @@ func hasSize(name string, size int) check {
 }
 
 func getRootNode(t *testing.T, r reader.Reader, opaque OverlayOpaqueType) *node {
-	rootNode, err := newNode(testStateLayerDigest, &testReader{r}, &testBlobState{10, 5}, 100, opaque, false, nil)
+	rootNode, err := newNode(testStateLayerDigest, &testReader{r}, &testBlobState{10, 5}, 100, opaque, false, nil, idtools.IDMap{})
 	if err != nil {
 		t.Fatalf("failed to get root node: %v", err)
 	}

fs/source/source.go

@@ -84,6 +84,9 @@ const (
 
 	// TargetSociIndexDigestLabel is a label which contains the digest of the soci index.
 	TargetSociIndexDigestLabel = "containerd.io/snapshot/remote/soci.index.digest"
+
+	// HasSociIndexDigest is a label that tells if the layer was pulled with a SOCI index.
+	HasSociIndexDigest = "containerd.io/snapshot/remote/has.soci.index.digest"
 )
 
 // RegistryHosts is copied from [github.com/awslabs/soci-snapshotter/service/resolver.RegistryHosts]

go.mod

@@ -27,6 +27,7 @@ require (
 	github.com/prometheus/client_golang v1.20.4
 	github.com/rs/xid v1.6.0
 	github.com/sirupsen/logrus v1.9.3
+	github.com/stretchr/testify v1.9.0
 	go.etcd.io/bbolt v1.3.11
 	golang.org/x/crypto v0.27.0
 	golang.org/x/sync v0.8.0
@@ -90,7 +91,6 @@ require (
 	github.com/prometheus/common v0.55.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/stretchr/testify v1.9.0 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect
 	go.opentelemetry.io/otel v1.21.0 // indirect