Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add ID-mapping capabilities #1392

Merged
merged 3 commits into from
Nov 4, 2024
Merged

Add ID-mapping capabilities #1392

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ab15619
Add id-mapped nerdctl to Makefile
sondavidb Oct 8, 2024
e390d85
Add idtools package
sondavidb Sep 26, 2024
4a2234d
Add support for idmapped layers
sondavidb Jun 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,9 @@ RUN cp $GOPATH/src/github.com/awslabs/soci-snapshotter/out/soci /usr/local/bin/
cp $GOPATH/src/github.com/awslabs/soci-snapshotter/soci-snapshotter.socket /etc/systemd/system
RUN curl -sSL --output /tmp/containerd.tgz https://github.com/containerd/containerd/releases/download/v${CONTAINERD_VERSION}/containerd-${CONTAINERD_VERSION}-linux-${TARGETARCH:-amd64}.tar.gz && \
tar zxvf /tmp/containerd.tgz -C /usr/local/ && \
rm -f /tmp/containerd.tgz
rm -f /tmp/containerd.tgz && \
cp $GOPATH/src/github.com/awslabs/soci-snapshotter/out/nerdctl-with-idmapping /usr/local/bin && \
chmod +x /usr/local/bin/nerdctl-with-idmapping
RUN curl -sSL --output /tmp/runc https://github.com/opencontainers/runc/releases/download/v${RUNC_VERSION}/runc.${TARGETARCH:-amd64} && \
cp /tmp/runc /usr/local/bin/ && \
chmod +x /usr/local/bin/runc && \
Expand Down
23 changes: 21 additions & 2 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,12 @@ SOCI_GRPC_PACKAGE_LIST=$(shell echo $(SOCI_LIBRARY_PACKAGE_LIST),$(shell cd $(SO

GO_BENCHMARK_TESTS?=.

NERDCTL_REPO = https://github.com/containerd/nerdctl.git
NERDCTL_TAG = v1.7.7
NERDCTL_PATCH = $(SOCI_SNAPSHOTTER_PROJECT_ROOT)/integration/config/nerdctl.patch

.PHONY: all build check flatc add-ltag install uninstall tidy vendor clean clean-coverage \
clean-integration test test-with-coverage show-test-coverage show-test-coverage-html \
clean-integration test test-with-coverage show-test-coverage show-test-coverage-html nerdctl-with-idmapping \
integration integration-with-coverage show-integration-coverage show-integration-coverage-html \
release benchmarks build-benchmarks benchmarks-perf-test benchmarks-comparison-test

Expand Down Expand Up @@ -162,7 +166,7 @@ $(COVDIR)/unit: $(COVDIR)
GO_BUILD_FLAGS="$(GO_BUILD_FLAGS) -coverpkg=$(SOCI_LIBRARY_PACKAGE_LIST)"\
$(MAKE) test

integration: build
integration: build nerdctl-with-idmapping
@echo "$@"
@echo "SOCI_SNAPSHOTTER_PROJECT_ROOT=$(SOCI_SNAPSHOTTER_PROJECT_ROOT)"
@GO111MODULE=$(GO111MODULE_VALUE) SOCI_SNAPSHOTTER_PROJECT_ROOT=$(SOCI_SNAPSHOTTER_PROJECT_ROOT) ENABLE_INTEGRATION_TEST=true go test $(GO_TEST_FLAGS) -v -timeout=0 ./integration
Expand All @@ -182,6 +186,21 @@ $(COVDIR)/integration: $(COVDIR)
GO_BUILD_FLAGS="$(GO_BUILD_FLAGS) -coverpkg=$(SOCI_CLI_PACKAGE_LIST),$(SOCI_GRPC_PACKAGE_LIST)" \
$(MAKE) integration

nerdctl-with-idmapping: $(OUTDIR)/nerdctl-with-idmapping

$(OUTDIR)/nerdctl-with-idmapping:
# Use a custom patch for testing ID-mapping as nerdctl doesn't fully support this yet.
rm -rf $(SOCI_SNAPSHOTTER_PROJECT_ROOT)/tempfolder

git clone $(NERDCTL_REPO) $(SOCI_SNAPSHOTTER_PROJECT_ROOT)/tempfolder
cd $(SOCI_SNAPSHOTTER_PROJECT_ROOT)/tempfolder && \
git checkout $(NERDCTL_TAG) && \
git apply $(NERDCTL_PATCH) && \
make && \
cp _output/nerdctl $(OUTDIR)/nerdctl-with-idmapping && \
cd ../
rm -rf $(SOCI_SNAPSHOTTER_PROJECT_ROOT)/tempfolder

release:
@echo "$@"
@$(SOCI_SNAPSHOTTER_PROJECT_ROOT)/scripts/create-releases.sh $(RELEASE_TAG)
Expand Down
113 changes: 94 additions & 19 deletions fs/fs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,13 @@ package fs
import (
"context"
"fmt"
"io"
golog "log"
"net/http"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"syscall"
"time"
Expand All @@ -59,6 +63,7 @@ import (
layermetrics "github.com/awslabs/soci-snapshotter/fs/metrics/layer"
"github.com/awslabs/soci-snapshotter/fs/remote"
"github.com/awslabs/soci-snapshotter/fs/source"
"github.com/awslabs/soci-snapshotter/idtools"
"github.com/awslabs/soci-snapshotter/metadata"
"github.com/awslabs/soci-snapshotter/snapshot"
"github.com/awslabs/soci-snapshotter/soci"
Expand All @@ -67,6 +72,7 @@ import (
ctdsnapshotters "github.com/containerd/containerd/pkg/snapshotters"
"github.com/containerd/containerd/reference"
"github.com/containerd/containerd/remotes/docker"
"github.com/containerd/errdefs"
"github.com/containerd/log"
metrics "github.com/docker/go-metrics"
fusefs "github.com/hanwen/go-fuse/v2/fs"
Expand Down Expand Up @@ -455,6 +461,58 @@ func (fs *filesystem) getSociContext(ctx context.Context, imageRef, indexDigest,
return c, err
}

func getIDMappedMountpoint(mountpoint, activeLayerID string) string {
d := filepath.Dir(mountpoint)
return filepath.Join(fmt.Sprintf("%s_%s", d, activeLayerID), "fs")
}

func (fs *filesystem) IDMapMount(ctx context.Context, mountpoint, activeLayerID string, idmapper idtools.IDMap) (string, error) {
newMountpoint := getIDMappedMountpoint(mountpoint, activeLayerID)
logger := log.G(ctx).WithField("mountpoint", newMountpoint)

logger.Debug("creating remote id-mapped mount")
if err := os.Mkdir(filepath.Dir(newMountpoint), 0700); err != nil {
return "", err
}
if err := os.Mkdir(newMountpoint, 0755); err != nil {
return "", err
}

fs.layerMu.Lock()
l := fs.layer[mountpoint]
if l == nil {
fs.layerMu.Unlock()
logger.Error("failed to create remote id-mapped mount")
return "", errdefs.ErrNotFound
}
fs.layer[newMountpoint] = l
Shubhranshu153 marked this conversation as resolved.
Show resolved Hide resolved
fs.layerMu.Unlock()
node, err := l.RootNode(0, idmapper)
if err != nil {
return "", err
}

fuseLogger := log.L.
WithField("mountpoint", mountpoint).
WriterLevel(logrus.TraceLevel)
austinvazquez marked this conversation as resolved.
Show resolved Hide resolved

return newMountpoint, fs.setupFuseServer(ctx, newMountpoint, node, l, fuseLogger, nil)
}

func (fs *filesystem) IDMapMountLocal(ctx context.Context, mountpoint, activeLayerID string, idmapper idtools.IDMap) (string, error) {
newMountpoint := getIDMappedMountpoint(mountpoint, activeLayerID)
logger := log.G(ctx).WithField("mountpoint", newMountpoint)

logger.Debug("creating local id-mapped mount")
if err := idtools.RemapDir(ctx, mountpoint, newMountpoint, idmapper); err != nil {
logger.WithError(err).Error("failed to create local mount")
return "", err
}

logger.Debug("successfully created local mountpoint")
return newMountpoint, nil
}

func (fs *filesystem) Mount(ctx context.Context, mountpoint string, labels map[string]string) (retErr error) {
// Setting the start time to measure the Mount operation duration.
start := time.Now()
Expand Down Expand Up @@ -560,7 +618,7 @@ func (fs *filesystem) Mount(ctx context.Context, mountpoint string, labels map[s
}
}()

node, err := l.RootNode(0)
node, err := l.RootNode(0, idtools.IDMap{})
if err != nil {
log.G(ctx).WithError(err).Warnf("Failed to get root node")
retErr = fmt.Errorf("failed to get root node: %w", err)
Expand All @@ -577,6 +635,17 @@ func (fs *filesystem) Mount(ctx context.Context, mountpoint string, labels map[s
fs.layerMu.Unlock()
fs.metricsController.Add(mountpoint, l)

// Pass in a logger to go-fuse with the layer digest
// The go-fuse logs are useful for tracing exactly what's happening at the fuse level.
fuseLogger := log.L.
WithField("layerDigest", labels[ctdsnapshotters.TargetLayerDigestLabel]).
WriterLevel(logrus.TraceLevel)

retErr = fs.setupFuseServer(ctx, mountpoint, node, l, fuseLogger, c)
return
}

func (fs *filesystem) setupFuseServer(ctx context.Context, mountpoint string, node fusefs.InodeEmbedder, l layer.Layer, logger *io.PipeWriter, c *sociContext) error {
// mount the node to the specified mountpoint
// TODO: bind mount the state directory as a read-only fs on snapshotter's side
rawFS := fusefs.NewNodeFS(node, &fusefs.Options{
Expand All @@ -585,11 +654,6 @@ func (fs *filesystem) Mount(ctx context.Context, mountpoint string, labels map[s
NegativeTimeout: &fs.negativeTimeout,
NullPermissions: true,
})
// Pass in a logger to go-fuse with the layer digest
// The go-fuse logs are useful for tracing exactly what's happening at the fuse level.
logger := log.L.
WithField("layerDigest", labels[ctdsnapshotters.TargetLayerDigestLabel]).
WriterLevel(logrus.TraceLevel)
mountOpts := &fuse.MountOptions{
AllowOther: true, // allow users other than root&mounter to access fs
FsName: "soci", // name this filesystem as "soci"
Expand All @@ -600,25 +664,26 @@ func (fs *filesystem) Mount(ctx context.Context, mountpoint string, labels map[s
if _, err := exec.LookPath(fusermountBin); err == nil {
mountOpts.Options = []string{"suid"} // option for fusermount; allow setuid inside container
} else {
log.G(ctx).WithError(err).Infof("%s not installed; trying direct mount", fusermountBin)
log.G(ctx).WithField("binary", fusermountBin).WithError(err).Info("fusermount binary not installed; trying direct mount")
mountOpts.DirectMount = true
}
server, err := fuse.NewServer(rawFS, mountpoint, mountOpts)
if err != nil {
log.G(ctx).WithError(err).Debug("failed to make filesystem server")
retErr = err
return
log.G(ctx).WithError(err).Error("failed to make filesystem server")
return err
}

go server.Serve()

// Send a signal to the background fetcher that a new image is being mounted
// and to pause all background fetches.
c.bgFetchPauseOnce.Do(func() {
if fs.bgFetcher != nil {
fs.bgFetcher.Pause()
}
})
if c != nil {
// Send a signal to the background fetcher that a new image is being mounted
// and to pause all background fetches.
c.bgFetchPauseOnce.Do(func() {
if fs.bgFetcher != nil {
fs.bgFetcher.Pause()
}
})
}

return server.WaitMount()
}
Expand Down Expand Up @@ -681,15 +746,25 @@ func (fs *filesystem) check(ctx context.Context, l layer.Layer, labels map[strin
return rErr
}

func isIDMappedDir(mountpoint string) bool {
dirName := filepath.Base(mountpoint)
return len(strings.Split(dirName, "_")) > 1
}

func (fs *filesystem) Unmount(ctx context.Context, mountpoint string) error {
fs.layerMu.Lock()
l, ok := fs.layer[mountpoint]
if !ok {
fs.layerMu.Unlock()
return fmt.Errorf("specified path %q isn't a mountpoint", mountpoint)
}
delete(fs.layer, mountpoint) // unregisters the corresponding layer
l.Done()

delete(fs.layer, mountpoint)
// If the mountpoint is an id-mapped layer, it is pointing to the
// underlying layer, so we cannot call done on it.
if !isIDMappedDir(mountpoint) {
l.Done()
}
fs.layerMu.Unlock()
fs.metricsController.Remove(mountpoint)
// The goroutine which serving the mountpoint possibly becomes not responding.
Expand Down
11 changes: 7 additions & 4 deletions fs/fs_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ import (
"github.com/awslabs/soci-snapshotter/fs/layer"
"github.com/awslabs/soci-snapshotter/fs/remote"
"github.com/awslabs/soci-snapshotter/fs/source"
"github.com/awslabs/soci-snapshotter/idtools"
"github.com/containerd/containerd/reference"
"github.com/containerd/containerd/remotes/docker"
fusefs "github.com/hanwen/go-fuse/v2/fs"
Expand Down Expand Up @@ -83,10 +84,12 @@ func (l *breakableLayer) Info() layer.Info {
Size: 1,
}
}
func (l *breakableLayer) DisableXAttrs() bool { return false }
func (l *breakableLayer) RootNode(uint32) (fusefs.InodeEmbedder, error) { return nil, nil }
func (l *breakableLayer) Verify(tocDigest digest.Digest) error { return nil }
func (l *breakableLayer) SkipVerify() {}
func (l *breakableLayer) DisableXAttrs() bool { return false }
func (l *breakableLayer) RootNode(uint32, idtools.IDMap) (fusefs.InodeEmbedder, error) {
return nil, nil
}
func (l *breakableLayer) Verify(tocDigest digest.Digest) error { return nil }
func (l *breakableLayer) SkipVerify() {}
func (l *breakableLayer) ReadAt([]byte, int64, ...remote.Option) (int, error) {
return 0, fmt.Errorf("fail")
}
Expand Down
7 changes: 4 additions & 3 deletions fs/layer/layer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ import (
"github.com/awslabs/soci-snapshotter/fs/remote"

spanmanager "github.com/awslabs/soci-snapshotter/fs/span-manager"
"github.com/awslabs/soci-snapshotter/idtools"
"github.com/awslabs/soci-snapshotter/metadata"
"github.com/awslabs/soci-snapshotter/soci"
"github.com/awslabs/soci-snapshotter/util/lrucache"
Expand Down Expand Up @@ -86,7 +87,7 @@ type Layer interface {
Info() Info

// RootNode returns the root node of this layer.
RootNode(baseInode uint32) (fusefs.InodeEmbedder, error)
RootNode(baseInode uint32, idMapper idtools.IDMap) (fusefs.InodeEmbedder, error)

// Check checks if the layer is still connectable.
Check() error
Expand Down Expand Up @@ -456,11 +457,11 @@ func (l *layerRef) Done() {
l.done()
}

func (l *layer) RootNode(baseInode uint32) (fusefs.InodeEmbedder, error) {
func (l *layer) RootNode(baseInode uint32, idMapper idtools.IDMap) (fusefs.InodeEmbedder, error) {
if l.isClosed() {
return nil, fmt.Errorf("layer is already closed")
}
return newNode(l.desc.Digest, l.r, l.blob, baseInode, l.resolver.overlayOpaqueType, l.resolver.config.LogFuseOperations, l.fuseOperationCounter)
return newNode(l.desc.Digest, l.r, l.blob, baseInode, l.resolver.overlayOpaqueType, l.resolver.config.LogFuseOperations, l.fuseOperationCounter, idMapper)
}

func (l *layer) ReadAt(p []byte, offset int64, opts ...remote.Option) (int, error) {
Expand Down
Loading
Loading