Implement llb.Symlink

* Add file.symlink.create capability
* Run codegen for new FileActionSymlink Message
* Add Symlink test
* Add user/group ownership and timestamps to symlink
 ** Symlinks have user/group ownership that are independent of those of the
    target file; in linux, the ownership of the symlink itself is only
    checked when the link resides in a directory with the sticky bit set and
    the link is the subject of removal or renaming. The sticky bit prevents
    files in the directory from being deleted or renamed by non-owners
    (members of the group that owns the file may not delete the file; the
    user must own the file).

    In addition to user/group restrictions, linux symlinks have timestamps
    that are independent of the timestamps on the target file.
* Expose symlink options to `llb` package
* Add symlink integration test
* Use tar exporter for tests
 ** Using the local exporter causes the files to be exported with the
    permissions of the user who does the exporting, instead of retaining
    their file permissions from within the container.

    Using the tar exporter instead preserves the permissions until they can
    be checked.
* Change symlink fields to `oldpath` and `newpath`
 ** Also run `make generated-files`
* Fix typo
* Add doc strings to exported `llb` identifiers
* Remove `requiresLinux` from integration test
* Revert "Remove `requiresLinux` from integration test"

Signed-off-by: Peter Engelbert <[email protected]>

client/client_test.go

@@ -221,6 +221,7 @@ var allTests = []func(t *testing.T, sb integration.Sandbox){
 	testLayerLimitOnMounts,
 	testFrontendVerifyPlatforms,
 	testRunValidExitCodes,
+	testFileOpSymlink,
 }
 
 func TestIntegration(t *testing.T) {
@@ -2409,6 +2410,91 @@ func testOCILayoutPlatformSource(t *testing.T, sb integration.Sandbox) {
 	}
 }
 
+func testFileOpSymlink(t *testing.T, sb integration.Sandbox) {
+	requiresLinux(t)
+
+	const (
+		fileOwner = 7777
+		fileGroup = 8888
+		linkOwner = 1111
+		linkGroup = 2222
+
+		dummyTimestamp = 42
+	)
+
+	dummyTime := time.Unix(dummyTimestamp, 0)
+
+	c, err := New(sb.Context(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	st := llb.Scratch().
+		File(llb.Mkdir("/foo", 0700).Mkfile("bar", 0600, []byte("contents"), llb.ChownOpt{
+			User: &llb.UserOpt{
+				UID: fileOwner,
+			},
+			Group: &llb.UserOpt{
+				UID: fileGroup,
+			},
+		})).
+		File(llb.Symlink("bar", "/baz", llb.WithCreatedTime(dummyTime), llb.ChownOpt{
+			User: &llb.UserOpt{
+				UID: linkOwner,
+			},
+			Group: &llb.UserOpt{
+				UID: linkGroup,
+			},
+		}))
+
+	def, err := st.Marshal(sb.Context())
+	require.NoError(t, err)
+
+	destDir := t.TempDir()
+
+	out := filepath.Join(destDir, "out.tar")
+	outW, err := os.Create(out)
+	require.NoError(t, err)
+	defer outW.Close()
+
+	_, err = c.Solve(sb.Context(), def, SolveOpt{
+		Exports: []ExportEntry{
+			{
+				Type:   ExporterTar,
+				Output: fixedWriteCloser(outW),
+			},
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	dt, err := os.ReadFile(out)
+	require.NoError(t, err)
+	m, err := testutil.ReadTarToMap(dt, false)
+	require.NoError(t, err)
+
+	entry, ok := m["bar"]
+	require.True(t, ok)
+
+	dt = entry.Data
+	header := entry.Header
+	require.NoError(t, err)
+
+	require.Equal(t, []byte("contents"), dt)
+	require.Equal(t, header.Uid, fileOwner)
+	require.Equal(t, header.Gid, fileGroup)
+
+	entry, ok = m["baz"]
+	dt = entry.Data
+	header = entry.Header
+	require.NoError(t, err)
+
+	// make sure it was chowned properly
+	require.Equal(t, true, ok)
+	require.Equal(t, header.Uid, linkOwner)
+	require.Equal(t, header.Gid, linkGroup)
+
+	require.Equal(t, header.ModTime, dummyTime)
+}
+
 func testFileOpRmWildcard(t *testing.T, sb integration.Sandbox) {
 	requiresLinux(t)
 	c, err := New(sb.Context(), sb.Address())

client/llb/fileop.go

@@ -85,6 +85,13 @@ func (fa *FileAction) Mkfile(p string, m os.FileMode, dt []byte, opt ...MkfileOp
 	return a
 }
 
+// Symlink creates a symlink at `newpath` that points to `oldpath`
+func (fa *FileAction) Symlink(oldpath, newpath string, opt ...SymlinkOption) *FileAction {
+	a := Symlink(oldpath, newpath, opt...)
+	a.prev = fa
+	return a
+}
+
 func (fa *FileAction) Rm(p string, opt ...RmOption) *FileAction {
 	a := Rm(p, opt...)
 	a.prev = fa
@@ -193,6 +200,7 @@ type ChownOption interface {
 	MkdirOption
 	MkfileOption
 	CopyOption
+	SymlinkOption
 }
 
 type mkdirOptionFunc func(*MkdirInfo)
@@ -290,6 +298,10 @@ func (co ChownOpt) SetCopyOption(mi *CopyInfo) {
 	mi.ChownOpt = &co
 }
 
+func (co ChownOpt) SetSymlinkOption(si *SymlinkInfo) {
+	si.ChownOpt = &co
+}
+
 func (co *ChownOpt) marshal(base pb.InputIndex) *pb.ChownOpt {
 	if co == nil {
 		return nil
@@ -337,6 +349,53 @@ func Mkfile(p string, m os.FileMode, dt []byte, opts ...MkfileOption) *FileActio
 	}
 }
 
+// SymlinkInfo is the modifiable options used to create symlinks
+type SymlinkInfo struct {
+	ChownOpt    *ChownOpt
+	CreatedTime *time.Time
+}
+
+func (si *SymlinkInfo) SetSymlinkOption(si2 *SymlinkInfo) {
+	*si2 = *si
+}
+
+type SymlinkOption interface {
+	SetSymlinkOption(*SymlinkInfo)
+}
+
+// Symlink creates a symlink at `newpath` that points to `oldpath`
+func Symlink(oldpath, newpath string, opts ...SymlinkOption) *FileAction {
+	var si SymlinkInfo
+	for _, o := range opts {
+		o.SetSymlinkOption(&si)
+	}
+
+	return &FileAction{
+		action: &fileActionSymlink{
+			oldpath: oldpath,
+			newpath: newpath,
+			info:    si,
+		},
+	}
+}
+
+type fileActionSymlink struct {
+	oldpath string
+	newpath string
+	info    SymlinkInfo
+}
+
+func (f *fileActionSymlink) toProtoAction(_ context.Context, _ string, base pb.InputIndex) (pb.IsFileAction, error) {
+	return &pb.FileAction_Symlink{
+		Symlink: &pb.FileActionSymlink{
+			Oldpath:   f.oldpath,
+			Newpath:   f.newpath,
+			Owner:     f.info.ChownOpt.marshal(base),
+			Timestamp: marshalTime(f.info.CreatedTime),
+		},
+	}, nil
+}
+
 type MkfileOption interface {
 	SetMkfileOption(*MkfileInfo)
 }
@@ -606,6 +665,10 @@ func (c CreatedTime) SetMkfileOption(mi *MkfileInfo) {
 	mi.CreatedTime = (*time.Time)(&c)
 }
 
+func (c CreatedTime) SetSymlinkOption(si *SymlinkInfo) {
+	si.CreatedTime = (*time.Time)(&c)
+}
+
 func (c CreatedTime) SetCopyOption(mi *CopyInfo) {
 	mi.CreatedTime = (*time.Time)(&c)
 }

client/llb/fileop_test.go

@@ -178,6 +178,41 @@ func TestFileMkfile(t *testing.T) {
 	require.Equal(t, int64(-1), mkdir.Timestamp)
 }
 
+func TestFileSymlink(t *testing.T) {
+	t.Parallel()
+
+	st := Scratch().
+		File(Mkfile("/foo", 0700, []byte("data"))).
+		File(Symlink("foo", "/bar"))
+	def, err := st.Marshal(context.TODO())
+
+	require.NoError(t, err)
+
+	m, arr := parseDef(t, def.Def)
+	require.Equal(t, 3, len(arr))
+
+	dgst, idx := last(t, arr)
+	require.Equal(t, 0, idx)
+	require.Equal(t, m[dgst], arr[1])
+
+	f := arr[1].Op.(*pb.Op_File).File
+	require.Equal(t, 1, len(arr[1].Inputs))
+	require.Equal(t, m[arr[1].Inputs[0].Digest], arr[0])
+	require.Equal(t, 0, int(arr[1].Inputs[0].Index))
+
+	require.Equal(t, 1, len(f.Actions))
+
+	action := f.Actions[0]
+	require.Equal(t, 0, int(action.Input))
+	require.Equal(t, -1, int(action.SecondaryInput))
+	require.Equal(t, 0, int(action.Output))
+
+	symlink := action.Action.(*pb.FileAction_Symlink).Symlink
+
+	require.Equal(t, "foo", symlink.Oldpath)
+	require.Equal(t, "/bar", symlink.Newpath)
+}
+
 func TestFileRm(t *testing.T) {
 	t.Parallel()
 

cmd/buildctl/debug/dumpllb.go

@@ -130,6 +130,8 @@ func attr(dgst digest.Digest, op *pb.Op) (string, string) {
 				name = fmt.Sprintf("mkdir{path=%s}", act.Mkdir.Path)
 			case *pb.FileAction_Rm:
 				name = fmt.Sprintf("rm{path=%s}", act.Rm.Path)
+			case *pb.FileAction_Symlink:
+				name = fmt.Sprintf("symlink{oldpath=%s, newpath=%s}", act.Symlink.Oldpath, act.Symlink.Newpath)
 			}
 
 			names = append(names, name)

solver/llbsolver/file/backend.go

@@ -67,6 +67,40 @@ func mkdir(d string, action *pb.FileActionMkDir, user *copy.User, idmap *idtools
 	return nil
 }
 
+func symlink(d string, action *pb.FileActionSymlink, user *copy.User, idmap *idtools.IdentityMapping) (err error) {
+	defer func() {
+		var osErr *os.PathError
+		if errors.As(err, &osErr) {
+			// remove system root from error path if present
+			osErr.Path = strings.TrimPrefix(osErr.Path, d)
+		}
+	}()
+
+	newpath, err := fs.RootPath(d, filepath.Join("/", action.Newpath))
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	ch, err := mapUserToChowner(user, idmap)
+	if err != nil {
+		return err
+	}
+
+	if err := os.Symlink(action.Oldpath, newpath); err != nil {
+		return errors.WithStack(err)
+	}
+
+	if err := copy.Chown(newpath, nil, ch); err != nil {
+		return errors.WithStack(err)
+	}
+
+	if err := copy.Utimes(newpath, timestampToTime(action.Timestamp)); err != nil {
+		return errors.WithStack(err)
+	}
+
+	return nil
+}
+
 func mkfile(d string, action *pb.FileActionMkFile, user *copy.User, idmap *idtools.IdentityMapping) (err error) {
 	defer func() {
 		var osErr *os.PathError
@@ -304,6 +338,27 @@ func (fb *Backend) Mkfile(ctx context.Context, m, user, group fileoptypes.Mount,
 	return mkfile(dir, action, u, mnt.m.IdentityMapping())
 }
 
+func (fb *Backend) Symlink(ctx context.Context, m, user, group fileoptypes.Mount, action *pb.FileActionSymlink) error {
+	mnt, ok := m.(*Mount)
+	if !ok {
+		return errors.Errorf("invalid mount type %T", m)
+	}
+
+	lm := snapshot.LocalMounter(mnt.m)
+	dir, err := lm.Mount()
+	if err != nil {
+		return err
+	}
+	defer lm.Unmount()
+
+	u, err := fb.readUserWrapper(action.Owner, user, group)
+	if err != nil {
+		return err
+	}
+
+	return symlink(dir, action, u, mnt.m.IdentityMapping())
+}
+
 func (fb *Backend) Rm(ctx context.Context, m fileoptypes.Mount, action *pb.FileActionRm) error {
 	mnt, ok := m.(*Mount)
 	if !ok {

solver/llbsolver/ops/file.go

@@ -84,6 +84,13 @@ func (f *fileOp) CacheMap(ctx context.Context, g session.Group, index int) (*sol
 			if err != nil {
 				return nil, false, err
 			}
+		case *pb.FileAction_Symlink:
+			p := a.Symlink.CloneVT()
+			markInvalid(action.Input)
+			dt, err = json.Marshal(p)
+			if err != nil {
+				return nil, false, err
+			}
 		case *pb.FileAction_Rm:
 			p := a.Rm.CloneVT()
 			markInvalid(action.Input)
@@ -586,6 +593,14 @@ func (s *FileOpSolver) getInput(ctx context.Context, idx int, inputs []fileoptyp
 			if err := s.b.Mkdir(ctx, inpMount, user, group, a.Mkdir); err != nil {
 				return input{}, err
 			}
+		case *pb.FileAction_Symlink:
+			user, group, err := loadOwner(ctx, a.Symlink.Owner)
+			if err != nil {
+				return input{}, err
+			}
+			if err := s.b.Symlink(ctx, inpMount, user, group, a.Symlink); err != nil {
+				return input{}, err
+			}
 		case *pb.FileAction_Mkfile:
 			user, group, err := loadOwner(ctx, a.Mkfile.Owner)
 			if err != nil {

solver/llbsolver/ops/file_test.go

@@ -600,6 +600,7 @@ type mod struct {
 	rm      *pb.FileActionRm
 	mkfile  *pb.FileActionMkFile
 	copy    *pb.FileActionCopy
+	symlink *pb.FileActionSymlink
 	copySrc []mod
 }
 
@@ -643,6 +644,13 @@ func (b *testFileBackend) Mkfile(_ context.Context, m, user, group fileoptypes.M
 	return nil
 }
 
+func (b *testFileBackend) Symlink(_ context.Context, m, user, group fileoptypes.Mount, a *pb.FileActionSymlink) error {
+	mm := m.(*testMount)
+	mm.id += "-symlink"
+	mm.chain = append(mm.chain, mod{symlink: a})
+	return nil
+}
+
 func (b *testFileBackend) Rm(_ context.Context, m fileoptypes.Mount, a *pb.FileActionRm) error {
 	mm := m.(*testMount)
 	mm.id += "-rm"

solver/llbsolver/ops/fileoptypes/types.go

@@ -19,6 +19,7 @@ type Mount interface {
 
 type Backend interface {
 	Mkdir(context.Context, Mount, Mount, Mount, *pb.FileActionMkDir) error
+	Symlink(context.Context, Mount, Mount, Mount, *pb.FileActionSymlink) error
 	Mkfile(context.Context, Mount, Mount, Mount, *pb.FileActionMkFile) error
 	Rm(context.Context, Mount, *pb.FileActionRm) error
 	Copy(context.Context, Mount, Mount, Mount, Mount, *pb.FileActionCopy) error

solver/llbsolver/vertex.go

@@ -391,6 +391,8 @@ func fileOpName(actions []*pb.FileAction) string {
 			names = append(names, fmt.Sprintf("mkdir %s", a.Mkdir.Path))
 		case *pb.FileAction_Mkfile:
 			names = append(names, fmt.Sprintf("mkfile %s", a.Mkfile.Path))
+		case *pb.FileAction_Symlink:
+			names = append(names, fmt.Sprintf("symlink %s -> %s", a.Symlink.Newpath, a.Symlink.Oldpath))
 		case *pb.FileAction_Rm:
 			names = append(names, fmt.Sprintf("rm %s", a.Rm.Path))
 		case *pb.FileAction_Copy: