From bc8877d3f20e7617e9a9ae175039c3dacc1ec1b0 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Wed, 12 Jul 2023 18:48:40 -0400 Subject: [PATCH 1/3] ipfs: add Files API support (WIP) --- go.mod | 8 + go.sum | 9 + internal/filesystem/ipfs/files.go | 454 ++++++++++++++++++++++++++++++ 3 files changed, 471 insertions(+) create mode 100644 internal/filesystem/ipfs/files.go diff --git a/go.mod b/go.mod index f959a1e4..cd3d4168 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,13 @@ require ( golang.org/x/term v0.9.0 ) +require ( + github.com/benbjohnson/clock v1.3.5 // indirect + github.com/libp2p/go-flow-metrics v0.1.0 // indirect + github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect + github.com/whyrusleeping/tar-utils v0.0.0-20180509141711-8c6c8ba81d5c // indirect +) + require ( github.com/alecthomas/chroma v0.10.0 // indirect github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect @@ -44,6 +51,7 @@ require ( github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.1.2 // indirect github.com/ipfs/go-datastore v0.6.0 // indirect + github.com/ipfs/go-ipfs-api v0.6.0 github.com/ipfs/go-ipfs-util v0.0.3 // indirect github.com/ipfs/go-ipld-legacy v0.2.1 // indirect github.com/ipfs/go-log v1.0.5 // indirect diff --git a/go.sum b/go.sum index 8f440922..e0e0fee4 100644 --- a/go.sum +++ b/go.sum @@ -12,11 +12,13 @@ github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuP github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= +github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM273bISc= github.com/charmbracelet/glamour v0.6.0/go.mod h1:taqWV4swIMMbWALc0m7AfE9JkPSU8om2538k9ITBxOc= +github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= @@ -85,6 +87,8 @@ github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LK github.com/ipfs/go-datastore v0.6.0 h1:JKyz+Gvz1QEZw0LsX1IBn+JFCJQH4SJVFtM4uWU0Myk= github.com/ipfs/go-datastore v0.6.0/go.mod h1:rt5M3nNbSO/8q1t4LNkLyUwRs8HupMeN/8O4Vn9YAT8= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= +github.com/ipfs/go-ipfs-api v0.6.0 h1:JARgG0VTbjyVhO5ZfesnbXv9wTcMvoKRBLF1SzJqzmg= +github.com/ipfs/go-ipfs-api v0.6.0/go.mod h1:iDC2VMwN9LUpQV/GzEeZ2zNqd8NUdRmWcFM+K/6odf0= github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= github.com/ipfs/go-ipfs-cmds v0.9.0 h1:K0VcXg1l1k6aY6sHnoxYcyimyJQbcV1ueXuWgThmK9Q= github.com/ipfs/go-ipfs-cmds v0.9.0/go.mod h1:SBFHK8WNwC416QWH9Vz1Ql42SSMAOqKpaHUMBu3jpLo= @@ -140,6 +144,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38yPW7c= +github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= +github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= github.com/libp2p/go-libp2p v0.27.7 h1:nhMs03CRxslKkkK2uLuN8f72uwNkE6RJS1JFb3H9UIQ= github.com/libp2p/go-libp2p v0.27.7/go.mod h1:oMfQGTb9CHnrOuSM6yMmyK2lXz3qIhnkn2+oK3B1Y2g= github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= @@ -274,6 +280,9 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:X github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa h1:EyA027ZAkuaCLoxVX4r1TZMPy1d31fM6hbfQ4OU4I5o= github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= +github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= +github.com/whyrusleeping/tar-utils v0.0.0-20180509141711-8c6c8ba81d5c h1:GGsyl0dZ2jJgVT+VvWBf/cNijrHRhkrTjkmp5wg7li0= +github.com/whyrusleeping/tar-utils v0.0.0-20180509141711-8c6c8ba81d5c/go.mod h1:xxcJeBb7SIUl/Wzkz1eVKJE/CB34YNrqX2TQI6jY9zs= github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 h1:jxZvjx8Ve5sOXorZG0KzTxbp0Cr1n3FEegfmyd9br1k= github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5/go.mod h1:uxjoF2jEYT3+x+vC2KJddEGdk/LU8pRowXmyVMHSV5I= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/internal/filesystem/ipfs/files.go b/internal/filesystem/ipfs/files.go new file mode 100644 index 00000000..f79e92e6 --- /dev/null +++ b/internal/filesystem/ipfs/files.go @@ -0,0 +1,454 @@ +package ipfs + +import ( + "bytes" + "context" + "errors" + "io" + "io/fs" + "net/http" + "os" + "strings" + "time" + + "github.com/djdv/go-filesystem-utils/internal/filesystem" + fserrors "github.com/djdv/go-filesystem-utils/internal/filesystem/errors" + "github.com/djdv/go-filesystem-utils/internal/generic" + "github.com/ipfs/boxo/mfs" + shell "github.com/ipfs/go-ipfs-api" + "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr/net" +) + +type ( + FilesFS struct { + ctx context.Context + cancel context.CancelFunc + shell *shell.Shell + info nodeInfo + } + FilesOption func(*FilesFS) error + filesOptions []FilesOption + filesShared struct { + ctx context.Context + cancel context.CancelFunc + shell *shell.Shell + mountTime *time.Time // Substitute for mtime (no data in UFSv1). + path string + permissions fs.FileMode // (No data in UFSv1.) + } + filesDirectory struct { + filesShared + entries []*shell.MfsLsEntry + } + filesFile struct { + root *FilesFS + filesShared + cursor int64 + } + filesInfo struct{ nodeInfo } + filesEntry struct { + mountTime *time.Time // Substitute for mtime (no data in UFS 1). + *shell.MfsLsEntry + parent string + permissions fs.FileMode + } +) + +const ( + FilesID = "IPFSFiles" + filesRoot = "/" +) + +var ( + _ fs.FS = (*FilesFS)(nil) + _ fs.StatFS = (*FilesFS)(nil) + _ filesystem.MkdirFS = (*FilesFS)(nil) + _ filesystem.OpenFileFS = (*FilesFS)(nil) + _ filesystem.CreateFileFS = (*FilesFS)(nil) + _ filesystem.TruncateFileFS = (*FilesFS)(nil) + _ filesystem.RenameFS = (*FilesFS)(nil) + _ filesystem.RemoveFS = (*FilesFS)(nil) + _ fs.File = (*filesFile)(nil) + _ io.Writer = (*filesFile)(nil) + _ filesystem.TruncateFile = (*filesFile)(nil) + _ fs.ReadDirFile = (*filesDirectory)(nil) +) + +func NewFilesFS(ctx context.Context, maddr multiaddr.Multiaddr, options ...FilesOption) (*FilesFS, error) { + shell, err := filesClient(maddr) + if err != nil { + return nil, err + } + var ( + fsCtx, cancel = context.WithCancel(ctx) + fsys = FilesFS{ + ctx: fsCtx, + cancel: cancel, + shell: shell, + info: nodeInfo{ + name: rootName, + modTime: time.Now(), + mode: fs.ModeDir | + readAll | executeAll, + }, + } + ) + if err := generic.ApplyOptions(&fsys, options...); err != nil { + cancel() + return nil, err + } + return &fsys, nil +} + +func (fsys *FilesFS) setPermissions(permissions fs.FileMode) { + fsys.info.mode = fsys.info.mode.Type() | permissions.Perm() +} + +func (fsys *FilesFS) Open(name string) (fs.File, error) { + const op = "open" + var ( + shellName = fsToFilesShell(name) + info, err = filesStat( + fsys.ctx, op, + fsys.shell, shellName, + fsys.info.modTime, fsys.info.mode.Perm(), + ) + ) + if err != nil { + return nil, err + } + var ( + ctx, cancel = context.WithCancel(fsys.ctx) + shared = filesShared{ + ctx: ctx, cancel: cancel, + shell: fsys.shell, + path: shellName, + mountTime: &fsys.info.modTime, + permissions: fsys.info.mode.Perm(), + } + ) + if info.IsDir() { + return &filesDirectory{ + filesShared: shared, + }, nil + } + return &filesFile{ + root: fsys, + filesShared: shared, + }, nil +} + +func (fsys *FilesFS) Stat(name string) (fs.FileInfo, error) { + const op = "stat" + return filesStat( + fsys.ctx, op, + fsys.shell, fsToFilesShell(name), + fsys.info.modTime, fsys.info.mode.Perm(), + ) +} + +func (fsys *FilesFS) Mkdir(name string, perm fs.FileMode) error { + return fsys.shell.FilesMkdir(fsys.ctx, fsToFilesShell(name)) +} + +func (fsys *FilesFS) OpenFile(name string, flag int, perm fs.FileMode) (fs.File, error) { + const op = "openfile" + if flag&os.O_CREATE != 0 { + _, err := fsys.Stat(name) + exists := err == nil + if exists { + if flag&os.O_EXCL != 0 { + err = generic.ConstError("file exists but O_EXCL flag provided") + return nil, newFSError(op, name, err, fserrors.Exist) + } + return fsys.Open(name) + } + return fsys.CreateFile(name) + } + return fsys.Open(name) +} + +func (fsys *FilesFS) CreateFile(name string) (fs.File, error) { + err := fsys.shell.FilesWrite( + fsys.ctx, fsToFilesShell(name), bytes.NewReader(nil), + shell.FilesWrite.Create(true), + shell.FilesWrite.Truncate(true), + ) + if err != nil { + return nil, err + } + return fsys.Open(name) +} + +func (fsys *FilesFS) Truncate(name string, size int64) error { + file, err := fsys.Open(name) + if err != nil { + return err + } + var errs []error + if err := truncateFilesFile(fsys.ctx, fsys.shell, + file, fsToFilesShell(name), size); err != nil { + errs = append(errs, err) + } + if err := file.Close(); err != nil { + errs = append(errs, err) + } + return errors.Join(errs...) +} + +func (fsys *FilesFS) Rename(oldName, newName string) error { + return fsys.shell.FilesMv(fsys.ctx, + fsToFilesShell(oldName), fsToFilesShell(newName), + ) +} + +func (fsys *FilesFS) Remove(name string) error { + const force = true + return fsys.shell.FilesRm(fsys.ctx, fsToFilesShell(name), force) +} + +func (fsh *filesShared) Stat() (fs.FileInfo, error) { + const op = "stat" + return filesStat( + fsh.ctx, op, + fsh.shell, fsh.path, + *fsh.mountTime, fsh.permissions, + ) +} + +func (fi *filesInfo) Name() string { + return filesShellToFs(fi.name) +} + +func (fd *filesDirectory) Read([]byte) (int, error) { + const op = "filesDirectory.Read" + return -1, newFSError(op, filesShellToFs(fd.path), ErrIsDir, fserrors.IsDir) +} + +func (fd *filesDirectory) ReadDir(count int) ([]fs.DirEntry, error) { + mfsEnts := fd.entries + if mfsEnts == nil { + var err error + if mfsEnts, err = fd.shell.FilesLs(fd.ctx, fd.path, shell.FilesLs.Stat(true)); err != nil { + return nil, err + } + fd.entries = mfsEnts + } + limit := len(mfsEnts) + if limit == 0 && count > 0 { + return nil, io.EOF + } + if count > 0 && limit > count { + limit = count + } + entries := make([]fs.DirEntry, limit) + for i := range entries { + entries[i] = filesEntry{ + parent: fd.path, + mountTime: fd.mountTime, + permissions: fd.permissions, + MfsLsEntry: mfsEnts[i], + } + } + fd.entries = mfsEnts[limit:] + return entries, nil +} + +func (fd *filesDirectory) Close() error { fd.cancel(); return nil } + +func (fe filesEntry) Name() string { return fe.MfsLsEntry.Name } +func (fe filesEntry) IsDir() bool { return mfs.NodeType(fe.MfsLsEntry.Type) == mfs.TDir } +func (fe filesEntry) Type() (mode fs.FileMode) { + if fe.IsDir() { + mode |= fs.ModeDir + } + return mode +} + +func (fe filesEntry) Info() (fs.FileInfo, error) { + return &nodeInfo{ + modTime: *fe.mountTime, + name: fe.Name(), + size: int64(fe.MfsLsEntry.Size), + mode: fe.Type() | fe.permissions, + }, nil +} + +func (ff *filesFile) Read(p []byte) (int, error) { + reader, err := ff.shell.FilesRead( + ff.ctx, ff.path, + shell.FilesRead.Count(int64(len(p))), + shell.FilesRead.Offset(ff.cursor), + ) + if err != nil { + return -1, err + } + var errs []error + n, err := reader.Read(p) + if err != nil { + errs = append(errs, err) + } + if err := reader.Close(); err != nil { + errs = append(errs, err) + } + ff.cursor += int64(n) + return n, errors.Join(errs...) +} + +func (ff *filesFile) Truncate(size int64) error { + return ff.root.Truncate(filesShellToFs(ff.path), size) +} + +func (ff *filesFile) Write(p []byte) (int, error) { + err := ff.shell.FilesWrite( + ff.ctx, ff.path, + bytes.NewReader(p), shell.FilesWrite.Offset(ff.cursor), + ) + // TODO: concern + // we don't know how many bytes were actually written + // Is there any way to get the API to tell us? + if err != nil { + return -1, err + } + written := len(p) + ff.cursor += int64(written) + return written, nil +} + +func (ff *filesFile) Seek(offset int64, whence int) (int64, error) { + const op = "seek" + switch whence { + case io.SeekStart: + if offset < 0 { + err := generic.ConstError( + "tried to seek to a position before the beginning of the file", + ) + return -1, newFSError( + op, filesShellToFs(ff.path), + err, fserrors.InvalidItem, + ) + } + ff.cursor = offset + case io.SeekCurrent: + ff.cursor += offset + case io.SeekEnd: + info, err := ff.Stat() + if err != nil { + return -1, err + } + end := info.Size() + ff.cursor = end + offset + } + return ff.cursor, nil +} + +func (f *filesFile) Close() error { f.cancel(); return nil } + +func filesClient(apiMaddr multiaddr.Multiaddr) (*shell.Shell, error) { + address, client, err := prepareClientTransport(apiMaddr) + if err != nil { + return nil, err + } + return shell.NewShellWithClient(address, client), nil +} + +func prepareClientTransport(apiMaddr multiaddr.Multiaddr) (string, *http.Client, error) { + // TODO: magic number; decide on good timeout and const it. + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + resolvedMaddr, err := resolveMaddr(ctx, apiMaddr) + if err != nil { + return "", nil, err + } + + // TODO: I think the upstream package needs a patch to handle this internally. + // we'll hack around it for now. Investigate later. + // (When trying to use a unix socket for the IPFS maddr + // the client returned from httpapi.NewAPI will complain on requests - forgot to copy the error lol) + network, address, err := manet.DialArgs(resolvedMaddr) + if err != nil { + return "", nil, err + } + switch network { + default: + client := &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DisableKeepAlives: true, + }, + } + return address, client, nil + case "unix": + address, client := udsHTTPClient(address) + return address, client, nil + } +} + +func fsToFilesShell(name string) string { + if name == rootName { + return filesRoot + } + return filesRoot + name +} + +func filesShellToFs(name string) string { + if name == filesRoot { + return rootName + } + const leadingSlash = 1 + return name[leadingSlash:] +} + +func filesStat(ctx context.Context, op string, + sh *shell.Shell, path string, + modTime time.Time, permissions fs.FileMode, +) (*nodeInfo, error) { + stat, err := sh.FilesStat(ctx, path) + if err != nil { + var shellErr *shell.Error + if errors.As(err, &shellErr) { + if strings.Contains(shellErr.Message, "not exist") { + return nil, newFSError( + op, filesShellToFs(path), + err, fserrors.NotExist, + ) + } + } + return nil, newFSError(op, path, err, fserrors.IO) + } + mode := permissions.Perm() + if stat.Type == "directory" { + mode |= fs.ModeDir + } + return &nodeInfo{ + modTime: modTime, + name: path, + size: int64(stat.Size), + mode: mode, + }, nil +} + +func truncateFilesFile(ctx context.Context, + sh *shell.Shell, file fs.File, path string, size int64, +) error { + var ( + errs []error + buffer bytes.Buffer + _, err = io.CopyN(&buffer, file, size) + ) + buffer.Grow(int(size)) + if err != nil { + errs = append(errs, err) + } + if err := file.Close(); err != nil { + errs = append(errs, err) + } + if err := errors.Join(errs...); err != nil { + return err + } + return sh.FilesWrite( + ctx, path, &buffer, + shell.FilesWrite.Truncate(true), + ) +} From 605c3e5b26a56b8b6e544e50d35c90553201f819 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Wed, 12 Jul 2023 18:49:39 -0400 Subject: [PATCH 2/3] cgofuse: add Statfs support (WIP) --- internal/filesystem/cgofuse/stat.go | 13 +- internal/filesystem/cgofuse/stat_darwin.go | 33 ++++++ internal/filesystem/cgofuse/stat_freebsd.go | 35 ++++++ internal/filesystem/cgofuse/stat_openbsd.go | 35 ++++++ internal/filesystem/cgofuse/stat_unix.go | 37 ++++++ internal/filesystem/cgofuse/stat_windows.go | 125 ++++++++++++++++++++ 6 files changed, 277 insertions(+), 1 deletion(-) create mode 100644 internal/filesystem/cgofuse/stat_darwin.go create mode 100644 internal/filesystem/cgofuse/stat_freebsd.go create mode 100644 internal/filesystem/cgofuse/stat_openbsd.go create mode 100644 internal/filesystem/cgofuse/stat_unix.go create mode 100644 internal/filesystem/cgofuse/stat_windows.go diff --git a/internal/filesystem/cgofuse/stat.go b/internal/filesystem/cgofuse/stat.go index bef2e098..932f3870 100644 --- a/internal/filesystem/cgofuse/stat.go +++ b/internal/filesystem/cgofuse/stat.go @@ -8,7 +8,17 @@ import ( func (gw *goWrapper) Statfs(path string, stat *fuse.Statfs_t) errNo { defer gw.systemLock.Access(path)() - return -fuse.ENOSYS + // TODO: optional "freesize" on host init + // tracked in write, delete, etc. calls + // (^ lots of software checks for free space + // before even trying to call `write`, so we need to + // emulate that somehow) + errNo, err := statfs(path, stat) + if err != nil { + gw.logError(path, err) + return -fuse.EIO + } + return errNo } func (gw *goWrapper) Getattr(path string, stat *fuse.Stat_t, fh fileDescriptor) errNo { @@ -61,6 +71,7 @@ func (gw *goWrapper) infoFromPath(path string) (fs.FileInfo, error) { func (gw *goWrapper) Utimens(path string, tmsp []fuse.Timespec) errNo { defer gw.systemLock.Modify(path)() + return operationSuccess return -fuse.ENOSYS } diff --git a/internal/filesystem/cgofuse/stat_darwin.go b/internal/filesystem/cgofuse/stat_darwin.go new file mode 100644 index 00000000..7b032821 --- /dev/null +++ b/internal/filesystem/cgofuse/stat_darwin.go @@ -0,0 +1,33 @@ +package cgofuse + +import ( + "syscall" + + "github.com/winfsp/cgofuse/fuse" + "golang.org/x/sys/unix" +) + +func statfs(path string, fStatfs *fuse.Statfs_t) (int, error) { + // HACK: see note in stat_windows.go + path = "/" + sysStat := &unix.Statfs_t{} + if err := unix.Statfs(path, sysStat); err != nil { + if errno, ok := err.(syscall.Errno); ok { + return int(errno), err + } + return -fuse.EACCES, err + } + + // NOTE: These values are ignored by cgofuse + // but fsid might be incorrect on some platforms too + fStatfs.Fsid = uint64(sysStat.Fsid.Val[0])<<32 | uint64(sysStat.Fsid.Val[1]) + fStatfs.Flag = uint64(sysStat.Flags) + + fStatfs.Bsize = uint64(sysStat.Bsize) + fStatfs.Blocks = sysStat.Blocks + fStatfs.Bfree = sysStat.Bfree + fStatfs.Bavail = sysStat.Bavail + fStatfs.Files = sysStat.Files + fStatfs.Ffree = sysStat.Ffree + return operationSuccess, nil +} diff --git a/internal/filesystem/cgofuse/stat_freebsd.go b/internal/filesystem/cgofuse/stat_freebsd.go new file mode 100644 index 00000000..0d5b336f --- /dev/null +++ b/internal/filesystem/cgofuse/stat_freebsd.go @@ -0,0 +1,35 @@ +package cgofuse + +import ( + "syscall" + + "github.com/winfsp/cgofuse/fuse" + "golang.org/x/sys/unix" +) + +func statfs(path string, fStatfs *fuse.Statfs_t) (int, error) { + // HACK: see note in stat_windows.go + path = "/" + sysStat := &unix.Statfs_t{} + if err := unix.Statfs(path, sysStat); err != nil { + if errno, ok := err.(syscall.Errno); ok { + return int(errno), err + } + return -fuse.EACCES, err + } + + // NOTE: These values are ignored by cgofuse + // but fsid might be incorrect on some platforms too + fStatfs.Fsid = uint64(sysStat.Fsid.Val[0])<<32 | uint64(sysStat.Fsid.Val[1]) + fStatfs.Flag = uint64(sysStat.Flags) + + fStatfs.Bsize = sysStat.Bsize + fStatfs.Blocks = sysStat.Blocks + fStatfs.Bfree = sysStat.Bfree + fStatfs.Bavail = uint64(sysStat.Bavail) + fStatfs.Files = sysStat.Files + fStatfs.Ffree = uint64(sysStat.Ffree) + fStatfs.Frsize = uint64(sysStat.Bsize) + fStatfs.Namemax = uint64(sysStat.Namemax) + return operationSuccess, nil +} diff --git a/internal/filesystem/cgofuse/stat_openbsd.go b/internal/filesystem/cgofuse/stat_openbsd.go new file mode 100644 index 00000000..2ff5bce3 --- /dev/null +++ b/internal/filesystem/cgofuse/stat_openbsd.go @@ -0,0 +1,35 @@ +package cgofuse + +import ( + "syscall" + + "github.com/winfsp/cgofuse/fuse" + "golang.org/x/sys/unix" +) + +func statfs(path string, fStatfs *fuse.Statfs_t) (int, error) { + // HACK: see note in stat_windows.go + path = "/" + sysStat := &unix.Statfs_t{} + if err := unix.Statfs(path, sysStat); err != nil { + if errno, ok := err.(syscall.Errno); ok { + return int(errno), err + } + return -fuse.EACCES, err + } + + // NOTE: These values are ignored by cgofuse + // but fsid might be incorrect on some platforms too + fStatfs.Fsid = uint64(sysStat.F_fsid.Val[0])<<32 | uint64(sysStat.F_fsid.Val[1]) + fStatfs.Flag = uint64(sysStat.F_flags) + + fStatfs.Bsize = uint64(sysStat.F_bsize) + fStatfs.Blocks = sysStat.F_blocks + fStatfs.Bfree = sysStat.F_bfree + fStatfs.Bavail = uint64(sysStat.F_bavail) + fStatfs.Files = sysStat.F_files + fStatfs.Ffree = uint64(sysStat.F_ffree) + fStatfs.Frsize = uint64(sysStat.F_bsize) + fStatfs.Namemax = uint64(sysStat.F_namemax) + return operationSuccess, nil +} diff --git a/internal/filesystem/cgofuse/stat_unix.go b/internal/filesystem/cgofuse/stat_unix.go new file mode 100644 index 00000000..67b48640 --- /dev/null +++ b/internal/filesystem/cgofuse/stat_unix.go @@ -0,0 +1,37 @@ +//go:build !windows && !darwin && !freebsd && !openbsd && !netbsd + +package cgofuse + +import ( + "syscall" + + "github.com/winfsp/cgofuse/fuse" + "golang.org/x/sys/unix" +) + +func statfs(path string, fStatfs *fuse.Statfs_t) (int, error) { + // HACK: see note in stat_windows.go + path = "/" + sysStat := &unix.Statfs_t{} + if err := unix.Statfs(path, sysStat); err != nil { + if errno, ok := err.(syscall.Errno); ok { + return int(errno), err + } + return -fuse.EACCES, err + } + + // NOTE: These values are ignored by cgofuse + // but fsid might be incorrect on some platforms too + fStatfs.Fsid = uint64(sysStat.Fsid.Val[0])<<32 | uint64(sysStat.Fsid.Val[1]) + fStatfs.Flag = uint64(sysStat.Flags) + + fStatfs.Bsize = uint64(sysStat.Bsize) + fStatfs.Blocks = sysStat.Blocks + fStatfs.Bfree = sysStat.Bfree + fStatfs.Bavail = sysStat.Bavail + fStatfs.Files = sysStat.Files + fStatfs.Ffree = sysStat.Ffree + fStatfs.Frsize = uint64(sysStat.Frsize) + fStatfs.Namemax = uint64(sysStat.Namelen) + return operationSuccess, nil +} diff --git a/internal/filesystem/cgofuse/stat_windows.go b/internal/filesystem/cgofuse/stat_windows.go new file mode 100644 index 00000000..2f420649 --- /dev/null +++ b/internal/filesystem/cgofuse/stat_windows.go @@ -0,0 +1,125 @@ +package cgofuse + +import ( + "path/filepath" + "unsafe" + + "github.com/winfsp/cgofuse/fuse" + "golang.org/x/sys/windows" +) + +const LOAD_LIBRARY_SEARCH_SYSTEM32 = 0x00000800 + +// TODO review + +func loadSystemDLL(name string) (*windows.DLL, error) { + modHandle, err := windows.LoadLibraryEx(name, 0, LOAD_LIBRARY_SEARCH_SYSTEM32) + if err != nil { + return nil, err + } + return &windows.DLL{Name: name, Handle: modHandle}, nil +} + +func statfs(path string, fStatfs *fuse.Statfs_t) (int, error) { + mod, err := loadSystemDLL("kernel32.dll") + if err != nil { + return -fuse.ENOMEM, err // kind of true, probably better than EIO + } + defer mod.Release() + + proc, err := mod.FindProc("GetDiskFreeSpaceExW") + if err != nil { + return -fuse.ENOMEM, err // kind of true, probably better than EIO + } + + var ( + FreeBytesAvailableToCaller, + TotalNumberOfBytes, + TotalNumberOfFreeBytes uint64 + + SectorsPerCluster, + BytesPerSector uint16 + // NumberOfFreeClusters, + // TotalNumberOfClusters uint16 + ) + // HACK: + // Explorer gets upset when we try to write files + // to a mountpoint that has 0 free space. + // + // We need some way to place a host path + // on the fuse system, that we can use as a reference + // For IPFS, this can be the volume the node is on + // when it's remote this can be the system volume / flag overrrideable. + // We need some option for "infinite". + // + // better still would be a decimal value of "free space" + // which we can maintain during calls to write, rm, etc. + // the initial value could be derived from the above sources + // before being pased to the fuse interface. + sysdrive, err := windows.GetWindowsDirectory() + if err != nil { + return -fuse.EIO, err + } + path = sysdrive + pathPtr, err := windows.UTF16PtrFromString(path) + if err != nil { + return -fuse.EFAULT, err // caller should check for syscall.EINVAL; NUL byte was in string + } + + r1, _, wErr := proc.Call(uintptr(unsafe.Pointer(pathPtr)), + uintptr(unsafe.Pointer(&FreeBytesAvailableToCaller)), + uintptr(unsafe.Pointer(&TotalNumberOfBytes)), + uintptr(unsafe.Pointer(&TotalNumberOfFreeBytes)), + ) + if r1 == 0 { + return -fuse.ENOMEM, wErr + } + + proc, _ = mod.FindProc("GetDiskFreeSpaceW") + r1, _, wErr = proc.Call(uintptr(unsafe.Pointer(pathPtr)), + uintptr(unsafe.Pointer(&SectorsPerCluster)), + uintptr(unsafe.Pointer(&BytesPerSector)), + // uintptr(unsafe.Pointer(&NumberOfFreeClusters)), + 0, + // uintptr(unsafe.Pointer(&TotalNumberOfClusters)), + 0, + ) + if r1 == 0 { + return -fuse.EIO, wErr + } + + var ( + componentLimit = new(uint32) + volumeFlags = new(uint32) + volumeSerial = new(uint32) + ) + + volumeRoot := filepath.VolumeName(path) + string(filepath.Separator) + pathPtr, err = windows.UTF16PtrFromString(volumeRoot) + if err != nil { + return -fuse.EFAULT, err // caller should check for syscall.EINVAL; NUL byte was in string + } + + if err = windows.GetVolumeInformation(pathPtr, nil, 0, volumeSerial, componentLimit, volumeFlags, nil, 0); err != nil { + return -fuse.EIO, err + } + + fStatfs.Bsize = uint64(SectorsPerCluster * BytesPerSector) + fStatfs.Frsize = uint64(BytesPerSector) + fStatfs.Blocks = TotalNumberOfBytes / uint64(BytesPerSector) + fStatfs.Bfree = TotalNumberOfFreeBytes / (uint64(BytesPerSector)) + fStatfs.Bavail = FreeBytesAvailableToCaller / (uint64(BytesPerSector)) + fStatfs.Files = ^uint64(0) + + // TODO: these have to come from our own file table + // fStatfs.Ffree = nodeBinding.AvailableHandles() + // fStatfs.Favail = fStatfs.Ffree + + fStatfs.Namemax = uint64(*componentLimit) + + // cgofuse ignores these but we have them + fStatfs.Flag = uint64(*volumeFlags) + fStatfs.Fsid = uint64(*volumeSerial) + + return operationSuccess, nil +} From a695340234a5cc70055ef98da231cd2357f52749 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Wed, 12 Jul 2023 18:51:17 -0400 Subject: [PATCH 3/3] commands: mount - add IPFS Files API command (WIP) --- internal/commands/mountpoint_ipfs.go | 44 +++++++++++++++++++++++ internal/filesystem/ipfs/mountpoint.go | 50 +++++++++++++++++++++++--- 2 files changed, 90 insertions(+), 4 deletions(-) diff --git a/internal/commands/mountpoint_ipfs.go b/internal/commands/mountpoint_ipfs.go index 9561207a..35344c5f 100644 --- a/internal/commands/mountpoint_ipfs.go +++ b/internal/commands/mountpoint_ipfs.go @@ -32,6 +32,9 @@ type ( keyFSSettings ipfs.KeyFSGuest keyFSOption func(*keyFSSettings) error keyFSOptions []keyFSOption + filesSettings ipfs.FilesGuest + filesOption func(*filesSettings) error + filesOptions []filesOption ) const ( @@ -49,6 +52,7 @@ func makeIPFSCommands[ ](host filesystem.Host, ) []command.Command { return []command.Command{ + makeMountCommand[HC, HM, filesOptions, filesSettings](host, ipfs.FilesID), makeMountCommand[HC, HM, ipfsOptions, ipfsSettings](host, ipfs.IPFSID), makeMountCommand[HC, HM, pinFSOptions, pinFSSettings](host, ipfs.PinFSID), makeMountCommand[HC, HM, ipnsOptions, ipnsSettings](host, ipfs.IPNSID), @@ -65,6 +69,7 @@ func makeIPFSGuests[ guests[ipfs.IPNSID] = newMountPointFunc[HC, ipfs.IPNSGuest](path) guests[ipfs.KeyFSID] = newMountPointFunc[HC, ipfs.KeyFSGuest](path) guests[ipfs.PinFSID] = newMountPointFunc[HC, ipfs.PinFSGuest](path) + guests[ipfs.FilesID] = newMountPointFunc[HC, ipfs.FilesGuest](path) } func guestOverlayText(overlay, overlaid filesystem.ID) string { @@ -274,6 +279,45 @@ func (set keyFSSettings) marshal(string) ([]byte, error) { return json.Marshal(set) } +func (*filesOptions) usage(filesystem.Host) string { + return string(ipfs.FilesID) + " provides access to the node's FilesAPI." +} + +func (fo *filesOptions) BindFlags(flagSet *flag.FlagSet) { + var ( + flagPrefix = prefixIDFlag(ipfs.FilesID) + apiUsage = string(ipfs.FilesID) + " API node `maddr`" + apiName = flagPrefix + ipfsAPIFileName + ) + flagSetFunc(flagSet, apiName, apiUsage, fo, + func(value multiaddr.Multiaddr, settings *filesSettings) error { + settings.APIMaddr = value + return nil + }) +} + +func (fo filesOptions) make() (filesSettings, error) { + settings, err := makeWithOptions(fo...) + if err != nil { + return filesSettings{}, err + } + if settings.APIMaddr == nil { + maddrs, err := getIPFSAPI() + if err != nil { + return filesSettings{}, fmt.Errorf( + "could not get default value for API: %w", + err, + ) + } + settings.APIMaddr = maddrs[0] + } + return settings, nil +} + +func (set filesSettings) marshal(string) ([]byte, error) { + return json.Marshal(set) +} + func getIPFSAPI() ([]multiaddr.Multiaddr, error) { location, err := getIPFSAPIPath() if err != nil { diff --git a/internal/filesystem/ipfs/mountpoint.go b/internal/filesystem/ipfs/mountpoint.go index 955c3430..2ffdb094 100644 --- a/internal/filesystem/ipfs/mountpoint.go +++ b/internal/filesystem/ipfs/mountpoint.go @@ -1,6 +1,7 @@ package ipfs import ( + "context" "encoding/json" "errors" "io/fs" @@ -14,6 +15,10 @@ import ( ) type ( + // multiformats/go-multiaddr issue #100 + maddrWorkaround struct { + APIMaddr multiaddrContainer `json:"apiMaddr,omitempty"` + } IPFSGuest struct { APIMaddr multiaddr.Multiaddr `json:"apiMaddr,omitempty"` APITimeout time.Duration `json:"apiTimeout,omitempty"` @@ -29,15 +34,15 @@ type ( CacheExpiry time.Duration `json:"cacheExpiry,omitempty"` } KeyFSGuest struct{ IPNSGuest } + FilesGuest struct { + APIMaddr multiaddr.Multiaddr `json:"apiMaddr,omitempty"` + } ) func (*IPFSGuest) GuestID() filesystem.ID { return IPFSID } func (ig *IPFSGuest) UnmarshalJSON(b []byte) error { - // multiformats/go-multiaddr issue #100 - var maddrWorkaround struct { - APIMaddr multiaddrContainer `json:"apiMaddr,omitempty"` - } + var maddrWorkaround maddrWorkaround if err := json.Unmarshal(b, &maddrWorkaround); err != nil { return err } @@ -249,3 +254,40 @@ func (kg *KeyFSGuest) MakeFS() (fs.FS, error) { WithIPNS(ipnsFS), ) } + +func (fg *FilesGuest) GuestID() filesystem.ID { return FilesID } + +func (fg *FilesGuest) UnmarshalJSON(b []byte) error { + var maddrWorkaround maddrWorkaround + if err := json.Unmarshal(b, &maddrWorkaround); err != nil { + return err + } + fg.APIMaddr = maddrWorkaround.APIMaddr.Multiaddr + return nil +} + +func (fg *FilesGuest) MakeFS() (fs.FS, error) { + ctx := context.Background() + return NewFilesFS( + ctx, fg.APIMaddr, + WithPermissions[FilesOption](0o755), + ) +} + +func (fg *FilesGuest) ParseField(key, value string) error { + const apiKey = "apiMaddr" + var err error + switch key { + case apiKey: + var maddr multiaddr.Multiaddr + if maddr, err = multiaddr.NewMultiaddr(value); err == nil { + fg.APIMaddr = maddr + } + default: + return p9fs.FieldError{ + Key: key, + Tried: []string{apiKey}, + } + } + return err +}