Skip to content

Commit

Permalink
ipfs: use cached nodes when resolving paths
Browse files Browse the repository at this point in the history
  • Loading branch information
@djdv
djdv committed Sep 13, 2023
1 parent 99cef89 commit a4e8c5c
Show file tree
Hide file tree
Showing 4 changed files with 72 additions and 100 deletions.
16 changes: 11 additions & 5 deletions internal/filesystem/ipfs/ipfs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ import (
coreiface "github.com/ipfs/boxo/coreiface"
coreoptions "github.com/ipfs/boxo/coreiface/options"
corepath "github.com/ipfs/boxo/coreiface/path"
ipath "github.com/ipfs/boxo/path"
"github.com/ipfs/boxo/path/resolver"
"github.com/ipfs/go-cid"
cbor "github.com/ipfs/go-ipld-cbor"
ipld "github.com/ipfs/go-ipld-format"
Expand All @@ -30,6 +32,7 @@ type (
ctx context.Context
cancel context.CancelFunc
core coreiface.CoreAPI
resolver resolver.Resolver
nodeCache *ipfsNodeCache
dirCache *ipfsDirCache
info nodeInfo
Expand Down Expand Up @@ -76,6 +79,7 @@ func NewIPFS(core coreiface.CoreAPI, options ...IPFSOption) (*IPFS, error) {
fsys.cancel()
return nil, err
}
fsys.resolver = newPathResolver(fsys.getNode)
return fsys, nil
}

Expand Down Expand Up @@ -201,7 +205,7 @@ func (fsys *IPFS) toCID(op, goPath string) (cid.Cid, error) {
if len(names) == 1 {
return rootCID, nil
}
nodeCID, err := fsys.walkLinks(rootCID, names[1:])
nodeCID, err := fsys.resolvePath(goPath)
if err != nil {
kind := resolveErrKind(err)
return cid.Cid{}, fserrors.New(op, goPath, err, kind)
Expand Down Expand Up @@ -299,12 +303,14 @@ func (fsys *IPFS) nodeContext() (context.Context, context.CancelFunc) {
return context.WithTimeout(ctx, timeout)
}

func (fsys *IPFS) walkLinks(root cid.Cid, names []string) (cid.Cid, error) {
func (fsys *IPFS) resolvePath(goPath string) (cid.Cid, error) {
var (
ctx = fsys.ctx
resolver = newPathResolver(fsys.core)
ctx = fsys.ctx
resolver = fsys.resolver
iPath = ipath.FromString(goPath)
leaf, _, err = resolver.ResolveToLastNode(ctx, iPath)
)
return walkLinks(ctx, root, names, resolver)
return leaf, err
}

func (fsys *IPFS) Open(name string) (fs.File, error) {
Expand Down
46 changes: 33 additions & 13 deletions internal/filesystem/ipfs/ipns.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,10 @@ import (
lru "github.com/hashicorp/golang-lru/v2"
coreiface "github.com/ipfs/boxo/coreiface"
corepath "github.com/ipfs/boxo/coreiface/path"
ipath "github.com/ipfs/boxo/path"
"github.com/ipfs/boxo/path/resolver"
"github.com/ipfs/go-cid"
ipld "github.com/ipfs/go-ipld-format"
)

type (
Expand All @@ -27,6 +30,7 @@ type (
IPNS struct {
ctx context.Context
core coreiface.CoreAPI
resolver resolver.Resolver
ipfs fs.FS
cancel context.CancelFunc
rootCache *ipnsRootCache
Expand Down Expand Up @@ -187,27 +191,43 @@ func (fsys *IPNS) toCID(op, goPath string) (cid.Cid, error) {
return rootCID, nil
}
var (
leafCid cid.Cid
err error
)
// Re-use cid cache if available.
// Otherwise resolve all directly.
if ipfs, ok := fsys.ipfs.(*IPFS); ok {
leafCid, err = ipfs.walkLinks(rootCID, names[1:])
} else {
var (
ctx = fsys.ctx
resolver = newPathResolver(fsys.core)
components = append(
[]string{rootCID.String()},
names[1:]...,
)
leafCid, err = walkLinks(ctx, rootCID, names[1:], resolver)
}
ipfsPath = path.Join(components...)
leafCid, err = fsys.resolvePath(ipfsPath)
)
if err != nil {
kind := resolveErrKind(err)
return cid.Cid{}, fserrors.New(op, goPath, err, kind)
}
return leafCid, nil
}

func (fsys *IPNS) fetchNode(cid cid.Cid) (ipld.Node, error) {
ctx, cancel := fsys.nodeContext()
defer cancel()
return fsys.core.Dag().Get(ctx, cid)
}

func (fsys *IPNS) resolvePath(goPath string) (cid.Cid, error) {
if ipfs, ok := fsys.ipfs.(*IPFS); ok {
return ipfs.resolvePath(goPath)
}
resolver := fsys.resolver
if resolver == nil {
resolver = newPathResolver(fsys.fetchNode)
fsys.resolver = resolver
}
var (
ctx = fsys.ctx
iPath = ipath.FromString(goPath)
leaf, _, err = resolver.ResolveToLastNode(ctx, iPath)
)
return leaf, err
}

func (fsys *IPNS) nodeContext() (context.Context, context.CancelFunc) {
var (
ctx = fsys.ctx
Expand Down
89 changes: 28 additions & 61 deletions internal/filesystem/ipfs/resolve.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,35 +2,29 @@ package ipfs

import (
"context"
"io"

fserrors "github.com/djdv/go-filesystem-utils/internal/filesystem/errors"
"github.com/ipfs/boxo/blockservice"
coreiface "github.com/ipfs/boxo/coreiface"
ipath "github.com/ipfs/boxo/coreiface/path"
"github.com/ipfs/boxo/exchange"
bsfetcher "github.com/ipfs/boxo/fetcher/impl/blockservice"
"github.com/ipfs/boxo/path/resolver"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
ipld "github.com/ipfs/go-ipld-format"
"github.com/ipfs/go-unixfsnode"
dagpb "github.com/ipld/go-codec-dagpb"
)

type (
coreBlockStore struct {
blocks coreiface.BlockAPI
validate bool
}
blockFetcher struct {
coreiface.APIDagService
}
fnBlockStore getNodeFunc
fnBlockFetcher getNodeFunc
getNodeFunc func(cid cid.Cid) (ipld.Node, error)
)

func newPathResolver(api coreiface.CoreAPI) resolver.Resolver {
func newPathResolver(getNodeFn getNodeFunc) resolver.Resolver {
var (
blockstore = newCoreBlockStore(api.Block())
fetcher = makeBlockFetcher(api.Dag())
blockstore = newCoreBlockStore(getNodeFn)
fetcher = makeBlockFetcher(getNodeFn)
service = blockservice.New(blockstore, fetcher)
config = bsfetcher.NewFetcherConfig(service)
)
Expand All @@ -39,89 +33,62 @@ func newPathResolver(api coreiface.CoreAPI) resolver.Resolver {
return resolver.NewBasicResolver(fetcherFactory)
}

func newCoreBlockStore(blocks coreiface.BlockAPI) *coreBlockStore {
return &coreBlockStore{blocks: blocks}
}

func makeBlockFetcher(dag coreiface.APIDagService) exchange.Interface {
return blockFetcher{APIDagService: dag}
func newCoreBlockStore(getNodeFn getNodeFunc) fnBlockStore {
return fnBlockStore(getNodeFn)
}

func (bs *coreBlockStore) fetch(ctx context.Context, c cid.Cid) (blocks.Block, error) {
blockReader, err := bs.blocks.Get(ctx, ipath.IpfsPath(c))
if err != nil {
return nil, err
}
blockData, err := io.ReadAll(blockReader)
if err != nil {
return nil, err
}
if bs.validate {
nc, err := c.Prefix().Sum(blockData)
if err != nil {
return nil, blocks.ErrWrongHash
}
if !nc.Equals(c) {
return nil, blocks.ErrWrongHash
}
}
return blocks.NewBlockWithCid(blockData, c)
func makeBlockFetcher(getNodeFn getNodeFunc) exchange.Interface {
return fnBlockFetcher(getNodeFn)
}

func (bs *coreBlockStore) Has(ctx context.Context, c cid.Cid) (bool, error) {
blk, err := bs.fetch(ctx, c)
func (getNodeFn fnBlockStore) Has(_ context.Context, c cid.Cid) (bool, error) {
blk, err := getNodeFn(c)
if err != nil {
return false, err
}
return blk != nil, nil
}

func (bs *coreBlockStore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) {
blk, err := bs.fetch(ctx, c)
if err != nil {
return nil, err
}
return blk, nil
func (getNodeFn fnBlockStore) Get(_ context.Context, c cid.Cid) (blocks.Block, error) {
return getNodeFn(c)
}

func (bs *coreBlockStore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
blk, err := bs.fetch(ctx, c)
func (getNodeFn fnBlockStore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
blk, err := getNodeFn(c)
if err != nil {
return 0, err
}
return len(blk.RawData()), nil
}

func (bs *coreBlockStore) HashOnRead(enabled bool) {
bs.validate = enabled
}
func (fnBlockStore) HashOnRead(bool) {}

func (*coreBlockStore) Put(context.Context, blocks.Block) error {
func (fnBlockStore) Put(context.Context, blocks.Block) error {
return fserrors.ErrUnsupported
}

func (*coreBlockStore) PutMany(context.Context, []blocks.Block) error {
func (fnBlockStore) PutMany(context.Context, []blocks.Block) error {
return fserrors.ErrUnsupported
}

func (*coreBlockStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
func (fnBlockStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
return nil, fserrors.ErrUnsupported
}

func (*coreBlockStore) DeleteBlock(context.Context, cid.Cid) error {
func (fnBlockStore) DeleteBlock(context.Context, cid.Cid) error {
return fserrors.ErrUnsupported
}

func (bf blockFetcher) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) {
return bf.APIDagService.Get(ctx, c)
func (getNodeFn fnBlockFetcher) GetBlock(_ context.Context, c cid.Cid) (blocks.Block, error) {
return getNodeFn(c)
}

func (bf blockFetcher) GetBlocks(ctx context.Context, cids []cid.Cid) (<-chan blocks.Block, error) {
func (blockGetter fnBlockFetcher) GetBlocks(ctx context.Context, cids []cid.Cid) (<-chan blocks.Block, error) {
out := make(chan blocks.Block)
go func() {
defer close(out)
for _, c := range cids {
block, err := bf.GetBlock(ctx, c)
block, err := blockGetter.GetBlock(ctx, c)
if err != nil {
return
}
Expand All @@ -135,10 +102,10 @@ func (bf blockFetcher) GetBlocks(ctx context.Context, cids []cid.Cid) (<-chan bl
return out, nil
}

func (blockFetcher) NotifyNewBlocks(ctx context.Context, blocks ...blocks.Block) error {
func (fnBlockFetcher) NotifyNewBlocks(ctx context.Context, blocks ...blocks.Block) error {
return nil
}

func (blockFetcher) Close() error {
func (fnBlockFetcher) Close() error {
return nil
}
21 changes: 0 additions & 21 deletions internal/filesystem/ipfs/shared.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"errors"
"io"
"io/fs"
"path"
"strings"
"time"

Expand All @@ -16,9 +15,6 @@ import (
dag "github.com/ipfs/boxo/ipld/merkledag"
"github.com/ipfs/boxo/ipld/unixfs"
unixpb "github.com/ipfs/boxo/ipld/unixfs/pb"
ipath "github.com/ipfs/boxo/path"
"github.com/ipfs/boxo/path/resolver"
"github.com/ipfs/go-cid"
ipfscmds "github.com/ipfs/go-ipfs-cmds"
cbor "github.com/ipfs/go-ipld-cbor"
ipld "github.com/ipfs/go-ipld-format"
Expand Down Expand Up @@ -56,7 +52,6 @@ type (
modTime time.Time
permissions fs.FileMode
}
getNodeFunc func(cid.Cid) (ipld.Node, error)
)

const (
Expand Down Expand Up @@ -370,22 +365,6 @@ func drainThenSendErr(ch chan filesystem.StreamDirEntry, err error) {
ch <- newErrorEntry(err)
}

func walkLinks(ctx context.Context,
root cid.Cid, names []string,
resolver resolver.Resolver,
) (cid.Cid, error) {
var (
iPath = ipath.FromCid(root)
components = append(
[]string{string(iPath)},
names...,
)
nodePath = ipath.FromString(path.Join(components...))
)
leaf, _, err := resolver.ResolveToLastNode(ctx, nodePath)
return leaf, err
}

func fsTypeName(mode fs.FileMode) string {
switch mode.Type() {
case fs.FileMode(0):
Expand Down

0 comments on commit a4e8c5c

Please sign in to comment.