diff --git a/cmd/neofs-cli/modules/storagegroup/put.go b/cmd/neofs-cli/modules/storagegroup/put.go index e81e1fa3c2..feb3c94b1f 100644 --- a/cmd/neofs-cli/modules/storagegroup/put.go +++ b/cmd/neofs-cli/modules/storagegroup/put.go @@ -12,6 +12,7 @@ import ( "github.com/nspcc-dev/neofs-node/cmd/neofs-cli/internal/key" objectCli "github.com/nspcc-dev/neofs-node/cmd/neofs-cli/modules/object" "github.com/nspcc-dev/neofs-node/pkg/services/object_manager/storagegroup" + "github.com/nspcc-dev/neofs-sdk-go/client" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" "github.com/nspcc-dev/neofs-sdk-go/object" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" @@ -83,6 +84,7 @@ func putSG(cmd *cobra.Command, _ []string) { headPrm internalclient.HeadObjectPrm putPrm internalclient.PutObjectPrm getCnrPrm internalclient.GetContainerPrm + getPrm internalclient.GetObjectPrm ) cli := internalclient.GetSDKClientByFlag(ctx, cmd, commonflags.RPC) @@ -99,12 +101,19 @@ func putSG(cmd *cobra.Command, _ []string) { headPrm.SetClient(cli) headPrm.SetPrivateKey(*pk) + headPrm.SetRawFlag(true) + getPrm.SetClient(cli) + getPrm.SetPrivateKey(*pk) + objectCli.Prepare(cmd, &getPrm) + sg, err := storagegroup.CollectMembers(sgHeadReceiver{ ctx: ctx, cmd: cmd, key: pk, ownerID: &ownerID, - prm: headPrm, + prmHead: headPrm, + cli: cli, + getPrm: getPrm, }, cnr, members, !resGetCnr.Container().IsHomomorphicHashingDisabled()) common.ExitOnErr(cmd, "could not collect storage group members: %w", err) @@ -141,13 +150,40 @@ type sgHeadReceiver struct { cmd *cobra.Command key *ecdsa.PrivateKey ownerID *user.ID - prm internalclient.HeadObjectPrm + prmHead internalclient.HeadObjectPrm + cli *client.Client + getPrm internalclient.GetObjectPrm +} + +type payloadWriter struct { + payload []byte +} + +func (pw *payloadWriter) Write(p []byte) (n int, err error) { + pw.payload = append(pw.payload, p...) + return len(p), nil +} + +func (c sgHeadReceiver) Get(addr oid.Address) (object.Object, error) { + pw := &payloadWriter{} + c.getPrm.SetPayloadWriter(pw) + c.getPrm.SetAddress(addr) + + res, err := internalclient.GetObject(c.ctx, c.getPrm) + if err != nil { + return object.Object{}, fmt.Errorf("rpc error: %w", err) + } + + obj := res.Header() + obj.SetPayload(pw.payload) + + return *obj, nil } func (c sgHeadReceiver) Head(addr oid.Address) (any, error) { - c.prm.SetAddress(addr) + c.prmHead.SetAddress(addr) - res, err := internalclient.HeadObject(c.ctx, c.prm) + res, err := internalclient.HeadObject(c.ctx, c.prmHead) var errSplitInfo *object.SplitInfoError diff --git a/pkg/services/object/util/chain.go b/pkg/services/object/util/chain.go index db59061bb4..80f959d7c1 100644 --- a/pkg/services/object/util/chain.go +++ b/pkg/services/object/util/chain.go @@ -4,17 +4,21 @@ import ( "errors" "fmt" + cid "github.com/nspcc-dev/neofs-sdk-go/container/id" "github.com/nspcc-dev/neofs-sdk-go/object" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" ) -// HeadReceiver is an interface of entity that can receive +// SplitInformer is an interface of entity that can receive // object header or the information about the object relations. -type HeadReceiver interface { +type SplitInformer interface { // Head must return one of: // * object header (*object.Object); // * structured information about split-chain (*object.SplitInfo). Head(id oid.Address) (any, error) + + // Get must return object by its address. + Get(address oid.Address) (object.Object, error) } // SplitMemberHandler is a handler of next split-chain element. @@ -24,7 +28,7 @@ type HeadReceiver interface { type SplitMemberHandler func(member *object.Object, reverseDirection bool) (stop bool) // IterateAllSplitLeaves is an iterator over all object split-tree leaves in direct order. -func IterateAllSplitLeaves(r HeadReceiver, addr oid.Address, h func(*object.Object)) error { +func IterateAllSplitLeaves(r SplitInformer, addr oid.Address, h func(*object.Object)) error { return IterateSplitLeaves(r, addr, func(leaf *object.Object) bool { h(leaf) return false @@ -34,45 +38,178 @@ func IterateAllSplitLeaves(r HeadReceiver, addr oid.Address, h func(*object.Obje // IterateSplitLeaves is an iterator over object split-tree leaves in direct order. // // If member handler returns true, then the iterator aborts without error. -func IterateSplitLeaves(r HeadReceiver, addr oid.Address, h func(*object.Object) bool) error { - var ( - reverse bool - leaves []*object.Object - ) - - if err := TraverseSplitChain(r, addr, func(member *object.Object, reverseDirection bool) (stop bool) { - reverse = reverseDirection - - if reverse { - leaves = append(leaves, member) - return false +func IterateSplitLeaves(r SplitInformer, addr oid.Address, h func(*object.Object) bool) error { + info, err := r.Head(addr) + if err != nil { + return fmt.Errorf("receiving information about the object: %w", err) + } + + switch res := info.(type) { + default: + panic(fmt.Sprintf("unexpected result of %T: %T", r, info)) + case *object.Object: + h(res) + case *object.SplitInfo: + if res.SplitID() == nil { + return iterateV2Split(r, res, addr.Container(), h) + } else { + return iterateV1Split(r, res, addr.Container(), h) } + } - return h(member) - }); err != nil { - return err + return nil +} + +func iterateV1Split(r SplitInformer, info *object.SplitInfo, cID cid.ID, handler func(*object.Object) bool) error { + var addr oid.Address + addr.SetContainer(cID) + + linkID, ok := info.Link() + if ok { + addr.SetObject(linkID) + + linkObj, err := headFromInformer(r, addr) + if err != nil { + return err + } + + for _, child := range linkObj.Children() { + addr.SetObject(child) + + childHeader, err := headFromInformer(r, addr) + if err != nil { + return err + } + + if stop := handler(childHeader); stop { + return nil + } + } + + handler(linkObj) + + return nil + } + + lastID, ok := info.LastPart() + if ok { + addr.SetObject(lastID) + return iterateFromLastObject(r, addr, handler) } - for i := len(leaves) - 1; i >= 0; i-- { - if h(leaves[i]) { + return errors.New("nor link, nor last object ID is found") +} + +func iterateV2Split(r SplitInformer, info *object.SplitInfo, cID cid.ID, handler func(*object.Object) bool) error { + var addr oid.Address + addr.SetContainer(cID) + + linkID, ok := info.Link() + if ok { + addr.SetObject(linkID) + + linkObjRaw, err := r.Get(addr) + if err != nil { + return fmt.Errorf("receiving link object %s: %w", addr, err) + } + + if stop := handler(&linkObjRaw); stop { + return nil + } + + var linkObj object.Link + err = linkObjRaw.ReadLink(&linkObj) + if err != nil { + return fmt.Errorf("decoding link object: %w", err) + } + + for _, child := range linkObj.Objects() { + addr.SetObject(child.ObjectID()) + + childObj, err := headFromInformer(r, addr) + if err != nil { + return fmt.Errorf("fetching child object: %w", err) + } + + if stop := handler(childObj); stop { + return nil + } + } + + return nil + } + + lastID, ok := info.LastPart() + if ok { + addr.SetObject(lastID) + return iterateFromLastObject(r, addr, handler) + } + + return errors.New("nor link, nor last object ID is found") +} + +func iterateFromLastObject(r SplitInformer, lastAddr oid.Address, handler func(*object.Object) bool) error { + var idBuff []oid.ID + addr := lastAddr + + for { + obj, err := headFromInformer(r, addr) + if err != nil { + return err + } + + oID, _ := obj.ID() + idBuff = append(idBuff, oID) + + prevOID, set := obj.PreviousID() + if !set { break } + + addr.SetObject(prevOID) + } + + for i := len(idBuff) - 1; i >= 0; i-- { + addr.SetObject(idBuff[i]) + + childObj, err := headFromInformer(r, addr) + if err != nil { + return err + } + + if stop := handler(childObj); stop { + return nil + } } return nil } +func headFromInformer(r SplitInformer, addr oid.Address) (*object.Object, error) { + res, err := r.Head(addr) + if err != nil { + return nil, fmt.Errorf("fetching information about %s", addr) + } + + switch v := res.(type) { + case *object.Object: + return v, nil + default: + return nil, fmt.Errorf("unexpected information: %T", res) + } +} + // TraverseSplitChain is an iterator over object split-tree leaves. // // Traversal occurs in one of two directions, which depends on what pslit info was received: // * in direct order for link part; // * in reverse order for last part. -func TraverseSplitChain(r HeadReceiver, addr oid.Address, h SplitMemberHandler) error { +func TraverseSplitChain(r SplitInformer, addr oid.Address, h SplitMemberHandler) error { _, err := traverseSplitChain(r, addr, h) return err } -func traverseSplitChain(r HeadReceiver, addr oid.Address, h SplitMemberHandler) (bool, error) { +func traverseSplitChain(r SplitInformer, addr oid.Address, h SplitMemberHandler) (bool, error) { v, err := r.Head(addr) if err != nil { return false, err diff --git a/pkg/services/object_manager/storagegroup/collect.go b/pkg/services/object_manager/storagegroup/collect.go index 8e8483d8a5..9458dc5a79 100644 --- a/pkg/services/object_manager/storagegroup/collect.go +++ b/pkg/services/object_manager/storagegroup/collect.go @@ -20,10 +20,10 @@ var ( ) // CollectMembers creates new storage group structure and fills it -// with information about members collected via HeadReceiver. +// with information about members collected via SplitInformer. // // Resulting storage group consists of physically stored objects only. -func CollectMembers(r objutil.HeadReceiver, cnr cid.ID, members []oid.ID, calcHomoHash bool) (*storagegroup.StorageGroup, error) { +func CollectMembers(r objutil.SplitInformer, cnr cid.ID, members []oid.ID, calcHomoHash bool) (*storagegroup.StorageGroup, error) { var ( sumPhySize uint64 phyMembers []oid.ID diff --git a/pkg/services/object_manager/storagegroup/collect_test.go b/pkg/services/object_manager/storagegroup/collect_test.go index c5e63f6a5d..e075c8a2f2 100644 --- a/pkg/services/object_manager/storagegroup/collect_test.go +++ b/pkg/services/object_manager/storagegroup/collect_test.go @@ -16,6 +16,10 @@ type mockedObjects struct { hdr *object.Object } +func (x *mockedObjects) Get(address oid.Address) (object.Object, error) { + return *x.hdr, nil +} + func (x *mockedObjects) Head(_ oid.Address) (any, error) { return x.hdr, nil }