From 65acd76b29a92e6a2a5991dd3f7e30194a77550c Mon Sep 17 00:00:00 2001 From: CHAMI Rachid Date: Fri, 6 Dec 2024 13:51:06 +0100 Subject: [PATCH] feat!: unify the blob.Proof and the CommitmentProof (#3821) --- blob/blob.go | 165 ++++++++++++++++------ blob/blob_fuzz_test.go | 10 +- blob/commitment_proof.go | 155 --------------------- blob/repro_test.go | 35 ++--- blob/service.go | 245 +++++---------------------------- blob/service_test.go | 210 +++++++++++++++------------- nodebuilder/blob/blob.go | 24 +--- nodebuilder/blob/mocks/api.go | 15 -- nodebuilder/share/mocks/api.go | 4 +- nodebuilder/share/share.go | 40 ++++-- 10 files changed, 318 insertions(+), 585 deletions(-) delete mode 100644 blob/commitment_proof.go diff --git a/blob/blob.go b/blob/blob.go index e207086386..76635d6539 100644 --- a/blob/blob.go +++ b/blob/blob.go @@ -11,8 +11,6 @@ import ( "github.com/celestiaorg/go-square/v2/inclusion" libshare "github.com/celestiaorg/go-square/v2/share" "github.com/celestiaorg/nmt" - "github.com/tendermint/tendermint/pkg/consts" - tmproto "github.com/tendermint/tendermint/proto/tendermint/types" coretypes "github.com/tendermint/tendermint/types" ) @@ -20,74 +18,151 @@ var errEmptyShares = errors.New("empty shares") // Proof constructs the proof of a blob to the data root. type Proof struct { - // ShareToRowRootProof the proofs of the shares to the row roots they belong to. + // SubtreeRoots are the subtree roots of the blob's data that are + // used to create the commitment. + SubtreeRoots [][]byte `json:"subtree_roots"` + // SubtreeRootProofs the proofs of the subtree roots to the row roots they belong to. // If the blob spans across multiple rows, then this will contain multiple proofs. - ShareToRowRootProof []*tmproto.NMTProof + SubtreeRootProofs []*nmt.Proof `json:"share_to_row_root_proofs"` // RowToDataRootProof the proofs of the row roots containing the blob shares // to the data root. - RowToDataRootProof coretypes.RowProof + RowToDataRootProof coretypes.RowProof `json:"row_to_data_root_proof"` } // namespaceToRowRootProof a proof of a set of namespace shares to the row // roots they belong to. type namespaceToRowRootProof []*nmt.Proof -// Verify takes a blob and a data root and verifies if the -// provided blob was committed to the given data root. -func (p *Proof) Verify(blob *Blob, dataRoot []byte) (bool, error) { - blobCommitment, err := inclusion.CreateCommitment(blob.Blob, merkle.HashFromByteSlices, appconsts.DefaultSubtreeRootThreshold) - if err != nil { - return false, err +// Commitment is a Merkle Root of the subtree built from shares of the Blob. +// It is computed by splitting the blob into shares and building the Merkle subtree to be included +// after Submit. +type Commitment []byte + +// Verify takes a data root and verifies if the +// provided proof's subtree roots were committed to the given data root. +func (p *Proof) Verify(dataRoot []byte) (bool, error) { + if len(dataRoot) == 0 { + return false, errors.New("root must be non-empty") + } + + subtreeRootThreshold := appconsts.SubtreeRootThreshold(appconsts.LatestVersion) + if subtreeRootThreshold <= 0 { + return false, errors.New("subtreeRootThreshold must be > 0") } - if !blob.Commitment.Equal(blobCommitment) { + + // this check is < instead of != because we can have two subtree roots + // at the same height, depending on the subtree root threshold, + // and they can be used to create the above inner node without needing a proof inner node. + if len(p.SubtreeRoots) < len(p.SubtreeRootProofs) { return false, fmt.Errorf( - "%w: generated commitment does not match the provided blob commitment", - ErrMismatchCommitment, + "the number of subtree roots %d should be bigger than the number of subtree root proofs %d", + len(p.SubtreeRoots), + len(p.SubtreeRootProofs), ) } - shares, err := BlobsToShares(blob) - if err != nil { - return false, err + + // for each row, one or more subtree roots' inclusion is verified against + // their corresponding row root. then, these row roots' inclusion is verified + // against the data root. so their number should be the same. + if len(p.SubtreeRootProofs) != len(p.RowToDataRootProof.Proofs) { + return false, fmt.Errorf( + "the number of subtree root proofs %d should be equal to the number of row root proofs %d", + len(p.SubtreeRootProofs), + len(p.RowToDataRootProof.Proofs), + ) + } + + // the row root proofs' ranges are defined as [startRow, endRow]. + if int(p.RowToDataRootProof.EndRow-p.RowToDataRootProof.StartRow+1) != len(p.RowToDataRootProof.RowRoots) { + return false, fmt.Errorf( + "the number of rows %d must equal the number of row roots %d", + int(p.RowToDataRootProof.EndRow-p.RowToDataRootProof.StartRow+1), + len(p.RowToDataRootProof.RowRoots), + ) + } + if len(p.RowToDataRootProof.Proofs) != len(p.RowToDataRootProof.RowRoots) { + return false, fmt.Errorf( + "the number of proofs %d must equal the number of row roots %d", + len(p.RowToDataRootProof.Proofs), + len(p.RowToDataRootProof.RowRoots), + ) } - return p.VerifyShares(libshare.ToBytes(shares), blob.Namespace(), dataRoot) -} -// VerifyShares takes a set of shares, a namespace and a data root, and verifies if the -// provided shares are committed to by the data root. -func (p *Proof) VerifyShares(rawShares [][]byte, namespace libshare.Namespace, dataRoot []byte) (bool, error) { - // verify the row proof + // verify the inclusion of the rows to the data root if err := p.RowToDataRootProof.Validate(dataRoot); err != nil { - return false, fmt.Errorf("%w: invalid row root to data root proof", err) + return false, err + } + + // computes the total number of shares proven given that each subtree root + // references a specific set of leaves. + numberOfShares := 0 + for _, proof := range p.SubtreeRootProofs { + numberOfShares += proof.End() - proof.Start() } - // verify the share proof - ns := append([]byte{namespace.Version()}, namespace.ID()...) - cursor := int32(0) - for i, proof := range p.ShareToRowRootProof { - sharesUsed := proof.End - proof.Start - if len(rawShares) < int(sharesUsed+cursor) { - return false, fmt.Errorf("%w: invalid number of shares", ErrInvalidProof) + // use the computed total number of shares to calculate the subtree roots + // width. + // the subtree roots width is defined in ADR-013: + // + //https://github.com/celestiaorg/celestia-app/blob/main/docs/architecture/adr-013-non-interactive-default-rules-for-zero-padding.md + subtreeRootsWidth := inclusion.SubTreeWidth(numberOfShares, subtreeRootThreshold) + + nmtHasher := nmt.NewNmtHasher(appconsts.NewBaseHashFunc(), libshare.NamespaceSize, true) + // verify the proof of the subtree roots + subtreeRootsCursor := 0 + for i, subtreeRootProof := range p.SubtreeRootProofs { + // calculate the share range that each subtree root commits to. + ranges, err := nmt.ToLeafRanges(subtreeRootProof.Start(), subtreeRootProof.End(), subtreeRootsWidth) + if err != nil { + return false, err } - nmtProof := nmt.NewInclusionProof( - int(proof.Start), - int(proof.End), - proof.Nodes, - true, - ) - valid := nmtProof.VerifyInclusion( - consts.NewBaseHashFunc(), - ns, - rawShares[cursor:sharesUsed+cursor], + + if len(p.SubtreeRoots) < subtreeRootsCursor { + return false, fmt.Errorf("len(commitmentProof.SubtreeRoots)=%d < subtreeRootsCursor=%d", + len(p.SubtreeRoots), subtreeRootsCursor) + } + if len(p.SubtreeRoots) < subtreeRootsCursor+len(ranges) { + return false, fmt.Errorf("len(commitmentProof.SubtreeRoots)=%d < subtreeRootsCursor+len(ranges)=%d", + len(p.SubtreeRoots), subtreeRootsCursor+len(ranges)) + } + valid, err := subtreeRootProof.VerifySubtreeRootInclusion( + nmtHasher, + p.SubtreeRoots[subtreeRootsCursor:subtreeRootsCursor+len(ranges)], + subtreeRootsWidth, p.RowToDataRootProof.RowRoots[i], ) + if err != nil { + return false, err + } if !valid { - return false, ErrInvalidProof + return false, + fmt.Errorf( + "subtree root proof for range [%d, %d) is invalid", + subtreeRootProof.Start(), + subtreeRootProof.End(), + ) } - cursor += sharesUsed + subtreeRootsCursor += len(ranges) } + return true, nil } +// GenerateCommitment generates the share commitment corresponding +// to the proof's subtree roots +func (p *Proof) GenerateCommitment() Commitment { + return merkle.HashFromByteSlices(p.SubtreeRoots) +} + +func (com Commitment) String() string { + return string(com) +} + +// Equal ensures that commitments are the same +func (com Commitment) Equal(c Commitment) bool { + return bytes.Equal(com, c) +} + // Blob represents any application-specific binary data that anyone can submit to Celestia. type Blob struct { *libshare.Blob `json:"blob"` @@ -210,6 +285,10 @@ func (b *Blob) UnmarshalJSON(data []byte) error { return nil } +func (b *Blob) ComputeSubtreeRoots() ([][]byte, error) { + return inclusion.GenerateSubtreeRoots(b.Blob, appconsts.DefaultSubtreeRootThreshold) +} + // proveRowRootsToDataRoot creates a set of binary merkle proofs for all the // roots defined by the range [start, end). func proveRowRootsToDataRoot(roots [][]byte, start, end int) []*merkle.Proof { diff --git a/blob/blob_fuzz_test.go b/blob/blob_fuzz_test.go index 1927ff3d46..1a9fd5f330 100644 --- a/blob/blob_fuzz_test.go +++ b/blob/blob_fuzz_test.go @@ -41,9 +41,9 @@ func FuzzProofEqual(f *testing.F) { } type verifyCorpus struct { - CP *CommitmentProof `json:"commitment_proof"` - Root []byte `json:"root"` - SThreshold int `json:"sub_threshold"` + Proof *Proof `json:"proof"` + Root []byte `json:"root"` + SThreshold int `json:"sub_threshold"` } func FuzzCommitmentProofVerify(f *testing.F) { @@ -83,10 +83,10 @@ func FuzzCommitmentProofVerify(f *testing.F) { if err := json.Unmarshal(valueJSON, val); err != nil { return } - commitProof := val.CP + commitProof := val.Proof if commitProof == nil { return } - _, _ = commitProof.Verify(val.Root, val.SThreshold) + _, _ = commitProof.Verify(val.Root) }) } diff --git a/blob/commitment_proof.go b/blob/commitment_proof.go deleted file mode 100644 index 887f420552..0000000000 --- a/blob/commitment_proof.go +++ /dev/null @@ -1,155 +0,0 @@ -package blob - -import ( - "bytes" - "errors" - "fmt" - - "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" - "github.com/celestiaorg/celestia-app/v3/pkg/proof" - "github.com/celestiaorg/go-square/v2/inclusion" - libshare "github.com/celestiaorg/go-square/v2/share" - "github.com/celestiaorg/nmt" - "github.com/celestiaorg/nmt/namespace" -) - -// Commitment is a Merkle Root of the subtree built from shares of the Blob. -// It is computed by splitting the blob into shares and building the Merkle subtree to be included -// after Submit. -type Commitment []byte - -// CommitmentProof is an inclusion proof of a commitment to the data root. -type CommitmentProof struct { - // SubtreeRoots are the subtree roots of the blob's data that are - // used to create the commitment. - SubtreeRoots [][]byte `json:"subtree_roots"` - // SubtreeRootProofs are the NMT proofs for the subtree roots - // to the row roots. - SubtreeRootProofs []*nmt.Proof `json:"subtree_root_proofs"` - // NamespaceID is the namespace id of the commitment being proven. This - // namespace id is used when verifying the proof. If the namespace id doesn't - // match the namespace of the shares, the proof will fail verification. - NamespaceID namespace.ID `json:"namespace_id"` - // RowProof is the proof of the rows containing the blob's data to the - // data root. - RowProof proof.RowProof `json:"row_proof"` - NamespaceVersion uint8 `json:"namespace_version"` -} - -func (com Commitment) String() string { - return string(com) -} - -// Equal ensures that commitments are the same -func (com Commitment) Equal(c Commitment) bool { - return bytes.Equal(com, c) -} - -// Validate performs basic validation to the commitment proof. -// Note: it doesn't verify if the proof is valid or not. -// Check Verify() for that. -func (commitmentProof *CommitmentProof) Validate() error { - if len(commitmentProof.SubtreeRoots) < len(commitmentProof.SubtreeRootProofs) { - return fmt.Errorf( - "the number of subtree roots %d should be bigger than the number of subtree root proofs %d", - len(commitmentProof.SubtreeRoots), - len(commitmentProof.SubtreeRootProofs), - ) - } - if len(commitmentProof.SubtreeRootProofs) != len(commitmentProof.RowProof.Proofs) { - return fmt.Errorf( - "the number of subtree root proofs %d should be equal to the number of row root proofs %d", - len(commitmentProof.SubtreeRootProofs), - len(commitmentProof.RowProof.Proofs), - ) - } - if int(commitmentProof.RowProof.EndRow-commitmentProof.RowProof.StartRow+1) != len(commitmentProof.RowProof.RowRoots) { - return fmt.Errorf( - "the number of rows %d must equal the number of row roots %d", - int(commitmentProof.RowProof.EndRow-commitmentProof.RowProof.StartRow+1), - len(commitmentProof.RowProof.RowRoots), - ) - } - if len(commitmentProof.RowProof.Proofs) != len(commitmentProof.RowProof.RowRoots) { - return fmt.Errorf( - "the number of proofs %d must equal the number of row roots %d", - len(commitmentProof.RowProof.Proofs), - len(commitmentProof.RowProof.RowRoots), - ) - } - return nil -} - -// Verify verifies that a commitment proof is valid, i.e., the subtree roots commit -// to some data that was posted to a square. -// Expects the commitment proof to be properly formulated and validated -// using the Validate() function. -func (commitmentProof *CommitmentProof) Verify(root []byte, subtreeRootThreshold int) (bool, error) { - if len(root) == 0 { - return false, errors.New("root must be non-empty") - } - - rp := commitmentProof.RowProof - if err := rp.Validate(root); err != nil { - return false, err - } - - nmtHasher := nmt.NewNmtHasher(appconsts.NewBaseHashFunc(), libshare.NamespaceSize, true) - - // computes the total number of shares proven. - numberOfShares := 0 - for _, proof := range commitmentProof.SubtreeRootProofs { - numberOfShares += proof.End() - proof.Start() - } - - if subtreeRootThreshold <= 0 { - return false, errors.New("subtreeRootThreshould must be > 0") - } - - // use the computed total number of shares to calculate the subtree roots - // width. - // the subtree roots width is defined in ADR-013: - // - //https://github.com/celestiaorg/celestia-app/blob/main/docs/architecture/adr-013-non-interactive-default-rules-for-zero-padding.md - subtreeRootsWidth := inclusion.SubTreeWidth(numberOfShares, subtreeRootThreshold) - - // verify the proof of the subtree roots - subtreeRootsCursor := 0 - for i, subtreeRootProof := range commitmentProof.SubtreeRootProofs { - // calculate the share range that each subtree root commits to. - ranges, err := nmt.ToLeafRanges(subtreeRootProof.Start(), subtreeRootProof.End(), subtreeRootsWidth) - if err != nil { - return false, err - } - - if len(commitmentProof.SubtreeRoots) < subtreeRootsCursor { - return false, fmt.Errorf("len(commitmentProof.SubtreeRoots)=%d < subtreeRootsCursor=%d", - len(commitmentProof.SubtreeRoots), subtreeRootsCursor) - } - if len(commitmentProof.SubtreeRoots) < subtreeRootsCursor+len(ranges) { - return false, fmt.Errorf("len(commitmentProof.SubtreeRoots)=%d < subtreeRootsCursor+len(ranges)=%d", - len(commitmentProof.SubtreeRoots), subtreeRootsCursor+len(ranges)) - } - valid, err := subtreeRootProof.VerifySubtreeRootInclusion( - nmtHasher, - commitmentProof.SubtreeRoots[subtreeRootsCursor:subtreeRootsCursor+len(ranges)], - subtreeRootsWidth, - commitmentProof.RowProof.RowRoots[i], - ) - if err != nil { - return false, err - } - if !valid { - return false, - fmt.Errorf( - "subtree root proof for range [%d, %d) is invalid", - subtreeRootProof.Start(), - subtreeRootProof.End(), - ) - } - subtreeRootsCursor += len(ranges) - } - - // verify row roots to data root proof - return commitmentProof.RowProof.VerifyProof(root), nil -} diff --git a/blob/repro_test.go b/blob/repro_test.go index 6cb39948ac..d3161f9c59 100644 --- a/blob/repro_test.go +++ b/blob/repro_test.go @@ -3,32 +3,33 @@ package blob import ( "testing" - "github.com/celestiaorg/celestia-app/v3/pkg/proof" + "github.com/celestiaorg/go-square/merkle" "github.com/celestiaorg/nmt" "github.com/celestiaorg/nmt/pb" + coretypes "github.com/tendermint/tendermint/types" ) // Reported at https://github.com/celestiaorg/celestia-node/issues/3731. -func TestCommitmentProofRowProofVerifyWithEmptyRoot(t *testing.T) { - cp := &CommitmentProof{ - RowProof: proof.RowProof{ - Proofs: []*proof.Proof{{}}, +func TestProofRowProofVerifyWithEmptyRoot(t *testing.T) { + cp := &Proof{ + RowToDataRootProof: coretypes.RowProof{ + Proofs: []*merkle.Proof{{}}, }, } root := []byte{0xd3, 0x4d, 0x34} - if _, err := cp.Verify(root, 1); err == nil { + if _, err := cp.Verify(root); err == nil { t.Fatal("expected a non-nil error") } } // Reported at https://github.com/celestiaorg/celestia-node/issues/3730. -func TestCommitmentProofRowProofVerify(t *testing.T) { - cp := &CommitmentProof{ - RowProof: proof.RowProof{ - Proofs: []*proof.Proof{{}}, +func TestProofRowProofVerify(t *testing.T) { + cp := &Proof{ + RowToDataRootProof: coretypes.RowProof{ + Proofs: []*merkle.Proof{{}}, }, } - if _, err := cp.Verify(nil, 1); err == nil { + if _, err := cp.Verify(nil); err == nil { t.Fatal("expected a non-nil error") } } @@ -36,20 +37,12 @@ func TestCommitmentProofRowProofVerify(t *testing.T) { // Reported at https://github.com/celestiaorg/celestia-node/issues/3729. func TestCommitmentProofVerifySliceBound(t *testing.T) { proof := nmt.ProtoToProof(pb.Proof{End: 1}) - cp := &CommitmentProof{ + cp := &Proof{ SubtreeRootProofs: []*nmt.Proof{ &proof, }, } - if _, err := cp.Verify(nil, 1); err == nil { - t.Fatal("expected a non-nil error") - } -} - -// Reported at https://github.com/celestiaorg/celestia-node/issues/3728. -func TestCommitmentProofVerifyZeroSubThreshold(t *testing.T) { - cp := new(CommitmentProof) - if _, err := cp.Verify(nil, 0); err == nil { + if _, err := cp.Verify(nil); err == nil { t.Fatal("expected a non-nil error") } } diff --git a/blob/service.go b/blob/service.go index b0d6f09b28..5b414f4f00 100644 --- a/blob/service.go +++ b/blob/service.go @@ -1,7 +1,6 @@ package blob import ( - "bytes" "context" "encoding/hex" "errors" @@ -11,7 +10,7 @@ import ( "github.com/cosmos/cosmos-sdk/types" logging "github.com/ipfs/go-log/v2" - tmproto "github.com/tendermint/tendermint/proto/tendermint/types" + "github.com/tendermint/tendermint/libs/bytes" core "github.com/tendermint/tendermint/types" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" @@ -23,7 +22,6 @@ import ( "github.com/celestiaorg/go-square/v2/inclusion" libshare "github.com/celestiaorg/go-square/v2/share" "github.com/celestiaorg/nmt" - "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/libs/utils" @@ -299,6 +297,8 @@ func (s *Service) getAll( // To ensure that blob was included in a specific height, we need: // 1. verify the provided commitment by recomputing it; // 2. verify the provided Proof against subtree roots that were used in 1.; +// Note: this method can be deprecated because it's doing processing that can +// be done locally. func (s *Service) Included( ctx context.Context, height uint64, @@ -314,25 +314,20 @@ func (s *Service) Included( attribute.Int64("height", int64(height)), attribute.String("namespace", namespace.String()), ) - - // In the current implementation, LNs will have to download all shares to recompute the commitment. - // TODO(@vgonkivs): rework the implementation to perform all verification without network requests. - sharesParser := &parser{verifyFn: func(blob *Blob) bool { - return blob.compareCommitments(commitment) - }} - blob, _, err := s.retrieve(ctx, height, namespace, sharesParser) - switch { - case err == nil: - case errors.Is(err, ErrBlobNotFound): - return false, nil - default: - return false, err + // verify that the blob subtree roots match the proof subtree roots + if proofCommitment := proof.GenerateCommitment(); !commitment.Equal(proofCommitment) { + return false, fmt.Errorf( + "%w: unequal blob commitment %s and proof commitment %s", + ErrInvalidProof, + hex.EncodeToString(commitment), + hex.EncodeToString(proofCommitment), + ) } header, err := s.headerGetter(ctx, height) if err != nil { return false, err } - return proof.Verify(blob, header.DataHash) + return proof.Verify(header.DataHash) } // retrieve retrieves blobs and their proofs by requesting the whole namespace and @@ -660,23 +655,24 @@ func (s *Service) retrieveBlobProof( if err != nil { return nil, nil, err } - tmShareToRowRootProofs := make([]*tmproto.NMTProof, 0, len(shareToRowRootProofs)) - for _, proof := range shareToRowRootProofs { - tmShareToRowRootProofs = append(tmShareToRowRootProofs, &tmproto.NMTProof{ - Start: proof.Start, - End: proof.End, - Nodes: proof.Nodes, - LeafHash: proof.LeafHash, - }) + + // convert the share to row root proof to an nmt.Proof + nmtShareToRowRootProofs := toNMTProof(shareToRowRootProofs) + + subtreeRoots, err := inclusion.GenerateSubtreeRoots(blob.Blob, appconsts.DefaultSubtreeRootThreshold) + if err != nil { + return nil, nil, err } + proof := Proof{ - ShareToRowRootProof: tmShareToRowRootProofs, + SubtreeRootProofs: nmtShareToRowRootProofs, RowToDataRootProof: core.RowProof{ RowRoots: rowRoots, Proofs: rowProofs, StartRow: uint32(inclusiveBlobStartRowIndex), EndRow: uint32(exclusiveBlobEndRowIndex) - 1, }, + SubtreeRoots: subtreeRoots, } return blob, &proof, nil } @@ -691,6 +687,15 @@ func (s *Service) retrieveBlobProof( return nil, nil, ErrBlobNotFound } +func toNMTProof(proofs []*pkgproof.NMTProof) []*nmt.Proof { + nmtShareToRowRootProofs := make([]*nmt.Proof, 0, len(proofs)) + for _, proof := range proofs { + nmtProof := nmt.NewInclusionProof(int(proof.Start), int(proof.End), proof.Nodes, true) + nmtShareToRowRootProofs = append(nmtShareToRowRootProofs, &nmtProof) + } + return nmtShareToRowRootProofs +} + // getBlobs retrieves the DAH and fetches all shares from the requested Namespace and converts // them to Blobs. func (s *Service) getBlobs( @@ -717,191 +722,3 @@ func (s *Service) getBlobs( _, _, err = s.retrieve(ctx, header.Height(), namespace, sharesParser) return blobs, err } - -func (s *Service) GetCommitmentProof( - ctx context.Context, - height uint64, - namespace libshare.Namespace, - shareCommitment []byte, -) (*CommitmentProof, error) { - log.Debugw("proving share commitment", "height", height, "commitment", shareCommitment, "namespace", namespace) - if height == 0 { - return nil, fmt.Errorf("height cannot be equal to 0") - } - - // get the blob to compute the subtree roots - log.Debugw( - "getting the blob", - "height", - height, - "commitment", - shareCommitment, - "namespace", - namespace, - ) - blb, err := s.Get(ctx, height, namespace, shareCommitment) - if err != nil { - return nil, err - } - - log.Debugw( - "converting the blob to shares", - "height", - height, - "commitment", - shareCommitment, - "namespace", - namespace, - ) - blobShares, err := BlobsToShares(blb) - if err != nil { - return nil, err - } - if len(blobShares) == 0 { - return nil, fmt.Errorf("the blob shares for commitment %s are empty", hex.EncodeToString(shareCommitment)) - } - - // get the extended header - log.Debugw( - "getting the extended header", - "height", - height, - ) - extendedHeader, err := s.headerGetter(ctx, height) - if err != nil { - return nil, err - } - - log.Debugw("getting eds", "height", height) - eds, err := s.shareGetter.GetEDS(ctx, extendedHeader) - if err != nil { - return nil, err - } - - return ProveCommitment(eds, namespace, blobShares) -} - -func ProveCommitment( - eds *rsmt2d.ExtendedDataSquare, - namespace libshare.Namespace, - blobShares []libshare.Share, -) (*CommitmentProof, error) { - // find the blob shares in the EDS - blobSharesStartIndex := -1 - for index, share := range eds.FlattenedODS() { - if bytes.Equal(share, blobShares[0].ToBytes()) { - blobSharesStartIndex = index - } - } - if blobSharesStartIndex < 0 { - return nil, fmt.Errorf("couldn't find the blob shares in the ODS") - } - - log.Debugw( - "generating the blob share proof for commitment", - "start_share", - blobSharesStartIndex, - "end_share", - blobSharesStartIndex+len(blobShares), - ) - sharesProof, err := pkgproof.NewShareInclusionProofFromEDS( - eds, - namespace, - libshare.NewRange(blobSharesStartIndex, blobSharesStartIndex+len(blobShares)), - ) - if err != nil { - return nil, err - } - - // convert the shares to row root proofs to nmt proofs - nmtProofs := make([]*nmt.Proof, 0) - for _, proof := range sharesProof.ShareProofs { - nmtProof := nmt.NewInclusionProof( - int(proof.Start), - int(proof.End), - proof.Nodes, - true, - ) - nmtProofs = append( - nmtProofs, - &nmtProof, - ) - } - - // compute the subtree roots of the blob shares - log.Debugw("computing the subtree roots") - subtreeRoots := make([][]byte, 0) - dataCursor := 0 - for _, proof := range nmtProofs { - // TODO: do we want directly use the default subtree root threshold - // or want to allow specifying which version to use? - ranges, err := nmt.ToLeafRanges( - proof.Start(), - proof.End(), - inclusion.SubTreeWidth(len(blobShares), appconsts.DefaultSubtreeRootThreshold), - ) - if err != nil { - return nil, err - } - roots, err := computeSubtreeRoots( - blobShares[dataCursor:dataCursor+proof.End()-proof.Start()], - ranges, - proof.Start(), - ) - if err != nil { - return nil, err - } - subtreeRoots = append(subtreeRoots, roots...) - dataCursor += proof.End() - proof.Start() - } - - log.Debugw("successfully proved the share commitment") - commitmentProof := CommitmentProof{ - SubtreeRoots: subtreeRoots, - SubtreeRootProofs: nmtProofs, - NamespaceID: namespace.ID(), - RowProof: *sharesProof.RowProof, - NamespaceVersion: namespace.Version(), - } - return &commitmentProof, nil -} - -// computeSubtreeRoots takes a set of shares and ranges and returns the corresponding subtree roots. -// the offset is the number of shares that are before the subtree roots we're calculating. -func computeSubtreeRoots(shares []libshare.Share, ranges []nmt.LeafRange, offset int) ([][]byte, error) { - if len(shares) == 0 { - return nil, fmt.Errorf("cannot compute subtree roots for an empty shares list") - } - if len(ranges) == 0 { - return nil, fmt.Errorf("cannot compute subtree roots for an empty ranges list") - } - if offset < 0 { - return nil, fmt.Errorf("the offset %d cannot be stricly negative", offset) - } - - // create a tree containing the shares to generate their subtree roots - tree := nmt.New( - appconsts.NewBaseHashFunc(), - nmt.IgnoreMaxNamespace(true), - nmt.NamespaceIDSize(libshare.NamespaceSize), - ) - for _, sh := range shares { - leafData := make([]byte, 0) - leafData = append(append(leafData, sh.Namespace().Bytes()...), sh.ToBytes()...) - err := tree.Push(leafData) - if err != nil { - return nil, err - } - } - - // generate the subtree roots - subtreeRoots := make([][]byte, 0) - for _, rg := range ranges { - root, err := tree.ComputeSubtreeRoot(rg.Start-offset, rg.End-offset) - if err != nil { - return nil, err - } - subtreeRoots = append(subtreeRoots, root) - } - return subtreeRoots, nil -} diff --git a/blob/service_test.go b/blob/service_test.go index 0e00a94c34..de9b0f2728 100644 --- a/blob/service_test.go +++ b/blob/service_test.go @@ -18,7 +18,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" tmrand "github.com/tendermint/tendermint/libs/rand" - "github.com/tendermint/tendermint/proto/tendermint/types" coretypes "github.com/tendermint/tendermint/types" "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" @@ -240,7 +239,7 @@ func TestBlobService_Get(t *testing.T) { assert.True(t, ok) verifyFn := func(t *testing.T, blob *Blob, proof *Proof) { - valid, err := proof.Verify(blob, header.DataHash) + valid, err := proof.Verify(header.DataHash) require.NoError(t, err) require.True(t, valid) } @@ -294,20 +293,35 @@ func TestBlobService_Get(t *testing.T) { { name: "not included", doFn: func() (interface{}, error) { - libBlob, err := libshare.GenerateV0Blobs([]int{10}, false) - require.NoError(t, err) - blob, err := convertBlobs(libBlob...) - require.NoError(t, err) - proof, err := service.GetProof(ctx, 1, blobsWithDiffNamespaces[1].Namespace(), blobsWithDiffNamespaces[1].Commitment, ) require.NoError(t, err) - return service.Included(ctx, 1, blob[0].Namespace(), proof, blob[0].Commitment) + + // tamper with the header getter to get a random data hash at height 12345 + tamperedService := *service + tamperedService.headerGetter = func(ctx context.Context, height uint64) (*header.ExtendedHeader, error) { + if height == 12345 { + return &header.ExtendedHeader{ + RawHeader: header.RawHeader{ + DataHash: []byte{0x01}, + }, + }, nil + } + return service.headerGetter(ctx, height) + } + // this blob was included in height 1, but we will check if it's included in height 2 + return tamperedService.Included( + ctx, + 12345, + blobsWithDiffNamespaces[1].Namespace(), + proof, + blobsWithDiffNamespaces[1].Commitment, + ) }, expectedResult: func(res interface{}, err error) { - require.NoError(t, err) + require.Error(t, err) included, ok := res.(bool) require.True(t, ok) require.False(t, included) @@ -343,7 +357,7 @@ func TestBlobService_Get(t *testing.T) { originalDataWidth := len(h.DAH.RowRoots) / 2 sizes := []int{blobSize0, blobSize1} for i, proof := range proofs { - require.True(t, sizes[i]/originalDataWidth+1 == len(proof.ShareToRowRootProof)) + require.True(t, sizes[i]/originalDataWidth+1 == len(proof.SubtreeRootProofs)) } }, }, @@ -393,7 +407,7 @@ func TestBlobService_Get(t *testing.T) { header, err := service.headerGetter(ctx, 1) require.NoError(t, err) - valid, err := proof.Verify(blobsWithDiffNamespaces[1], header.DataHash) + valid, err := proof.Verify(header.DataHash) require.NoError(t, err) require.True(t, valid) }, @@ -989,25 +1003,17 @@ func proveAndVerifyShareCommitments(t *testing.T, blobSize int) { blobShares, err := BlobsToShares(blb) require.NoError(t, err) // compute the commitment - actualCommitmentProof, err := ProveCommitment(eds, nss[msgIndex], blobShares) + actualCommitmentProof, err := proveCommitment(eds, nss[msgIndex], blobs[msgIndex], blobShares) require.NoError(t, err) // make sure the actual commitment attests to the data - require.NoError(t, actualCommitmentProof.Validate()) - valid, err := actualCommitmentProof.Verify( - dataRoot, - appconsts.DefaultSubtreeRootThreshold, - ) + valid, err := actualCommitmentProof.Verify(dataRoot) require.NoError(t, err) require.True(t, valid) // generate an expected proof and verify it's valid - expectedCommitmentProof := generateCommitmentProofFromBlock(t, eds, nss[msgIndex], blobs[msgIndex], dataRoot) - require.NoError(t, expectedCommitmentProof.Validate()) - valid, err = expectedCommitmentProof.Verify( - dataRoot, - appconsts.DefaultSubtreeRootThreshold, - ) + expectedCommitmentProof := generateProofFromBlock(t, eds, nss[msgIndex], blobs[msgIndex], dataRoot) + valid, err = expectedCommitmentProof.Verify(dataRoot) require.NoError(t, err) require.True(t, valid) @@ -1021,15 +1027,15 @@ func proveAndVerifyShareCommitments(t *testing.T, blobSize int) { } } -// generateCommitmentProofFromBlock takes a block and a PFB index and generates the commitment proof +// generateProofFromBlock takes a block and a PFB index and generates the commitment proof // using the traditional way of doing, instead of using the API. -func generateCommitmentProofFromBlock( +func generateProofFromBlock( t *testing.T, eds *rsmt2d.ExtendedDataSquare, ns libshare.Namespace, blob *libshare.Blob, dataRoot []byte, -) CommitmentProof { +) Proof { // create the blob from the data blb, err := NewBlob(blob.ShareVersion(), ns, @@ -1062,38 +1068,13 @@ func generateCommitmentProofFromBlock( require.NoError(t, sharesProof.Validate(dataRoot)) // calculate the subtree roots - subtreeRoots := make([][]byte, 0) - dataCursor := 0 - for _, proof := range sharesProof.ShareProofs { - ranges, err := nmt.ToLeafRanges( - int(proof.Start), - int(proof.End), - inclusion.SubTreeWidth(len(blobShares), appconsts.DefaultSubtreeRootThreshold), - ) - require.NoError(t, err) - roots, err := computeSubtreeRoots( - blobShares[dataCursor:int32(dataCursor)+proof.End-proof.Start], - ranges, - int(proof.Start), - ) - require.NoError(t, err) - subtreeRoots = append(subtreeRoots, roots...) - dataCursor += int(proof.End - proof.Start) - } - - // convert the nmt proof to be accepted by the commitment proof - nmtProofs := make([]*nmt.Proof, 0) - for _, proof := range sharesProof.ShareProofs { - nmtProof := nmt.NewInclusionProof(int(proof.Start), int(proof.End), proof.Nodes, true) - nmtProofs = append(nmtProofs, &nmtProof) - } + subtreeRoots, err := blb.ComputeSubtreeRoots() + require.NoError(t, err) - commitmentProof := CommitmentProof{ - SubtreeRoots: subtreeRoots, - SubtreeRootProofs: nmtProofs, - NamespaceID: sharesProof.NamespaceId, - RowProof: *sharesProof.RowProof, - NamespaceVersion: uint8(sharesProof.NamespaceVersion), + commitmentProof := Proof{ + SubtreeRoots: subtreeRoots, + SubtreeRootProofs: toNMTProof(sharesProof.ShareProofs), + RowToDataRootProof: toCoreRowProof(sharesProof.RowProof), } return commitmentProof @@ -1143,20 +1124,13 @@ func TestBlobVerify(t *testing.T) { require.NoError(t, err) require.NoError(t, sharesProof.Validate(dataRoot)) - tmShareToRowRootProofs := make([]*types.NMTProof, 0, len(sharesProof.ShareProofs)) - for _, proof := range sharesProof.ShareProofs { - tmShareToRowRootProofs = append(tmShareToRowRootProofs, &types.NMTProof{ - Start: proof.Start, - End: proof.End, - Nodes: proof.Nodes, - LeafHash: proof.LeafHash, - }) - } - + subtreeRoots, err := blob.ComputeSubtreeRoots() + require.NoError(t, err) coreRowProof := toCoreRowProof(sharesProof.RowProof) blobProof := Proof{ - ShareToRowRootProof: tmShareToRowRootProofs, - RowToDataRootProof: coreRowProof, + SubtreeRoots: subtreeRoots, + SubtreeRootProofs: toNMTProof(sharesProof.ShareProofs), + RowToDataRootProof: coreRowProof, } tests := []struct { name string @@ -1192,13 +1166,10 @@ func TestBlobVerify(t *testing.T) { name: "malformed blob and proof", dataRoot: dataRoot, proof: func() Proof { + inclusionProof := nmt.NewInclusionProof(1, 3, [][]byte{{0x01}}, true) return Proof{ - ShareToRowRootProof: []*types.NMTProof{{ - Start: 1, - End: 3, - Nodes: [][]byte{{0x01}}, - LeafHash: nil, - }}, + SubtreeRoots: subtreeRoots, + SubtreeRootProofs: []*nmt.Proof{&inclusionProof}, RowToDataRootProof: blobProof.RowToDataRootProof, } }(), @@ -1214,7 +1185,13 @@ func TestBlobVerify(t *testing.T) { dataRoot: dataRoot, proof: func() Proof { p := blobProof - p.ShareToRowRootProof[0].End = 15 + invalidProof := nmt.NewInclusionProof( + blobProof.SubtreeRootProofs[0].Start(), + 15, + blobProof.SubtreeRootProofs[0].Nodes(), + true, + ) + p.SubtreeRootProofs[0] = &invalidProof return p }(), blob: *blob, @@ -1232,28 +1209,10 @@ func TestBlobVerify(t *testing.T) { dataRoot: dataRoot, blob: *blob, proof: func() Proof { - sharesProof, err := pkgproof.NewShareInclusionProofFromEDS( - eds, - nss[5], - libshare.NewRange(startShareIndex, startShareIndex+len(blobShares)), - ) - require.NoError(t, err) - require.NoError(t, sharesProof.Validate(dataRoot)) - - tmShareToRowRootProofs := make([]*types.NMTProof, 0, len(sharesProof.ShareProofs)) - for _, proof := range sharesProof.ShareProofs { - tmShareToRowRootProofs = append(tmShareToRowRootProofs, &types.NMTProof{ - Start: proof.Start, - End: proof.End, - Nodes: proof.Nodes, - LeafHash: proof.LeafHash, - }) - } - - coreRowProof := toCoreRowProof(sharesProof.RowProof) return Proof{ - ShareToRowRootProof: tmShareToRowRootProofs, - RowToDataRootProof: coreRowProof, + SubtreeRootProofs: toNMTProof(sharesProof.ShareProofs), + RowToDataRootProof: coreRowProof, + SubtreeRoots: subtreeRoots, } }(), }, @@ -1261,7 +1220,7 @@ func TestBlobVerify(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - valid, err := test.proof.Verify(&test.blob, test.dataRoot) + valid, err := test.proof.Verify(test.dataRoot) if test.expectErr { assert.Error(t, err) } else { @@ -1291,3 +1250,58 @@ func toCoreRowProof(proof *pkgproof.RowProof) coretypes.RowProof { EndRow: proof.EndRow, } } + +func proveCommitment( + eds *rsmt2d.ExtendedDataSquare, + namespace libshare.Namespace, + blb *libshare.Blob, + blobShares []libshare.Share, +) (*Proof, error) { + // find the blob shares in the EDS + blobSharesStartIndex := -1 + for index, share := range eds.FlattenedODS() { + if bytes.Equal(share, blobShares[0].ToBytes()) { + blobSharesStartIndex = index + } + } + if blobSharesStartIndex < 0 { + return nil, fmt.Errorf("couldn't find the blob shares in the ODS") + } + + sharesProof, err := pkgproof.NewShareInclusionProofFromEDS( + eds, + namespace, + libshare.NewRange(blobSharesStartIndex, blobSharesStartIndex+len(blobShares)), + ) + if err != nil { + return nil, err + } + + // convert the shares to row root proofs to nmt proofs + nmtProofs := make([]*nmt.Proof, 0) + for _, proof := range sharesProof.ShareProofs { + nmtProof := nmt.NewInclusionProof( + int(proof.Start), + int(proof.End), + proof.Nodes, + true, + ) + nmtProofs = append( + nmtProofs, + &nmtProof, + ) + } + + // compute the subtree roots of the blob shares + subtreeRoots, err := inclusion.GenerateSubtreeRoots(blb, appconsts.SubtreeRootThreshold(appVersion)) + if err != nil { + return nil, err + } + + commitmentProof := Proof{ + SubtreeRoots: subtreeRoots, + SubtreeRootProofs: nmtProofs, + RowToDataRootProof: toCoreRowProof(sharesProof.RowProof), + } + return &commitmentProof, nil +} diff --git a/nodebuilder/blob/blob.go b/nodebuilder/blob/blob.go index b8069f82b5..2a7b12ea23 100644 --- a/nodebuilder/blob/blob.go +++ b/nodebuilder/blob/blob.go @@ -34,14 +34,7 @@ type Module interface { GetProof(_ context.Context, height uint64, _ libshare.Namespace, _ blob.Commitment) (*blob.Proof, error) // Included checks whether a blob's given commitment(Merkle subtree root) is included at // given height and under the namespace. - Included(_ context.Context, height uint64, _ libshare.Namespace, _ *blob.Proof, _ blob.Commitment) (bool, error) - // GetCommitmentProof generates a commitment proof for a share commitment. - GetCommitmentProof( - ctx context.Context, - height uint64, - namespace libshare.Namespace, - shareCommitment []byte, - ) (*blob.CommitmentProof, error) + Included(_ context.Context, height uint64, _ share.Namespace, _ *blob.Proof, _ blob.Commitment) (bool, error) // Subscribe to published blobs from the given namespace as they are included. Subscribe(_ context.Context, _ libshare.Namespace) (<-chan *blob.SubscriptionResponse, error) } @@ -77,12 +70,6 @@ type API struct { *blob.Proof, blob.Commitment, ) (bool, error) `perm:"read"` - GetCommitmentProof func( - ctx context.Context, - height uint64, - namespace libshare.Namespace, - shareCommitment []byte, - ) (*blob.CommitmentProof, error) `perm:"read"` Subscribe func( context.Context, libshare.Namespace, @@ -116,15 +103,6 @@ func (api *API) GetProof( return api.Internal.GetProof(ctx, height, namespace, commitment) } -func (api *API) GetCommitmentProof( - ctx context.Context, - height uint64, - namespace libshare.Namespace, - shareCommitment []byte, -) (*blob.CommitmentProof, error) { - return api.Internal.GetCommitmentProof(ctx, height, namespace, shareCommitment) -} - func (api *API) Included( ctx context.Context, height uint64, diff --git a/nodebuilder/blob/mocks/api.go b/nodebuilder/blob/mocks/api.go index a7c1d7d909..e567d8f17f 100644 --- a/nodebuilder/blob/mocks/api.go +++ b/nodebuilder/blob/mocks/api.go @@ -67,21 +67,6 @@ func (mr *MockModuleMockRecorder) GetAll(arg0, arg1, arg2 interface{}) *gomock.C return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAll", reflect.TypeOf((*MockModule)(nil).GetAll), arg0, arg1, arg2) } -// GetCommitmentProof mocks base method. -func (m *MockModule) GetCommitmentProof(arg0 context.Context, arg1 uint64, arg2 share.Namespace, arg3 []byte) (*blob.CommitmentProof, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetCommitmentProof", arg0, arg1, arg2, arg3) - ret0, _ := ret[0].(*blob.CommitmentProof) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetCommitmentProof indicates an expected call of GetCommitmentProof. -func (mr *MockModuleMockRecorder) GetCommitmentProof(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCommitmentProof", reflect.TypeOf((*MockModule)(nil).GetCommitmentProof), arg0, arg1, arg2, arg3) -} - // GetProof mocks base method. func (m *MockModule) GetProof(arg0 context.Context, arg1 uint64, arg2 share.Namespace, arg3 blob.Commitment) (*blob.Proof, error) { m.ctrl.T.Helper() diff --git a/nodebuilder/share/mocks/api.go b/nodebuilder/share/mocks/api.go index cccc81a452..51b1a8eef2 100644 --- a/nodebuilder/share/mocks/api.go +++ b/nodebuilder/share/mocks/api.go @@ -54,10 +54,10 @@ func (mr *MockModuleMockRecorder) GetEDS(arg0, arg1 interface{}) *gomock.Call { } // GetRange mocks base method. -func (m *MockModule) GetRange(arg0 context.Context, arg1 uint64, arg2, arg3 int) (*share.GetRangeResult, error) { +func (m *MockModule) GetRange(arg0 context.Context, arg1 uint64, arg2, arg3 int) (*share.RangeResult, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetRange", arg0, arg1, arg2, arg3) - ret0, _ := ret[0].(*share.GetRangeResult) + ret0, _ := ret[0].(*share.RangeResult) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/nodebuilder/share/share.go b/nodebuilder/share/share.go index 8a3efcc757..3220acb39d 100644 --- a/nodebuilder/share/share.go +++ b/nodebuilder/share/share.go @@ -1,7 +1,10 @@ package share import ( + "bytes" "context" + "errors" + "fmt" "github.com/tendermint/tendermint/types" @@ -16,11 +19,30 @@ import ( var _ Module = (*API)(nil) -// GetRangeResult wraps the return value of the GetRange endpoint -// because Json-RPC doesn't support more than two return values. -type GetRangeResult struct { - Shares []libshare.Share - Proof *types.ShareProof +// RangeResult wraps the return value of the GetRange endpoint. +// It contains a set of shares along with their proof to +// the data root. +type RangeResult struct { + // Shares the queried shares. + Shares []share.Share `json:"shares"` + // Proof the proof of Shares up to the data root. + Proof *types.ShareProof `json:"proof"` +} + +// Verify verifies if the shares and proof in the range +// are being committed to by the provided data root. +// Returns nil if the proof is valid and a sensible error otherwise. +func (r RangeResult) Verify(dataRoot []byte) error { + if len(dataRoot) == 0 { + return errors.New("root must be non-empty") + } + + for index, data := range r.Shares { + if !bytes.Equal(data, r.Proof.Data[index]) { + return fmt.Errorf("mismatching share %d between the range result and the proof", index) + } + } + return r.Proof.Validate(dataRoot) } // Module provides access to any data square or block share on the network. @@ -53,7 +75,7 @@ type Module interface { ctx context.Context, height uint64, namespace libshare.Namespace, ) (shwap.NamespaceData, error) // GetRange gets a list of shares and their corresponding proof. - GetRange(ctx context.Context, height uint64, start, end int) (*GetRangeResult, error) + GetRange(ctx context.Context, height uint64, start, end int) (*RangeResult, error) } // API is a wrapper around Module for the RPC. @@ -78,7 +100,7 @@ type API struct { ctx context.Context, height uint64, start, end int, - ) (*GetRangeResult, error) `perm:"read"` + ) (*RangeResult, error) `perm:"read"` } } @@ -94,7 +116,7 @@ func (api *API) GetEDS(ctx context.Context, height uint64) (*rsmt2d.ExtendedData return api.Internal.GetEDS(ctx, height) } -func (api *API) GetRange(ctx context.Context, height uint64, start, end int) (*GetRangeResult, error) { +func (api *API) GetRange(ctx context.Context, height uint64, start, end int) (*RangeResult, error) { return api.Internal.GetRange(ctx, height, start, end) } @@ -151,7 +173,7 @@ func (m module) GetRange(ctx context.Context, height uint64, start, end int) (*G if err != nil { return nil, err } - return &GetRangeResult{ + return &RangeResult{ Shares: shares, Proof: proof, }, nil