Implement SegWit (similar transactions with different signatures are …

…no longer double spends) (#389)

* Start refactor of mempool to support signed transaction ID and transaction ID correctly.

* Fix a part of compile errors

* Feat: upgraded to correct iota.go

* Fix: fixed more bugs

* Fix: fixed first memleak

* Fix: fixed memleak in test framework

* Fix: fixed more bugs

* Feat: upgraded to iota.go

* Fix: started to remove specific types from mempool

* Update iota.go

* Feat: it compiles

* Fix: removed ExecutionOutput

* Feat: started cleaning up

* Feat: cleaned up code

* Fix unit tests

* Feat: upgraded to latest iota.go

* Fix: fixed evil wallet

* Fix: fixed bug in evil spammer

* Fix: fixed linter errors

* Fix: fixed more linter issues

* Fix: fixed linter issues

* Update pkg/protocol/engine/mempool/v1/mempool.go

---------

Co-authored-by: Piotr Macek <[email protected]>

components/dashboard/jsonresponse.go

@@ -130,13 +130,13 @@ type Transaction struct {
 
 // NewTransaction returns a Transaction from the given iotago.SignedTransaction.
 func NewTransaction(signedTx *iotago.SignedTransaction) *Transaction {
-	txID, err := signedTx.ID()
+	txID, err := signedTx.Transaction.ID()
 	if err != nil {
 		return nil
 	}
 
-	inputs := make([]*Input, len(signedTx.Transaction.Inputs))
-	for i, input := range signedTx.Transaction.Inputs {
+	inputs := make([]*Input, len(signedTx.Transaction.TransactionEssence.Inputs))
+	for i, input := range signedTx.Transaction.TransactionEssence.Inputs {
 		inputs[i] = NewInput(input)
 	}
 

components/dashboard/visualizer.go

@@ -43,7 +43,7 @@ type tipinfo struct {
 
 func sendVertex(blk *blocks.Block, confirmed bool) {
 	modelBlk, _ := model.BlockFromBlock(blk.ProtocolBlock())
-	tx, isTx := modelBlk.SignedTransaction()
+	signedTransaction, isTx := modelBlk.SignedTransaction()
 
 	broadcastWsBlock(&wsblk{MsgTypeVertex, &vertex{
 		ID:                  blk.ID().ToHex(),
@@ -54,7 +54,7 @@ func sendVertex(blk *blocks.Block, confirmed bool) {
 		IsTx:                isTx,
 		IsTxAccepted: func() bool {
 			if isTx {
-				txMetadata, exists := deps.Protocol.MainEngineInstance().Ledger.MemPool().TransactionMetadata(lo.PanicOnErr(tx.ID()))
+				txMetadata, exists := deps.Protocol.MainEngineInstance().Ledger.MemPool().TransactionMetadata(lo.PanicOnErr(signedTransaction.Transaction.ID()))
 				if exists {
 					return txMetadata.IsAccepted()
 				}
@@ -85,9 +85,9 @@ func runVisualizer(component *app.Component) {
 			deps.Protocol.Events.Engine.BlockDAG.BlockAttached.Hook(func(block *blocks.Block) {
 				sendVertex(block, false)
 
-				tx, hasTx := block.SignedTransaction()
+				signedTransaction, hasTx := block.SignedTransaction()
 				if hasTx {
-					txMetadata, exists := deps.Protocol.MainEngineInstance().Ledger.MemPool().TransactionMetadata(lo.PanicOnErr(tx.ID()))
+					txMetadata, exists := deps.Protocol.MainEngineInstance().Ledger.MemPool().TransactionMetadata(lo.PanicOnErr(signedTransaction.Transaction.ID()))
 					if exists {
 						txMetadata.OnAccepted(func() {
 							sendTxAccepted(block.ID(), true)

components/debugapi/transactions.go

@@ -18,7 +18,7 @@ func init() {
 func storeTransactionsPerSlot(scd *notarization.SlotCommittedDetails) error {
 	slot := scd.Commitment.Slot()
 	stateDiff := deps.Protocol.MainEngineInstance().Ledger.MemPool().StateDiff(slot)
-	mutationsTree := ads.NewSet(mapdb.NewMapDB(), iotago.TransactionID.Bytes, iotago.SlotIdentifierFromBytes)
+	mutationsTree := ads.NewSet(mapdb.NewMapDB(), iotago.TransactionID.Bytes, iotago.TransactionIDFromBytes)
 	tcs := &TransactionsChangesResponse{
 		Index:                slot,
 		IncludedTransactions: make([]string, 0),

components/metrics/metrics_slots.go

@@ -71,7 +71,7 @@ var SlotMetrics = collector.NewCollection(slotNamespace,
 		collector.WithInitFunc(func() {
 			deps.Protocol.MainEngineInstance().Ledger.OnTransactionAttached(func(transactionMetadata mempool.TransactionMetadata) {
 				transactionMetadata.OnAccepted(func() {
-					for _, attachmentBlockID := range transactionMetadata.Attachments() {
+					for _, attachmentBlockID := range transactionMetadata.ValidAttachments() {
 						if block, exists := deps.Protocol.MainEngineInstance().BlockCache.Block(attachmentBlockID); exists && block.IsAccepted() {
 							deps.Collector.Increment(slotNamespace, acceptedAttachments, strconv.Itoa(int(attachmentBlockID.Slot())))
 						}
@@ -95,7 +95,7 @@ var SlotMetrics = collector.NewCollection(slotNamespace,
 
 			deps.Protocol.Events.Engine.ConflictDAG.ConflictCreated.Hook(func(conflictID iotago.TransactionID) {
 				if txMetadata, exists := deps.Protocol.MainEngineInstance().Ledger.TransactionMetadata(conflictID); exists {
-					for _, attachment := range txMetadata.Attachments() {
+					for _, attachment := range txMetadata.ValidAttachments() {
 						deps.Collector.Increment(slotNamespace, createdConflicts, strconv.Itoa(int(attachment.Slot())))
 					}
 				}
@@ -117,7 +117,7 @@ var SlotMetrics = collector.NewCollection(slotNamespace,
 
 			deps.Protocol.Events.Engine.ConflictDAG.ConflictAccepted.Hook(func(conflictID iotago.TransactionID) {
 				if txMetadata, exists := deps.Protocol.MainEngineInstance().Ledger.TransactionMetadata(conflictID); exists {
-					for _, attachmentBlockID := range txMetadata.Attachments() {
+					for _, attachmentBlockID := range txMetadata.ValidAttachments() {
 						if attachment, exists := deps.Protocol.MainEngineInstance().BlockCache.Block(attachmentBlockID); exists && attachment.IsAccepted() {
 							deps.Collector.Increment(slotNamespace, acceptedConflicts, strconv.Itoa(int(attachment.ID().Slot())))
 						}
@@ -141,7 +141,7 @@ var SlotMetrics = collector.NewCollection(slotNamespace,
 
 			deps.Protocol.Events.Engine.ConflictDAG.ConflictRejected.Hook(func(conflictID iotago.TransactionID) {
 				if txMetadata, exists := deps.Protocol.MainEngineInstance().Ledger.TransactionMetadata(conflictID); exists {
-					for _, attachmentBlockID := range txMetadata.Attachments() {
+					for _, attachmentBlockID := range txMetadata.ValidAttachments() {
 						if attachment, exists := deps.Protocol.MainEngineInstance().BlockCache.Block(attachmentBlockID); exists && attachment.IsAccepted() {
 							deps.Collector.Increment(slotNamespace, rejectedConflicts, strconv.Itoa(int(attachment.ID().Slot())))
 						}

go.mod

@@ -26,7 +26,7 @@ require (
 	github.com/iotaledger/hive.go/stringify v0.0.0-20230929122509-67f34bfed40d
 	github.com/iotaledger/inx-app v1.0.0-rc.3.0.20230927140518-622f63be6182
 	github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230927140257-bfa0bb0af2bd
-	github.com/iotaledger/iota.go/v4 v4.0.0-20231002120511-9ab88bf44daf
+	github.com/iotaledger/iota.go/v4 v4.0.0-20231003101444-5687809cd68a
 	github.com/labstack/echo/v4 v4.11.1
 	github.com/labstack/gommon v0.4.0
 	github.com/libp2p/go-libp2p v0.30.0
@@ -61,7 +61,7 @@ require (
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/eclipse/paho.mqtt.golang v1.4.3 // indirect
 	github.com/elastic/gosigar v0.14.2 // indirect
-	github.com/ethereum/go-ethereum v1.13.1 // indirect
+	github.com/ethereum/go-ethereum v1.13.2 // indirect
 	github.com/fatih/structs v1.1.0 // indirect
 	github.com/felixge/fgprof v0.9.3 // indirect
 	github.com/fjl/memsize v0.0.2 // indirect

go.sum

@@ -96,8 +96,8 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/ethereum/go-ethereum v1.13.1 h1:UF2FaUKPIy5jeZk3X06ait3y2Q4wI+vJ1l7+UARp+60=
-github.com/ethereum/go-ethereum v1.13.1/go.mod h1:xHQKzwkHSl0gnSjZK1mWa06XEdm9685AHqhRknOzqGQ=
+github.com/ethereum/go-ethereum v1.13.2 h1:g9mCpfPWqCA1OL4e6C98PeVttb0HadfBRuKTGvMnOvw=
+github.com/ethereum/go-ethereum v1.13.2/go.mod h1:gkQ5Ygi64ZBh9M/4iXY1R8WqoNCx1Ey0CkYn2BD4/fw=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
 github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
@@ -305,8 +305,8 @@ github.com/iotaledger/inx-app v1.0.0-rc.3.0.20230927140518-622f63be6182 h1:lQikt
 github.com/iotaledger/inx-app v1.0.0-rc.3.0.20230927140518-622f63be6182/go.mod h1:q24QEsS887ZWJVX76w2kwSgC84KS7wIKOy1otuqZ2ZM=
 github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230927140257-bfa0bb0af2bd h1:nFG3Zq/zFA4KhBYFX2IezX1C74zfE0DqCt0LrgTa9Ig=
 github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230927140257-bfa0bb0af2bd/go.mod h1:c5778OnWpLq108YE+Eb2m8Ri/t/4ydV0TvI/Sy5YivQ=
-github.com/iotaledger/iota.go/v4 v4.0.0-20231002120511-9ab88bf44daf h1:TNt6qra1H62HctwYhoxujPml/uN2AtnE1zMkB5kkVfI=
-github.com/iotaledger/iota.go/v4 v4.0.0-20231002120511-9ab88bf44daf/go.mod h1:+e3bsJFDr9HxmUMe+eQOLNut5wfcB/ivhJdouOJgOnE=
+github.com/iotaledger/iota.go/v4 v4.0.0-20231003101444-5687809cd68a h1:xgh1YQvLN+Y3KwX1G9/znGbCaQsfpDtpSLn8nKvaP8s=
+github.com/iotaledger/iota.go/v4 v4.0.0-20231003101444-5687809cd68a/go.mod h1:+e3bsJFDr9HxmUMe+eQOLNut5wfcB/ivhJdouOJgOnE=
 github.com/ipfs/boxo v0.10.0 h1:tdDAxq8jrsbRkYoF+5Rcqyeb91hgWe2hp7iLu7ORZLY=
 github.com/ipfs/boxo v0.10.0/go.mod h1:Fg+BnfxZ0RPzR0nOodzdIq3A7KgoWAOWsEIImrIQdBM=
 github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s=

pkg/core/promise/promise.go

@@ -212,6 +212,22 @@ func (p *Promise[T]) WasCompleted() bool {
 	return p.complete
 }
 
+// Result returns the result of the promise (or the zero value if the promise was not resolved).
+func (p *Promise[T]) Result() T {
+	p.mutex.RLock()
+	defer p.mutex.RUnlock()
+
+	return p.result
+}
+
+// Err returns the error of the promise (or nil if the promise was not rejected).
+func (p *Promise[T]) Err() error {
+	p.mutex.RLock()
+	defer p.mutex.RUnlock()
+
+	return p.err
+}
+
 // IsEmpty returns true if the promise has no updateCallbacks.
 func (p *Promise[T]) IsEmpty() bool {
 	p.mutex.RLock()

pkg/protocol/engine/booker/inmemorybooker/booker.go

@@ -27,7 +27,7 @@ type Booker struct {
 
 	blockCache *blocks.Blocks
 
-	conflictDAG conflictdag.ConflictDAG[iotago.TransactionID, iotago.OutputID, ledger.BlockVoteRank]
+	conflictDAG conflictdag.ConflictDAG[iotago.TransactionID, mempool.StateID, ledger.BlockVoteRank]
 
 	ledger ledger.Ledger
 
@@ -98,23 +98,26 @@ var _ booker.Booker = new(Booker)
 
 // Queue checks if payload is solid and then adds the block to a Booker's CausalOrder.
 func (b *Booker) Queue(block *blocks.Block) error {
-	transactionMetadata, containsTransaction := b.ledger.AttachTransaction(block)
+	signedTransactionMetadata, containsTransaction := b.ledger.AttachTransaction(block)
 
 	if !containsTransaction {
 		b.bookingOrder.Queue(block)
 		return nil
 	}
 
-	if transactionMetadata == nil {
+	if signedTransactionMetadata == nil {
 		b.retainBlockFailure(block.ID(), apimodels.BlockFailurePayloadInvalid)
 
 		return ierrors.Errorf("transaction in %s was not attached", block.ID())
 	}
 
 	// Based on the assumption that we always fork and the UTXO and Tangle past cones are always fully known.
-	transactionMetadata.OnBooked(func() {
-		block.SetPayloadConflictIDs(transactionMetadata.ConflictIDs())
-		b.bookingOrder.Queue(block)
+	signedTransactionMetadata.OnSignaturesValid(func() {
+		transactionMetadata := signedTransactionMetadata.TransactionMetadata()
+		transactionMetadata.OnBooked(func() {
+			block.SetPayloadConflictIDs(transactionMetadata.ConflictIDs())
+			b.bookingOrder.Queue(block)
+		})
 	})
 
 	return nil
@@ -171,7 +174,7 @@ func (b *Booker) inheritConflicts(block *blocks.Block) (conflictIDs ds.Set[iotag
 		case iotago.ShallowLikeParentType:
 			// Check whether the parent contains a conflicting TX,
 			// otherwise reference is invalid and the block should be marked as invalid as well.
-			if tx, hasTx := parentBlock.SignedTransaction(); !hasTx || !parentBlock.PayloadConflictIDs().Has(lo.PanicOnErr(tx.ID())) {
+			if signedTransaction, hasTx := parentBlock.SignedTransaction(); !hasTx || !parentBlock.PayloadConflictIDs().Has(lo.PanicOnErr(signedTransaction.Transaction.ID())) {
 				return nil, ierrors.Wrapf(err, "shallow like parent %s does not contain a conflicting transaction", parent.ID.String())
 			}
 

pkg/protocol/engine/events.go

@@ -14,6 +14,7 @@ import (
 	"github.com/iotaledger/iota-core/pkg/protocol/engine/eviction"
 	"github.com/iotaledger/iota-core/pkg/protocol/engine/filter"
 	"github.com/iotaledger/iota-core/pkg/protocol/engine/ledger"
+	"github.com/iotaledger/iota-core/pkg/protocol/engine/mempool"
 	"github.com/iotaledger/iota-core/pkg/protocol/engine/mempool/conflictdag"
 	"github.com/iotaledger/iota-core/pkg/protocol/engine/notarization"
 	"github.com/iotaledger/iota-core/pkg/protocol/engine/syncmanager"
@@ -41,7 +42,7 @@ type Events struct {
 	SybilProtection  *sybilprotection.Events
 	Ledger           *ledger.Events
 	Notarization     *notarization.Events
-	ConflictDAG      *conflictdag.Events[iotago.TransactionID, iotago.OutputID]
+	ConflictDAG      *conflictdag.Events[iotago.TransactionID, mempool.StateID]
 	Scheduler        *scheduler.Events
 	SeatManager      *seatmanager.Events
 	SyncManager      *syncmanager.Events
@@ -68,7 +69,7 @@ var NewEvents = event.CreateGroupConstructor(func() (newEvents *Events) {
 		SybilProtection:        sybilprotection.NewEvents(),
 		Ledger:                 ledger.NewEvents(),
 		Notarization:           notarization.NewEvents(),
-		ConflictDAG:            conflictdag.NewEvents[iotago.TransactionID, iotago.OutputID](),
+		ConflictDAG:            conflictdag.NewEvents[iotago.TransactionID, mempool.StateID](),
 		Scheduler:              scheduler.NewEvents(),
 		SeatManager:            seatmanager.NewEvents(),
 		SyncManager:            syncmanager.NewEvents(),

pkg/protocol/engine/ledger/ledger.go

@@ -16,7 +16,7 @@ import (
 )
 
 type Ledger interface {
-	AttachTransaction(block *blocks.Block) (transactionMetadata mempool.TransactionMetadata, containsTransaction bool)
+	AttachTransaction(block *blocks.Block) (signedTransactionMetadata mempool.SignedTransactionMetadata, containsTransaction bool)
 	OnTransactionAttached(callback func(transactionMetadata mempool.TransactionMetadata), opts ...event.Option)
 	TransactionMetadata(id iotago.TransactionID) (transactionMetadata mempool.TransactionMetadata, exists bool)
 	TransactionMetadataByAttachment(blockID iotago.BlockID) (transactionMetadata mempool.TransactionMetadata, exists bool)
@@ -30,7 +30,7 @@ type Ledger interface {
 	ForEachUnspentOutput(func(output *utxoledger.Output) bool) error
 	AddGenesisUnspentOutput(unspentOutput *utxoledger.Output) error
 
-	ConflictDAG() conflictdag.ConflictDAG[iotago.TransactionID, iotago.OutputID, BlockVoteRank]
+	ConflictDAG() conflictdag.ConflictDAG[iotago.TransactionID, mempool.StateID, BlockVoteRank]
 	MemPool() mempool.MemPool[BlockVoteRank]
 	SlotDiffs(slot iotago.SlotIndex) (*utxoledger.SlotDiff, error)