Merge pull request #399 from iotaledger/feat/rejected-conflict-eviction

UTXO-level MCA threshold for unincluded conflicts

pkg/protocol/engine/booker/inmemorybooker/booker.go

@@ -114,6 +114,13 @@ func (b *Booker) Queue(block *blocks.Block) error {
 	// Based on the assumption that we always fork and the UTXO and Tangle past cones are always fully known.
 	signedTransactionMetadata.OnSignaturesValid(func() {
 		transactionMetadata := signedTransactionMetadata.TransactionMetadata()
+
+		if orphanedSlot, isOrphaned := transactionMetadata.OrphanedSlot(); isOrphaned && orphanedSlot <= block.SlotCommitmentID().Slot() {
+			block.SetInvalid()
+
+			return
+		}
+
 		transactionMetadata.OnBooked(func() {
 			block.SetPayloadConflictIDs(transactionMetadata.ConflictIDs())
 			b.bookingOrder.Queue(block)
@@ -142,6 +149,21 @@ func (b *Booker) book(block *blocks.Block) error {
 		return ierrors.Wrapf(err, "failed to inherit conflicts for block %s", block.ID())
 	}
 
+	// The block is invalid if it carries a conflict that has been orphaned with respect to its commitment.
+	for it := conflictsToInherit.Iterator(); it.HasNext(); {
+		conflictID := it.Next()
+
+		txMetadata, exists := b.ledger.MemPool().TransactionMetadata(conflictID)
+		if !exists {
+			return ierrors.Errorf("failed to load transaction %s for block %s", conflictID.String(), block.ID())
+		}
+
+		if orphanedSlot, orphaned := txMetadata.OrphanedSlot(); orphaned && orphanedSlot <= block.SlotCommitmentID().Slot() {
+			// Merge-to-master orphaned conflicts.
+			conflictsToInherit.Delete(conflictID)
+		}
+	}
+
 	block.SetConflictIDs(conflictsToInherit)
 	block.SetBooked()
 	b.events.BlockBooked.Trigger(block)

pkg/protocol/engine/ledger/ledger/ledger.go

@@ -68,7 +68,7 @@ func NewProvider() module.Provider[*engine.Engine, ledger.Ledger] {
 
 			l.setRetainTransactionFailureFunc(e.Retainer.RetainTransactionFailure)
 
-			l.memPool = mempoolv1.New(NewVM(l), l.resolveState, e.Workers.CreateGroup("MemPool"), l.conflictDAG, l.errorHandler, mempoolv1.WithForkAllTransactions[ledger.BlockVoteRank](true))
+			l.memPool = mempoolv1.New(NewVM(l), l.resolveState, e.Workers.CreateGroup("MemPool"), l.conflictDAG, l.apiProvider, l.errorHandler, mempoolv1.WithForkAllTransactions[ledger.BlockVoteRank](true))
 			e.EvictionState.Events.SlotEvicted.Hook(l.memPool.Evict)
 
 			l.manaManager = mana.NewManager(l.apiProvider, l.resolveAccountOutput, l.accountsLedger.Account)

pkg/protocol/engine/mempool/tests/tests.go

@@ -11,6 +11,7 @@ import (
 	"github.com/iotaledger/hive.go/runtime/memanalyzer"
 	"github.com/iotaledger/iota-core/pkg/protocol/engine/mempool"
 	iotago "github.com/iotaledger/iota.go/v4"
+	"github.com/iotaledger/iota.go/v4/tpkg"
 )
 
 const (
@@ -210,11 +211,10 @@ func TestSetInclusionSlot(t *testing.T, tf *TestFramework) {
 	require.True(t, exists)
 
 	tf.CommitSlot(1)
-	// time.Sleep(1 * time.Second)
-	transactionDeletionState := map[string]bool{"tx1": true, "tx2": false, "tx3": false}
+	transactionDeletionState := map[string]bool{"tx1": false, "tx2": false, "tx3": false}
 	tf.RequireTransactionsEvicted(transactionDeletionState)
 
-	attachmentDeletionState := map[string]bool{"block1": true, "block2": false, "block3": false}
+	attachmentDeletionState := map[string]bool{"block1": false, "block2": false, "block3": false}
 	tf.RequireAttachmentsEvicted(attachmentDeletionState)
 
 	tf.Instance.Evict(1)
@@ -223,9 +223,8 @@ func TestSetInclusionSlot(t *testing.T, tf *TestFramework) {
 	tf.RequireBooked("tx3")
 
 	tf.CommitSlot(2)
-	// time.Sleep(1 * time.Second)
-	tf.RequireTransactionsEvicted(lo.MergeMaps(transactionDeletionState, map[string]bool{"tx2": true}))
-	tf.RequireAttachmentsEvicted(lo.MergeMaps(attachmentDeletionState, map[string]bool{"block2": true}))
+	tf.RequireTransactionsEvicted(lo.MergeMaps(transactionDeletionState, map[string]bool{"tx2": false}))
+	tf.RequireAttachmentsEvicted(lo.MergeMaps(attachmentDeletionState, map[string]bool{"block1": true}))
 
 	tf.Instance.Evict(2)
 	tf.RequireBooked("tx3")
@@ -234,14 +233,13 @@ func TestSetInclusionSlot(t *testing.T, tf *TestFramework) {
 	tf.RequireAccepted(map[string]bool{"tx3": true})
 
 	tf.CommitSlot(3)
-	// time.Sleep(1 * time.Second)
-	tf.RequireTransactionsEvicted(lo.MergeMaps(transactionDeletionState, map[string]bool{"tx3": true}))
+	tf.RequireTransactionsEvicted(transactionDeletionState)
 
-	require.False(t, tx1Metadata.IsOrphaned())
-	require.False(t, tx2Metadata.IsOrphaned())
-	require.False(t, tx3Metadata.IsOrphaned())
+	require.False(t, lo.Return2(tx1Metadata.OrphanedSlot()))
+	require.False(t, lo.Return2(tx2Metadata.OrphanedSlot()))
+	require.False(t, lo.Return2(tx3Metadata.OrphanedSlot()))
 
-	tf.RequireAttachmentsEvicted(lo.MergeMaps(attachmentDeletionState, map[string]bool{"block3": true}))
+	tf.RequireAttachmentsEvicted(lo.MergeMaps(attachmentDeletionState, map[string]bool{"block1": true, "block2": true, "block3": false}))
 }
 
 func TestSetTransactionOrphanage(t *testing.T, tf *TestFramework) {
@@ -273,13 +271,14 @@ func TestSetTransactionOrphanage(t *testing.T, tf *TestFramework) {
 
 	tf.Instance.Evict(1)
 
-	tf.RequireTransactionsEvicted(map[string]bool{"tx1": true, "tx2": true, "tx3": true})
+	// We only evict after MCA
+	tf.RequireTransactionsEvicted(map[string]bool{"tx1": false, "tx2": false, "tx3": false})
 
-	require.True(t, tx1Metadata.IsOrphaned())
-	require.True(t, tx2Metadata.IsOrphaned())
-	require.True(t, tx3Metadata.IsOrphaned())
+	require.True(t, lo.Return2(tx1Metadata.OrphanedSlot()))
+	require.True(t, tx2Metadata.IsPending())
+	require.True(t, tx3Metadata.IsPending())
 
-	tf.RequireAttachmentsEvicted(map[string]bool{"block1": true, "block2": true, "block3": true})
+	tf.RequireAttachmentsEvicted(map[string]bool{"block1": true, "block2": false, "block3": false})
 }
 
 func TestSetTxOrphanageMultipleAttachments(t *testing.T, tf *TestFramework) {
@@ -313,27 +312,28 @@ func TestSetTxOrphanageMultipleAttachments(t *testing.T, tf *TestFramework) {
 	require.False(t, tx3Metadata.IsAccepted())
 
 	tf.Instance.Evict(1)
-	require.False(t, tx1Metadata.IsOrphaned())
-	require.False(t, tx2Metadata.IsOrphaned())
-	require.False(t, tx3Metadata.IsOrphaned())
+	require.False(t, lo.Return2(tx1Metadata.OrphanedSlot()))
+	require.False(t, lo.Return2(tx2Metadata.OrphanedSlot()))
+	require.False(t, lo.Return2(tx3Metadata.OrphanedSlot()))
 
 	require.True(t, lo.Return2(tf.ConflictDAG.ConflictSets(tf.TransactionID("tx1"))))
 	require.True(t, lo.Return2(tf.ConflictDAG.ConflictSets(tf.TransactionID("tx2"))))
 	require.True(t, lo.Return2(tf.ConflictDAG.ConflictSets(tf.TransactionID("tx3"))))
 
 	tf.Instance.Evict(2)
 
-	require.True(t, tx1Metadata.IsOrphaned())
-	require.True(t, tx2Metadata.IsOrphaned())
-	require.True(t, tx3Metadata.IsOrphaned())
+	require.True(t, lo.Return2(tx1Metadata.OrphanedSlot()))
+	require.True(t, lo.Return2(tx2Metadata.OrphanedSlot()))
+	require.True(t, lo.Return2(tx3Metadata.OrphanedSlot()))
 
-	require.False(t, lo.Return2(tf.ConflictDAG.ConflictSets(tf.TransactionID("tx1"))))
-	require.False(t, lo.Return2(tf.ConflictDAG.ConflictSets(tf.TransactionID("tx2"))))
-	require.False(t, lo.Return2(tf.ConflictDAG.ConflictSets(tf.TransactionID("tx3"))))
+	// All conflicts still exist, as they are kept around until MCA
+	require.True(t, lo.Return2(tf.ConflictDAG.ConflictSets(tf.TransactionID("tx1"))))
+	require.True(t, lo.Return2(tf.ConflictDAG.ConflictSets(tf.TransactionID("tx2"))))
+	require.True(t, lo.Return2(tf.ConflictDAG.ConflictSets(tf.TransactionID("tx3"))))
 
-	tf.RequireTransactionsEvicted(map[string]bool{"tx1": true, "tx2": true, "tx3": true})
+	tf.RequireTransactionsEvicted(map[string]bool{"tx1": false, "tx2": false, "tx3": false})
 
-	tf.RequireAttachmentsEvicted(map[string]bool{"block1.1": true, "block1.2": true, "block2": true, "block3": true})
+	tf.RequireAttachmentsEvicted(map[string]bool{"block1.1": true, "block1.2": true, "block2": false, "block3": false})
 }
 
 func TestStateDiff(t *testing.T, tf *TestFramework) {
@@ -482,7 +482,13 @@ func TestMemoryRelease(t *testing.T, tf *TestFramework) {
 	txIndex, prevStateAlias := issueTransactions(1, 20000, "genesis")
 	tf.WaitChildren()
 
-	issueTransactions(txIndex, 20000, prevStateAlias)
+	txIndex, _ = issueTransactions(txIndex, 20000, prevStateAlias)
+
+	// Eviction is delayed by MCA, so we force Commit and Eviction.
+	for index := txIndex; index <= txIndex+int(tpkg.TestAPI.ProtocolParameters().MaxCommittableAge()); index++ {
+		tf.CommitSlot(iotago.SlotIndex(index))
+		tf.Instance.Evict(iotago.SlotIndex(index))
+	}
 
 	tf.Cleanup()
 

pkg/protocol/engine/mempool/transaction_metadata.go

@@ -57,15 +57,15 @@ type inclusionFlags interface {
 
 	OnAccepted(callback func())
 
-	IsCommitted() bool
+	CommittedSlot() (slot iotago.SlotIndex, isCommitted bool)
 
-	OnCommitted(callback func())
+	OnCommittedSlotUpdated(callback func(slot iotago.SlotIndex))
 
 	IsRejected() bool
 
 	OnRejected(callback func())
 
-	IsOrphaned() bool
+	OrphanedSlot() (slot iotago.SlotIndex, isOrphaned bool)
 
-	OnOrphaned(callback func())
+	OnOrphanedSlotUpdated(callback func(slot iotago.SlotIndex))
 }

pkg/protocol/engine/mempool/v1/inclusion_flags.go

@@ -3,30 +3,38 @@ package mempoolv1
 import (
 	"github.com/iotaledger/hive.go/ds/reactive"
 	"github.com/iotaledger/hive.go/runtime/promise"
+	iotago "github.com/iotaledger/iota.go/v4"
 )
 
 // inclusionFlags represents important flags and events that relate to the inclusion of an entity in the distributed ledger.
 type inclusionFlags struct {
 	// accepted gets triggered when the entity gets marked as accepted.
 	accepted reactive.Variable[bool]
 
-	// committed gets triggered when the entity gets marked as committed.
-	committed *promise.Event
+	// committedSlot gets set to the slot in which the entity gets marked as committed.
+	committedSlot reactive.Variable[iotago.SlotIndex]
 
 	// rejected gets triggered when the entity gets marked as rejected.
 	rejected *promise.Event
 
-	// orphaned gets triggered when the entity gets marked as orphaned.
-	orphaned *promise.Event
+	// orphanedSlot gets set to the slot in which the entity gets marked as orphaned.
+	orphanedSlot reactive.Variable[iotago.SlotIndex]
 }
 
 // newInclusionFlags creates a new inclusionFlags instance.
 func newInclusionFlags() *inclusionFlags {
 	return &inclusionFlags{
-		accepted:  reactive.NewVariable[bool](),
-		committed: promise.NewEvent(),
-		rejected:  promise.NewEvent(),
-		orphaned:  promise.NewEvent(),
+		accepted:      reactive.NewVariable[bool](),
+		committedSlot: reactive.NewVariable[iotago.SlotIndex](),
+		rejected:      promise.NewEvent(),
+		// Make sure the oldest orphaned index doesn't get overridden by newer TX spending the orphaned conflict further.
+		orphanedSlot: reactive.NewVariable[iotago.SlotIndex](func(currentValue, newValue iotago.SlotIndex) iotago.SlotIndex {
+			if currentValue != 0 {
+				return currentValue
+			}
+
+			return newValue
+		}),
 	}
 }
 
@@ -68,21 +76,25 @@ func (s *inclusionFlags) OnRejected(callback func()) {
 }
 
 // IsCommitted returns true if the entity was committed.
-func (s *inclusionFlags) IsCommitted() bool {
-	return s.committed.WasTriggered()
+func (s *inclusionFlags) CommittedSlot() (slot iotago.SlotIndex, isCommitted bool) {
+	return s.committedSlot.Get(), s.committedSlot.Get() != 0
 }
 
 // OnCommitted registers a callback that gets triggered when the entity gets committed.
-func (s *inclusionFlags) OnCommitted(callback func()) {
-	s.committed.OnTrigger(callback)
+func (s *inclusionFlags) OnCommittedSlotUpdated(callback func(slot iotago.SlotIndex)) {
+	s.committedSlot.OnUpdate(func(_, newValue iotago.SlotIndex) {
+		callback(newValue)
+	})
 }
 
 // IsOrphaned returns true if the entity was orphaned.
-func (s *inclusionFlags) IsOrphaned() bool {
-	return s.orphaned.WasTriggered()
+func (s *inclusionFlags) OrphanedSlot() (slot iotago.SlotIndex, isOrphaned bool) {
+	return s.orphanedSlot.Get(), s.orphanedSlot.Get() != 0
 }
 
 // OnOrphaned registers a callback that gets triggered when the entity gets orphaned.
-func (s *inclusionFlags) OnOrphaned(callback func()) {
-	s.orphaned.OnTrigger(callback)
+func (s *inclusionFlags) OnOrphanedSlotUpdated(callback func(slot iotago.SlotIndex)) {
+	s.orphanedSlot.OnUpdate(func(_, newValue iotago.SlotIndex) {
+		callback(newValue)
+	})
 }