Merge remote-tracking branch 'origin/develop' into test/rewards-docke…

…rtest

pkg/protocol/chain.go

@@ -57,6 +57,9 @@ type Chain struct {
 	// IsEvicted contains a flag that indicates whether this chain was evicted.
 	IsEvicted reactive.Event
 
+	// IsSolid contains a flag that indicates whether this chain is solid (has a continuous connection to the root).
+	IsSolid reactive.Event
+
 	// shouldEvict contains a flag that indicates whether this chain should be evicted.
 	shouldEvict reactive.Event
 
@@ -86,6 +89,7 @@ func newChain(chains *Chains) *Chain {
 		StartEngine:              reactive.NewVariable[bool](),
 		Engine:                   reactive.NewVariable[*engine.Engine](),
 		IsEvicted:                reactive.NewEvent(),
+		IsSolid:                  reactive.NewEvent(),
 		shouldEvict:              reactive.NewEvent(),
 
 		chains:      chains,
@@ -201,13 +205,15 @@ func (c *Chain) initLogger() (shutdown func()) {
 		c.LatestSyncedSlot.LogUpdates(c, log.LevelTrace, "LatestSyncedSlot"),
 		c.OutOfSyncThreshold.LogUpdates(c, log.LevelTrace, "OutOfSyncThreshold"),
 		c.ForkingPoint.LogUpdates(c, log.LevelTrace, "ForkingPoint", (*Commitment).LogName),
+		c.ParentChain.LogUpdates(c, log.LevelTrace, "ParentChain", (*Chain).LogName),
 		c.LatestCommitment.LogUpdates(c, log.LevelTrace, "LatestCommitment", (*Commitment).LogName),
 		c.LatestAttestedCommitment.LogUpdates(c, log.LevelTrace, "LatestAttestedCommitment", (*Commitment).LogName),
 		c.LatestProducedCommitment.LogUpdates(c, log.LevelDebug, "LatestProducedCommitment", (*Commitment).LogName),
 		c.RequestAttestations.LogUpdates(c, log.LevelTrace, "RequestAttestations"),
 		c.StartEngine.LogUpdates(c, log.LevelDebug, "StartEngine"),
 		c.Engine.LogUpdates(c, log.LevelTrace, "Engine", (*engine.Engine).LogName),
 		c.IsEvicted.LogUpdates(c, log.LevelTrace, "IsEvicted"),
+		c.IsSolid.LogUpdates(c, log.LevelTrace, "IsSolid"),
 		c.shouldEvict.LogUpdates(c, log.LevelTrace, "shouldEvict"),
 
 		c.Logger.Shutdown,
@@ -233,6 +239,8 @@ func (c *Chain) initDerivedProperties() (shutdown func()) {
 							c.deriveShouldEvict(forkingPoint, parentChain),
 						)
 					}),
+
+					c.IsSolid.InheritFrom(forkingPoint.IsSolid),
 				)
 			}),
 		),

pkg/protocol/chains.go

@@ -323,7 +323,9 @@ func (c *Chains) initChainSwitching() (shutdown func()) {
 
 	return lo.BatchReverse(
 		c.HeaviestClaimedCandidate.WithNonEmptyValue(func(heaviestClaimedCandidate *Chain) (shutdown func()) {
-			return heaviestClaimedCandidate.RequestAttestations.ToggleValue(true)
+			return heaviestClaimedCandidate.IsSolid.WithNonEmptyValue(func(_ bool) (teardown func()) {
+				return heaviestClaimedCandidate.RequestAttestations.ToggleValue(true)
+			})
 		}),
 
 		c.HeaviestAttestedCandidate.OnUpdate(func(_ *Chain, heaviestAttestedCandidate *Chain) {

pkg/protocol/commitment.go

@@ -60,6 +60,10 @@ type Commitment struct {
 	// IsRoot contains a flag indicating if this Commitment is the root of the Chain.
 	IsRoot reactive.Event
 
+	// IsSolid contains a flag indicating if this Commitment is solid (has all the commitments in its past cone until
+	// the RootCommitment).
+	IsSolid reactive.Event
+
 	// IsAttested contains a flag indicating if we have received attestations for this Commitment.
 	IsAttested reactive.Event
 
@@ -108,6 +112,7 @@ func newCommitment(commitments *Commitments, model *model.Commitment) *Commitmen
 		CumulativeAttestedWeight:        reactive.NewVariable[uint64](),
 		CumulativeVerifiedWeight:        reactive.NewVariable[uint64](),
 		IsRoot:                          reactive.NewEvent(),
+		IsSolid:                         reactive.NewEvent(),
 		IsAttested:                      reactive.NewEvent(),
 		IsSynced:                        reactive.NewEvent(),
 		IsCommittable:                   reactive.NewEvent(),
@@ -219,6 +224,7 @@ func (c *Commitment) initLogger() (shutdown func()) {
 		c.CumulativeAttestedWeight.LogUpdates(c, log.LevelTrace, "CumulativeAttestedWeight"),
 		c.CumulativeVerifiedWeight.LogUpdates(c, log.LevelTrace, "CumulativeVerifiedWeight"),
 		c.IsRoot.LogUpdates(c, log.LevelTrace, "IsRoot"),
+		c.IsSolid.LogUpdates(c, log.LevelTrace, "IsSolid"),
 		c.IsAttested.LogUpdates(c, log.LevelTrace, "IsAttested"),
 		c.IsSynced.LogUpdates(c, log.LevelTrace, "IsSynced"),
 		c.IsCommittable.LogUpdates(c, log.LevelTrace, "IsCommittable"),
@@ -235,6 +241,7 @@ func (c *Commitment) initDerivedProperties() (shutdown func()) {
 	return lo.BatchReverse(
 		// mark commitments that are marked as root as verified
 		c.IsVerified.InheritFrom(c.IsRoot),
+		c.IsSolid.InheritFrom(c.IsRoot),
 
 		// mark commitments that are marked as verified as attested and synced
 		c.IsAttested.InheritFrom(c.IsVerified),
@@ -266,6 +273,8 @@ func (c *Commitment) initDerivedProperties() (shutdown func()) {
 						}),
 					)
 				}),
+
+				c.IsSolid.InheritFrom(parent.IsSolid),
 			)
 		}),
 

pkg/protocol/engine/accounts/accountsledger/manager.go

@@ -314,7 +314,7 @@ func (m *Manager) Rollback(targetSlot iotago.SlotIndex) error {
 		}
 	}
 
-	return nil
+	return m.accountsTree.Commit()
 }
 
 // AddAccount adds a new account to the Account tree, allotting to it the balance on the given output.

pkg/protocol/engine/blocks/block.go

@@ -2,6 +2,7 @@ package blocks
 
 import (
 	"fmt"
+	"sync/atomic"
 	"time"
 
 	"github.com/iotaledger/hive.go/ds"
@@ -11,10 +12,20 @@ import (
 	"github.com/iotaledger/hive.go/stringify"
 	"github.com/iotaledger/iota-core/pkg/core/account"
 	"github.com/iotaledger/iota-core/pkg/model"
+	"github.com/iotaledger/iota-core/pkg/protocol/engine/mempool"
 	iotago "github.com/iotaledger/iota.go/v4"
 )
 
 type Block struct {
+	// ParentsBooked is triggered when all parents of the block are booked.
+	ParentsBooked reactive.Event
+
+	// PayloadDependenciesAvailable is triggered when the dependencies of the block's payload are available.
+	PayloadDependenciesAvailable reactive.Event
+
+	// SignedTransactionMetadata contains the signed transaction metadata of the block.
+	SignedTransactionMetadata reactive.Variable[mempool.SignedTransactionMetadata]
+
 	// BlockDAG block
 	missing             bool
 	missingBlockID      iotago.BlockID
@@ -76,6 +87,10 @@ func (r *rootBlock) String() string {
 
 func newEmptyBlock() *Block {
 	return &Block{
+		ParentsBooked:                reactive.NewEvent(),
+		PayloadDependenciesAvailable: reactive.NewEvent(),
+		SignedTransactionMetadata:    reactive.NewVariable[mempool.SignedTransactionMetadata](),
+
 		witnesses:             ds.NewSet[account.SeatIndex](),
 		spenderIDs:            ds.NewSet[iotago.TransactionID](),
 		payloadSpenderIDs:     ds.NewSet[iotago.TransactionID](),
@@ -112,6 +127,8 @@ func NewRootBlock(blockID iotago.BlockID, commitmentID iotago.CommitmentID, issu
 	b.scheduled = true
 
 	// This should be true since we commit and evict on acceptance.
+	b.ParentsBooked.Set(true)
+	b.PayloadDependenciesAvailable.Set(true)
 	b.solid.Init(true)
 	b.booked.Init(true)
 	b.weightPropagated.Init(true)
@@ -663,3 +680,30 @@ func (b *Block) ModelBlock() *model.Block {
 func (b *Block) WorkScore() iotago.WorkScore {
 	return b.workScore
 }
+
+func (b *Block) WaitForPayloadDependencies(dependencies ds.Set[mempool.StateMetadata]) {
+	if dependencies == nil || dependencies.Size() == 0 {
+		b.PayloadDependenciesAvailable.Trigger()
+
+		return
+	}
+
+	var unreferencedOutputCount atomic.Int32
+	unreferencedOutputCount.Store(int32(dependencies.Size()))
+
+	dependencies.Range(func(dependency mempool.StateMetadata) {
+		dependencyReady := false
+
+		dependency.OnAccepted(func() {
+			dependency.OnInclusionSlotUpdated(func(_ iotago.SlotIndex, inclusionSlot iotago.SlotIndex) {
+				if !dependencyReady && inclusionSlot <= b.ID().Slot() {
+					dependencyReady = true
+
+					if unreferencedOutputCount.Add(-1) == 0 {
+						b.PayloadDependenciesAvailable.Trigger()
+					}
+				}
+			})
+		})
+	})
+}

pkg/protocol/engine/booker/inmemorybooker/booker.go

@@ -96,15 +96,14 @@ func (b *Booker) Init(ledger ledger.Ledger, loadBlockFromStorage func(iotago.Blo
 // Queue checks if payload is solid and then sets up the block to react to its parents.
 func (b *Booker) Queue(block *blocks.Block) error {
 	signedTransactionMetadata, containsTransaction := b.ledger.AttachTransaction(block)
-
 	if !containsTransaction {
 		b.setupBlock(block)
-		return nil
-	}
 
-	if signedTransactionMetadata == nil {
+		return nil
+	} else if signedTransactionMetadata == nil {
 		return ierrors.Errorf("transaction in block %s was not attached", block.ID())
 	}
+	block.SignedTransactionMetadata.Set(signedTransactionMetadata)
 
 	// Based on the assumption that we always fork and the UTXO and Tangle past cones are always fully known.
 	signedTransactionMetadata.OnSignaturesValid(func() {
@@ -137,6 +136,12 @@ func (b *Booker) Queue(block *blocks.Block) error {
 func (b *Booker) Reset() { /* nothing to reset but comply with interface */ }
 
 func (b *Booker) setupBlock(block *blocks.Block) {
+	var payloadDependencies, directlyReferencedPayloadDependencies ds.Set[mempool.StateMetadata]
+	if signedTransactionMetadata := block.SignedTransactionMetadata.Get(); signedTransactionMetadata != nil && signedTransactionMetadata.SignaturesInvalid() == nil && !signedTransactionMetadata.TransactionMetadata().IsInvalid() {
+		payloadDependencies = signedTransactionMetadata.TransactionMetadata().Inputs()
+		directlyReferencedPayloadDependencies = ds.NewSet[mempool.StateMetadata]()
+	}
+
 	var unbookedParentsCount atomic.Int32
 	unbookedParentsCount.Store(int32(len(block.Parents())))
 
@@ -149,13 +154,15 @@ func (b *Booker) setupBlock(block *blocks.Block) {
 		}
 
 		parentBlock.Booked().OnUpdateOnce(func(_ bool, _ bool) {
-			if unbookedParentsCount.Add(-1) == 0 {
-				if err := b.book(block); err != nil {
-					if block.SetInvalid() {
-						b.events.BlockInvalid.Trigger(block, ierrors.Wrap(err, "failed to book block"))
-					}
+			if directlyReferencedPayloadDependencies != nil {
+				if parentTransactionMetadata := parentBlock.SignedTransactionMetadata.Get(); parentTransactionMetadata != nil {
+					directlyReferencedPayloadDependencies.AddAll(parentTransactionMetadata.TransactionMetadata().Outputs())
 				}
 			}
+
+			if unbookedParentsCount.Add(-1) == 0 {
+				block.ParentsBooked.Trigger()
+			}
 		})
 
 		parentBlock.Invalid().OnUpdateOnce(func(_ bool, _ bool) {
@@ -164,6 +171,22 @@ func (b *Booker) setupBlock(block *blocks.Block) {
 			}
 		})
 	})
+
+	block.ParentsBooked.OnTrigger(func() {
+		if directlyReferencedPayloadDependencies != nil {
+			payloadDependencies.DeleteAll(directlyReferencedPayloadDependencies)
+		}
+
+		block.WaitForPayloadDependencies(payloadDependencies)
+	})
+
+	block.PayloadDependenciesAvailable.OnTrigger(func() {
+		if err := b.book(block); err != nil {
+			if block.SetInvalid() {
+				b.events.BlockInvalid.Trigger(block, ierrors.Wrap(err, "failed to book block"))
+			}
+		}
+	})
 }
 
 func (b *Booker) book(block *blocks.Block) error {

pkg/protocol/engine/ledger/tests/state.go

@@ -10,13 +10,15 @@ type MockedState struct {
 	id           iotago.OutputID
 	output       *MockedOutput
 	creationSlot iotago.SlotIndex
+	slotBooked   iotago.SlotIndex
 }
 
 func NewMockedState(transactionID iotago.TransactionID, index uint16) *MockedState {
 	return &MockedState{
 		id:           iotago.OutputIDFromTransactionIDAndIndex(transactionID, index),
 		output:       &MockedOutput{},
 		creationSlot: iotago.SlotIndex(0),
+		slotBooked:   iotago.SlotIndex(0),
 	}
 }
 
@@ -44,6 +46,10 @@ func (m *MockedState) SlotCreated() iotago.SlotIndex {
 	return m.creationSlot
 }
 
+func (m *MockedState) SlotBooked() iotago.SlotIndex {
+	return m.slotBooked
+}
+
 func (m *MockedState) String() string {
 	return "MockedOutput(" + m.id.ToHex() + ")"
 }

pkg/protocol/engine/mempool/signed_transaction_metadata.go

@@ -11,6 +11,8 @@ type SignedTransactionMetadata interface {
 
 	OnSignaturesInvalid(callback func(err error)) (unsubscribe func())
 
+	SignaturesInvalid() error
+
 	TransactionMetadata() TransactionMetadata
 
 	Attachments() []iotago.BlockID

pkg/protocol/engine/mempool/state.go

@@ -15,6 +15,9 @@ type State interface {
 
 	// Whether the state is read only.
 	IsReadOnly() bool
+
+	// SlotBooked returns the slot index of the state if it is booked.
+	SlotBooked() iotago.SlotIndex
 }
 
 // A thin wrapper around a resolved commitment.
@@ -34,6 +37,10 @@ func (s CommitmentInputState) IsReadOnly() bool {
 	return true
 }
 
+func (s CommitmentInputState) SlotBooked() iotago.SlotIndex {
+	return s.Commitment.Slot
+}
+
 func CommitmentInputStateFromCommitment(commitment *iotago.Commitment) CommitmentInputState {
 	return CommitmentInputState{
 		Commitment: commitment,

pkg/protocol/engine/mempool/state_metadata.go

@@ -6,6 +6,8 @@ import (
 )
 
 type StateMetadata interface {
+	CreatingTransaction() TransactionMetadata
+
 	State() State
 
 	SpenderIDs() reactive.Set[iotago.TransactionID]
@@ -16,5 +18,9 @@ type StateMetadata interface {
 
 	OnAcceptedSpenderUpdated(callback func(spender TransactionMetadata))
 
+	InclusionSlot() iotago.SlotIndex
+
+	OnInclusionSlotUpdated(callback func(prevSlot iotago.SlotIndex, newSlot iotago.SlotIndex))
+
 	inclusionFlags
 }

pkg/protocol/engine/mempool/v1/mempool.go

@@ -466,9 +466,12 @@ func (m *MemPool[VoteRank]) requestState(stateRef mempool.StateReference, waitIf
 		request := m.resolveState(stateRef)
 
 		request.OnSuccess(func(state mempool.State) {
+			inclusionSlot := state.SlotBooked()
+
 			// The output was resolved from the ledger, meaning it was actually persisted as it was accepted and
 			// committed: otherwise we would have found it in cache or the request would have never resolved.
 			stateMetadata := NewStateMetadata(state)
+			stateMetadata.inclusionSlot.Set(&inclusionSlot)
 			stateMetadata.accepted.Set(true)
 
 			p.Resolve(stateMetadata)

pkg/protocol/engine/mempool/v1/signed_transaction_metadata.go

@@ -50,6 +50,10 @@ func (s *SignedTransactionMetadata) OnSignaturesInvalid(callback func(error)) (u
 	})
 }
 
+func (s *SignedTransactionMetadata) SignaturesInvalid() error {
+	return s.signaturesInvalid.Get()
+}
+
 func (s *SignedTransactionMetadata) OnSignaturesValid(callback func()) (unsubscribe func()) {
 	return s.signaturesValid.OnTrigger(callback)
 }