diff --git a/components/debugapi/commitment.go b/components/debugapi/commitment.go index 815d5319a..62c6b7796 100644 --- a/components/debugapi/commitment.go +++ b/components/debugapi/commitment.go @@ -69,18 +69,22 @@ func prepareCommitmentGraph(g *graphviz.Graphviz, rootCommitment *protocol.Commi return nil, parentErr } - if err = parentCommitment.Children.ForEach(func(childCommitment *protocol.Commitment) error { - child, childErr := createNode(graph, childCommitment) - if childErr != nil { - return childErr - } + // TODO: this should be removed once eviction of commitments is properly implemented + if parentCommitment.Children.IsEmpty() { + if childCommitment, exists := parentCommitment.Chain.Get().Commitment(parentCommitment.Slot() + 1); exists { + if err = renderChild(childCommitment, graph, parentCommitment, parent); err != nil { + return nil, err + } - if childCommitment.Chain.Get() == deps.Protocol.Chains.Main.Get() { - child.SetColor("green") + commitmentWalker.Push(childCommitment) + + continue } + } - if _, edgeErr := graph.CreateEdge(fmt.Sprintf("%s -> %s", parentCommitment.ID().String()[:8], childCommitment.ID().String()[:8]), parent, child); edgeErr != nil { - return ierrors.Wrapf(edgeErr, "could not create edge %s -> %s", parentCommitment.ID().String()[:8], childCommitment.ID().String()[:8]) + if err = parentCommitment.Children.ForEach(func(childCommitment *protocol.Commitment) error { + if err = renderChild(childCommitment, graph, parentCommitment, parent); err != nil { + return err } commitmentWalker.Push(childCommitment) @@ -94,6 +98,23 @@ func prepareCommitmentGraph(g *graphviz.Graphviz, rootCommitment *protocol.Commi return graph, nil } +func renderChild(childCommitment *protocol.Commitment, graph *cgraph.Graph, parentCommitment *protocol.Commitment, parent *cgraph.Node) error { + child, err := createNode(graph, childCommitment) + if err != nil { + return err + } + + if childCommitment.Chain.Get() == deps.Protocol.Chains.Main.Get() { + child.SetColor("green") + } + + if _, edgeErr := graph.CreateEdge(fmt.Sprintf("%s -> %s", parentCommitment.ID().String()[:8], childCommitment.ID().String()[:8]), parent, child); edgeErr != nil { + return ierrors.Wrapf(edgeErr, "could not create edge %s -> %s", parentCommitment.ID().String()[:8], childCommitment.ID().String()[:8]) + } + + return nil +} + func createNode(graph *cgraph.Graph, commitment *protocol.Commitment) (*cgraph.Node, error) { node, err := graph.CreateNode(fmt.Sprintf("%d-%s", commitment.ID().Slot(), commitment.ID().Identifier().String()[:8])) if err != nil { diff --git a/components/debugapi/component.go b/components/debugapi/component.go index fa5f513ef..8781d631b 100644 --- a/components/debugapi/component.go +++ b/components/debugapi/component.go @@ -90,11 +90,13 @@ func configure() error { routeGroup := deps.RestRouteManager.AddRoute("debug/v2") + debugAPIWorkerPool := workerpool.NewGroup("DebugAPI").CreatePool("DebugAPI", workerpool.WithWorkerCount(1)) + deps.Protocol.Events.Engine.BlockDAG.BlockAttached.Hook(func(block *blocks.Block) { blocksPerSlot.Set(block.ID().Slot(), append(lo.Return1(blocksPerSlot.GetOrCreate(block.ID().Slot(), func() []*blocks.Block { return make([]*blocks.Block, 0) })), block)) - }) + }, event.WithWorkerPool(debugAPIWorkerPool)) deps.Protocol.Events.Engine.SlotGadget.SlotFinalized.Hook(func(index iotago.SlotIndex) { epoch := deps.Protocol.APIForSlot(index).TimeProvider().EpochFromSlot(index) @@ -113,15 +115,15 @@ func configure() error { } } - }, event.WithWorkerPool(workerpool.NewGroup("DebugAPI").CreatePool("PruneDebugAPI", workerpool.WithWorkerCount(1)))) + }, event.WithWorkerPool(debugAPIWorkerPool)) deps.Protocol.Events.Engine.Notarization.SlotCommitted.Hook(func(scd *notarization.SlotCommittedDetails) { if err := storeTransactionsPerSlot(scd); err != nil { Component.LogWarnf(">> DebugAPI Error: %s\n", err) } - }) + }, event.WithWorkerPool(debugAPIWorkerPool)) - deps.Protocol.Events.Engine.EvictionState.SlotEvicted.Hook(func(index iotago.SlotIndex) { + deps.Protocol.Events.Engine.Evict.Hook(func(index iotago.SlotIndex) { blocksInSlot, exists := blocksPerSlot.Get(index) if !exists { return @@ -129,7 +131,8 @@ func configure() error { for _, block := range blocksInSlot { if block.ProtocolBlock() == nil { - Component.LogInfof("block is a root block", block.ID()) + Component.LogInfof("block is a root block %s", block.ID()) + continue } @@ -146,7 +149,7 @@ func configure() error { } blocksPerSlot.Delete(index) - }) + }, event.WithWorkerPool(debugAPIWorkerPool)) routeGroup.GET(RouteBlockMetadata, func(c echo.Context) error { blockID, err := httpserver.ParseBlockIDParam(c, api.ParameterBlockID) diff --git a/components/debugapi/transactions.go b/components/debugapi/transactions.go index 55a8314d9..d5f7252a6 100644 --- a/components/debugapi/transactions.go +++ b/components/debugapi/transactions.go @@ -4,7 +4,6 @@ import ( "github.com/iotaledger/hive.go/ads" "github.com/iotaledger/hive.go/ierrors" "github.com/iotaledger/hive.go/kvstore/mapdb" - "github.com/iotaledger/iota-core/pkg/protocol/engine/mempool" "github.com/iotaledger/iota-core/pkg/protocol/engine/notarization" iotago "github.com/iotaledger/iota.go/v4" ) @@ -17,10 +16,7 @@ func init() { func storeTransactionsPerSlot(scd *notarization.SlotCommittedDetails) error { slot := scd.Commitment.Slot() - stateDiff, err := deps.Protocol.Engines.Main.Get().Ledger.MemPool().StateDiff(slot) - if err != nil { - return ierrors.Wrapf(err, "failed to retrieve state diff for slot %d", slot) - } + mutationsTree := ads.NewSet[iotago.Identifier]( mapdb.NewMapDB(), iotago.Identifier.Bytes, @@ -33,23 +29,24 @@ func storeTransactionsPerSlot(scd *notarization.SlotCommittedDetails) error { IncludedTransactions: make([]string, 0), } - var innerErr error - stateDiff.ExecutedTransactions().ForEach(func(_ iotago.TransactionID, txMeta mempool.TransactionMetadata) bool { - tcs.IncludedTransactions = append(tcs.IncludedTransactions, txMeta.ID().String()) - if err := mutationsTree.Add(txMeta.ID()); err != nil { - innerErr = ierrors.Wrapf(err, "failed to add transaction to mutations tree, txID: %s", txMeta.ID()) + for _, transaction := range scd.Mutations { + txID, err := transaction.ID() + if err != nil { + return ierrors.Wrapf(err, "failed to calculate transactionID") + } - return false + tcs.IncludedTransactions = append(tcs.IncludedTransactions, txID.String()) + if err = mutationsTree.Add(txID); err != nil { + return ierrors.Wrapf(err, "failed to add transaction to mutations tree, txID: %s", txID) } - return true - }) + } tcs.MutationsRoot = mutationsTree.Root().String() transactionsPerSlot[slot] = tcs - return innerErr + return nil } func getSlotTransactionIDs(slot iotago.SlotIndex) (*TransactionsChangesResponse, error) { diff --git a/components/prometheus/metrics_commitments.go b/components/prometheus/metrics_commitments.go index e00c59ba0..70058c0ba 100644 --- a/components/prometheus/metrics_commitments.go +++ b/components/prometheus/metrics_commitments.go @@ -59,19 +59,28 @@ var CommitmentsMetrics = collector.NewCollection(commitmentsNamespace, collector.WithHelp("Number of accepted blocks by the node per slot."), collector.WithLabels("slot"), collector.WithPruningDelay(10*time.Minute), - collector.WithResetBeforeCollecting(true), collector.WithInitFunc(func() { deps.Protocol.Events.Engine.Notarization.SlotCommitted.Hook(func(details *notarization.SlotCommittedDetails) { deps.Collector.Update(commitmentsNamespace, acceptedBlocks, float64(details.AcceptedBlocks.Size()), strconv.Itoa(int(details.Commitment.Slot()))) }, event.WithWorkerPool(Component.WorkerPool)) }), )), + collector.WithMetric(collector.NewMetric(transactions, + collector.WithType(collector.Gauge), + collector.WithHelp("Number of accepted transactions by the node per slot."), + collector.WithLabels("slot"), + collector.WithPruningDelay(10*time.Minute), + collector.WithInitFunc(func() { + deps.Protocol.Events.Engine.Notarization.SlotCommitted.Hook(func(details *notarization.SlotCommittedDetails) { + deps.Collector.Update(commitmentsNamespace, transactions, float64(len(details.Mutations)), strconv.Itoa(int(details.Commitment.Slot()))) + }, event.WithWorkerPool(Component.WorkerPool)) + }), + )), collector.WithMetric(collector.NewMetric(validators, collector.WithType(collector.Gauge), collector.WithHelp("Number of active validators per slot."), collector.WithLabels("slot"), collector.WithPruningDelay(10*time.Minute), - collector.WithResetBeforeCollecting(true), collector.WithInitFunc(func() { deps.Protocol.Events.Engine.Notarization.SlotCommitted.Hook(func(details *notarization.SlotCommittedDetails) { deps.Collector.Update(commitmentsNamespace, validators, float64(details.ActiveValidatorsCount), strconv.Itoa(int(details.Commitment.Slot()))) diff --git a/pkg/protocol/engine/blocks/blocks.go b/pkg/protocol/engine/blocks/blocks.go index facb969c0..8d3e9b69b 100644 --- a/pkg/protocol/engine/blocks/blocks.go +++ b/pkg/protocol/engine/blocks/blocks.go @@ -2,7 +2,6 @@ package blocks import ( "github.com/iotaledger/hive.go/core/memstorage" - "github.com/iotaledger/hive.go/runtime/event" "github.com/iotaledger/hive.go/runtime/syncutils" "github.com/iotaledger/iota-core/pkg/model" "github.com/iotaledger/iota-core/pkg/protocol/engine/eviction" @@ -10,7 +9,6 @@ import ( ) type Blocks struct { - Evict *event.Event1[iotago.SlotIndex] blocks *memstorage.IndexedStorage[iotago.SlotIndex, iotago.BlockID, *Block] evictionState *eviction.State apiProvider iotago.APIProvider @@ -19,16 +17,13 @@ type Blocks struct { func New(evictionState *eviction.State, apiProvider iotago.APIProvider) *Blocks { return &Blocks{ - Evict: event.New1[iotago.SlotIndex](), blocks: memstorage.NewIndexedStorage[iotago.SlotIndex, iotago.BlockID, *Block](), evictionState: evictionState, apiProvider: apiProvider, } } -func (b *Blocks) EvictUntil(slot iotago.SlotIndex) { - b.Evict.Trigger(slot) - +func (b *Blocks) Evict(slot iotago.SlotIndex) { b.evictionMutex.Lock() defer b.evictionMutex.Unlock() diff --git a/pkg/protocol/engine/booker/inmemorybooker/booker.go b/pkg/protocol/engine/booker/inmemorybooker/booker.go index f4dcb3774..9b8dbfb8e 100644 --- a/pkg/protocol/engine/booker/inmemorybooker/booker.go +++ b/pkg/protocol/engine/booker/inmemorybooker/booker.go @@ -157,7 +157,7 @@ func (b *Booker) setupBlock(block *blocks.Block) { parentBlock.Invalid().OnUpdateOnce(func(_ bool, _ bool) { if block.SetInvalid() { - b.events.BlockInvalid.Trigger(block, ierrors.New("block marked as invalid in Booker")) + b.events.BlockInvalid.Trigger(block, ierrors.Errorf("block marked as invalid in Booker because parent block is invalid %s", parentBlock.ID())) } }) }) diff --git a/pkg/protocol/engine/engine.go b/pkg/protocol/engine/engine.go index 65622aa2c..f089aee37 100644 --- a/pkg/protocol/engine/engine.go +++ b/pkg/protocol/engine/engine.go @@ -461,8 +461,6 @@ func (e *Engine) setupBlockStorage() { } func (e *Engine) setupEvictionState() { - e.Events.EvictionState.LinkTo(e.EvictionState.Events) - wp := e.Workers.CreatePool("EvictionState", workerpool.WithWorkerCount(1)) // Using just 1 worker to avoid contention e.Events.BlockGadget.BlockAccepted.Hook(func(block *blocks.Block) { @@ -471,9 +469,17 @@ func (e *Engine) setupEvictionState() { e.Events.Notarization.LatestCommitmentUpdated.Hook(func(commitment *model.Commitment) { e.EvictionState.AdvanceActiveWindowToIndex(commitment.Slot()) - }, event.WithWorkerPool(wp)) + e.BlockRequester.EvictUntil(commitment.Slot()) + }) - e.Events.EvictionState.SlotEvicted.Hook(e.BlockCache.EvictUntil) + // We evict the block cache and trigger the eviction event in a separate worker pool. + // The block cache can be evicted asynchronously, as its internal state is defined via the EvictionState, and it will + // be updated accordingly on LatestCommitmentUpdated (atomically). + evictionWP := e.Workers.CreatePool("Eviction", workerpool.WithWorkerCount(1)) // Using just 1 worker to avoid contention + e.Events.Notarization.LatestCommitmentUpdated.Hook(func(commitment *model.Commitment) { + e.BlockCache.Evict(commitment.Slot()) + e.Events.Evict.Trigger(commitment.Slot()) + }, event.WithWorkerPool(evictionWP)) e.EvictionState.Initialize(e.Storage.Settings().LatestCommitment().Slot()) } @@ -481,8 +487,6 @@ func (e *Engine) setupEvictionState() { func (e *Engine) setupBlockRequester() { e.Events.BlockRequester.LinkTo(e.BlockRequester.Events) - e.Events.EvictionState.SlotEvicted.Hook(e.BlockRequester.EvictUntil) - // We need to hook to make sure that the request is created before the block arrives to avoid a race condition // where we try to delete the request again before it is created. Thus, continuing to request forever. e.Events.BlockDAG.BlockMissing.Hook(func(block *blocks.Block) { diff --git a/pkg/protocol/engine/events.go b/pkg/protocol/engine/events.go index 209736147..bb1eff125 100644 --- a/pkg/protocol/engine/events.go +++ b/pkg/protocol/engine/events.go @@ -10,7 +10,6 @@ import ( "github.com/iotaledger/iota-core/pkg/protocol/engine/congestioncontrol/scheduler" "github.com/iotaledger/iota-core/pkg/protocol/engine/consensus/blockgadget" "github.com/iotaledger/iota-core/pkg/protocol/engine/consensus/slotgadget" - "github.com/iotaledger/iota-core/pkg/protocol/engine/eviction" "github.com/iotaledger/iota-core/pkg/protocol/engine/filter/postsolidfilter" "github.com/iotaledger/iota-core/pkg/protocol/engine/filter/presolidfilter" "github.com/iotaledger/iota-core/pkg/protocol/engine/ledger" @@ -27,9 +26,9 @@ import ( type Events struct { BlockProcessed *event.Event1[iotago.BlockID] AcceptedBlockProcessed *event.Event1[*blocks.Block] + Evict *event.Event1[iotago.SlotIndex] StoragePruned *event.Event1[iotago.EpochIndex] - EvictionState *eviction.Events PreSolidFilter *presolidfilter.Events PostSolidFilter *postsolidfilter.Events BlockRequester *eventticker.Events[iotago.SlotIndex, iotago.BlockID] @@ -55,8 +54,8 @@ var NewEvents = event.CreateGroupConstructor(func() (newEvents *Events) { return &Events{ BlockProcessed: event.New1[iotago.BlockID](), AcceptedBlockProcessed: event.New1[*blocks.Block](), + Evict: event.New1[iotago.SlotIndex](), StoragePruned: event.New1[iotago.EpochIndex](), - EvictionState: eviction.NewEvents(), PreSolidFilter: presolidfilter.NewEvents(), PostSolidFilter: postsolidfilter.NewEvents(), BlockRequester: eventticker.NewEvents[iotago.SlotIndex, iotago.BlockID](), diff --git a/pkg/protocol/engine/eviction/events.go b/pkg/protocol/engine/eviction/events.go deleted file mode 100644 index 8db6f14a2..000000000 --- a/pkg/protocol/engine/eviction/events.go +++ /dev/null @@ -1,19 +0,0 @@ -package eviction - -import ( - "github.com/iotaledger/hive.go/runtime/event" - iotago "github.com/iotaledger/iota.go/v4" -) - -type Events struct { - SlotEvicted *event.Event1[iotago.SlotIndex] - - event.Group[Events, *Events] -} - -// NewEvents contains the constructor of the Events object (it is generated by a generic factory). -var NewEvents = event.CreateGroupConstructor(func() (self *Events) { - return &Events{ - SlotEvicted: event.New1[iotago.SlotIndex](), - } -}) diff --git a/pkg/protocol/engine/eviction/state.go b/pkg/protocol/engine/eviction/state.go index 51054709b..54f4649f2 100644 --- a/pkg/protocol/engine/eviction/state.go +++ b/pkg/protocol/engine/eviction/state.go @@ -18,8 +18,6 @@ import ( // State represents the state of the eviction and keeps track of the root blocks. type State struct { - Events *Events - settings *permanent.Settings rootBlockStorageFunc func(iotago.SlotIndex) (*slotstore.Store[iotago.BlockID, iotago.CommitmentID], error) lastCommittedSlot iotago.SlotIndex @@ -30,7 +28,6 @@ type State struct { func NewState(settings *permanent.Settings, rootBlockStorageFunc func(iotago.SlotIndex) (*slotstore.Store[iotago.BlockID, iotago.CommitmentID], error)) (state *State) { return &State{ settings: settings, - Events: NewEvents(), rootBlockStorageFunc: rootBlockStorageFunc, } } @@ -42,18 +39,13 @@ func (s *State) Initialize(lastCommittedSlot iotago.SlotIndex) { func (s *State) AdvanceActiveWindowToIndex(slot iotago.SlotIndex) { s.evictionMutex.Lock() + defer s.evictionMutex.Unlock() if slot <= s.lastCommittedSlot { - s.evictionMutex.Unlock() return } s.lastCommittedSlot = slot - - s.evictionMutex.Unlock() - - // We only delay eviction in the Eviction State, but components evict on commitment, which in this context is slot. - s.Events.SlotEvicted.Trigger(slot) } func (s *State) LastEvictedSlot() iotago.SlotIndex { diff --git a/pkg/protocol/engine/ledger/ledger.go b/pkg/protocol/engine/ledger/ledger.go index 3c2a1f971..77b815bad 100644 --- a/pkg/protocol/engine/ledger/ledger.go +++ b/pkg/protocol/engine/ledger/ledger.go @@ -37,7 +37,7 @@ type Ledger interface { ManaManager() *mana.Manager RMCManager() *rmc.Manager - CommitSlot(slot iotago.SlotIndex) (stateRoot, mutationRoot, accountRoot iotago.Identifier, created utxoledger.Outputs, consumed utxoledger.Spents, err error) + CommitSlot(slot iotago.SlotIndex) (stateRoot, mutationRoot, accountRoot iotago.Identifier, created utxoledger.Outputs, consumed utxoledger.Spents, mutations []*iotago.Transaction, err error) Import(reader io.ReadSeeker) error Export(writer io.WriteSeeker, targetSlot iotago.SlotIndex) error diff --git a/pkg/protocol/engine/ledger/ledger/ledger.go b/pkg/protocol/engine/ledger/ledger/ledger.go index c281951ee..14fea5c4d 100644 --- a/pkg/protocol/engine/ledger/ledger/ledger.go +++ b/pkg/protocol/engine/ledger/ledger/ledger.go @@ -72,7 +72,6 @@ func NewProvider() module.Provider[*engine.Engine, ledger.Ledger] { l.setRetainTransactionFailureFunc(e.Retainer.RetainTransactionFailure) l.memPool = mempoolv1.New(NewVM(l), l.resolveState, e.Storage.Mutations, e.Workers.CreateGroup("MemPool"), l.spendDAG, l.apiProvider, l.errorHandler) - e.EvictionState.Events.SlotEvicted.Hook(l.memPool.Evict) l.manaManager = mana.NewManager(l.apiProvider, l.resolveAccountOutput, l.accountsLedger.Account) latestCommittedSlot := e.Storage.Settings().LatestCommitment().Slot() @@ -143,19 +142,19 @@ func (l *Ledger) AttachTransaction(block *blocks.Block) (attachedTransaction mem return nil, false } -func (l *Ledger) CommitSlot(slot iotago.SlotIndex) (stateRoot iotago.Identifier, mutationRoot iotago.Identifier, accountRoot iotago.Identifier, created utxoledger.Outputs, consumed utxoledger.Spents, err error) { +func (l *Ledger) CommitSlot(slot iotago.SlotIndex) (stateRoot iotago.Identifier, mutationRoot iotago.Identifier, accountRoot iotago.Identifier, created utxoledger.Outputs, consumed utxoledger.Spents, mutations []*iotago.Transaction, err error) { ledgerIndex, err := l.utxoLedger.ReadLedgerSlot() if err != nil { - return iotago.Identifier{}, iotago.Identifier{}, iotago.Identifier{}, nil, nil, err + return iotago.Identifier{}, iotago.Identifier{}, iotago.Identifier{}, nil, nil, nil, err } if slot != ledgerIndex+1 { panic(ierrors.Errorf("there is a gap in the ledgerstate %d vs %d", ledgerIndex+1, slot)) } - stateDiff, err := l.memPool.StateDiff(slot) + stateDiff, err := l.memPool.CommitStateDiff(slot) if err != nil { - return iotago.Identifier{}, iotago.Identifier{}, iotago.Identifier{}, nil, nil, ierrors.Errorf("failed to retrieve state diff for slot %d: %w", slot, err) + return iotago.Identifier{}, iotago.Identifier{}, iotago.Identifier{}, nil, nil, nil, ierrors.Errorf("failed to retrieve state diff for slot %d: %w", slot, err) } // collect outputs and allotments from the "uncompacted" stateDiff @@ -163,7 +162,7 @@ func (l *Ledger) CommitSlot(slot iotago.SlotIndex) (stateRoot iotago.Identifier, // and retrieve intermediate outputs to show to the user spenders, outputs, accountDiffs, err := l.processStateDiffTransactions(stateDiff) if err != nil { - return iotago.Identifier{}, iotago.Identifier{}, iotago.Identifier{}, nil, nil, ierrors.Errorf("failed to process state diff transactions in slot %d: %w", slot, err) + return iotago.Identifier{}, iotago.Identifier{}, iotago.Identifier{}, nil, nil, nil, ierrors.Errorf("failed to process state diff transactions in slot %d: %w", slot, err) } // Now we process the collected account changes, for that we consume the "compacted" state diff to get the overall @@ -172,7 +171,7 @@ func (l *Ledger) CommitSlot(slot iotago.SlotIndex) (stateRoot iotago.Identifier, // output side createdAccounts, consumedAccounts, destroyedAccounts, err := l.processCreatedAndConsumedAccountOutputs(stateDiff, accountDiffs) if err != nil { - return iotago.Identifier{}, iotago.Identifier{}, iotago.Identifier{}, nil, nil, ierrors.Errorf("failed to process outputs consumed and created in slot %d: %w", slot, err) + return iotago.Identifier{}, iotago.Identifier{}, iotago.Identifier{}, nil, nil, nil, ierrors.Errorf("failed to process outputs consumed and created in slot %d: %w", slot, err) } l.prepareAccountDiffs(accountDiffs, slot, consumedAccounts, createdAccounts) @@ -180,7 +179,7 @@ func (l *Ledger) CommitSlot(slot iotago.SlotIndex) (stateRoot iotago.Identifier, // Commit the changes // Update the UTXO ledger if err = l.utxoLedger.ApplyDiff(slot, outputs, spenders); err != nil { - return iotago.Identifier{}, iotago.Identifier{}, iotago.Identifier{}, nil, nil, ierrors.Errorf("failed to apply diff to UTXO ledger for slot %d: %w", slot, err) + return iotago.Identifier{}, iotago.Identifier{}, iotago.Identifier{}, nil, nil, nil, ierrors.Errorf("failed to apply diff to UTXO ledger for slot %d: %w", slot, err) } // Update the Accounts ledger @@ -192,24 +191,33 @@ func (l *Ledger) CommitSlot(slot iotago.SlotIndex) (stateRoot iotago.Identifier, } rmcForSlot, err := l.rmcManager.RMC(rmcSlot) if err != nil { - return iotago.Identifier{}, iotago.Identifier{}, iotago.Identifier{}, nil, nil, ierrors.Errorf("ledger failed to get RMC for slot %d: %w", rmcSlot, err) + return iotago.Identifier{}, iotago.Identifier{}, iotago.Identifier{}, nil, nil, nil, ierrors.Errorf("ledger failed to get RMC for slot %d: %w", rmcSlot, err) } if err = l.accountsLedger.ApplyDiff(slot, rmcForSlot, accountDiffs, destroyedAccounts); err != nil { - return iotago.Identifier{}, iotago.Identifier{}, iotago.Identifier{}, nil, nil, ierrors.Errorf("failed to apply diff to Accounts ledger for slot %d: %w", slot, err) + return iotago.Identifier{}, iotago.Identifier{}, iotago.Identifier{}, nil, nil, nil, ierrors.Errorf("failed to apply diff to Accounts ledger for slot %d: %w", slot, err) } // Update the mana manager's cache if err = l.manaManager.ApplyDiff(slot, destroyedAccounts, createdAccounts, accountDiffs); err != nil { - return iotago.Identifier{}, iotago.Identifier{}, iotago.Identifier{}, nil, nil, ierrors.Errorf("failed to apply diff to mana manager for slot %d: %w", slot, err) + return iotago.Identifier{}, iotago.Identifier{}, iotago.Identifier{}, nil, nil, nil, ierrors.Errorf("failed to apply diff to mana manager for slot %d: %w", slot, err) } // Mark each transaction as committed so the mempool can evict it stateDiff.ExecutedTransactions().ForEach(func(_ iotago.TransactionID, tx mempool.TransactionMetadata) bool { tx.Commit() + + //nolint:forcetypeassert // we only handle a single type of transaction currently + mutations = append(mutations, tx.Transaction().(*iotago.Transaction)) + return true }) - return l.utxoLedger.StateTreeRoot(), stateDiff.Mutations().Root(), l.accountsLedger.AccountsTreeRoot(), outputs, spenders, nil + // We evict the mempool before we return, after updating the last committed slot value. + // We need to evict after updating the last committed slot value because otherwise transactions would be orphaned first + // (due to removing the attachments) and then committed, which would result in a broken state of the transaction. + l.memPool.Evict(slot) + + return l.utxoLedger.StateTreeRoot(), stateDiff.Mutations().Root(), l.accountsLedger.AccountsTreeRoot(), outputs, spenders, mutations, nil } func (l *Ledger) AddAccount(output *utxoledger.Output, blockIssuanceCredits iotago.BlockIssuanceCredits) error { diff --git a/pkg/protocol/engine/mempool/mempool.go b/pkg/protocol/engine/mempool/mempool.go index 85bc96f90..b92840871 100644 --- a/pkg/protocol/engine/mempool/mempool.go +++ b/pkg/protocol/engine/mempool/mempool.go @@ -25,7 +25,7 @@ type MemPool[VoteRank spenddag.VoteRankType[VoteRank]] interface { TransactionMetadataByAttachment(blockID iotago.BlockID) (transaction TransactionMetadata, exists bool) - StateDiff(slot iotago.SlotIndex) (StateDiff, error) + CommitStateDiff(slot iotago.SlotIndex) (StateDiff, error) Evict(slot iotago.SlotIndex) @@ -33,7 +33,7 @@ type MemPool[VoteRank spenddag.VoteRankType[VoteRank]] interface { Reset() } -// Denotes the type of state. +// StateType denotes the type of state. type StateType byte const ( diff --git a/pkg/protocol/engine/mempool/tests/testframework.go b/pkg/protocol/engine/mempool/tests/testframework.go index f82bbb2ed..eb5c75f9d 100644 --- a/pkg/protocol/engine/mempool/tests/testframework.go +++ b/pkg/protocol/engine/mempool/tests/testframework.go @@ -143,7 +143,7 @@ func (t *TestFramework) AttachTransaction(signedTransactionAlias, transactionAli } func (t *TestFramework) CommitSlot(slot iotago.SlotIndex) { - stateDiff, err := t.Instance.StateDiff(slot) + stateDiff, err := t.Instance.CommitStateDiff(slot) if err != nil { panic(err) } @@ -165,6 +165,8 @@ func (t *TestFramework) CommitSlot(slot iotago.SlotIndex) { return true }) + + t.Instance.Evict(slot) } func (t *TestFramework) TransactionMetadata(alias string) (mempool.TransactionMetadata, bool) { @@ -373,7 +375,7 @@ func (t *TestFramework) requireMarkedBooked(transactionAliases ...string) { } func (t *TestFramework) AssertStateDiff(slot iotago.SlotIndex, spentOutputAliases, createdOutputAliases, transactionAliases []string) { - stateDiff, err := t.Instance.StateDiff(slot) + stateDiff, err := t.Instance.CommitStateDiff(slot) require.NoError(t.test, err) require.Equal(t.test, len(spentOutputAliases), stateDiff.DestroyedStates().Size()) diff --git a/pkg/protocol/engine/mempool/tests/tests.go b/pkg/protocol/engine/mempool/tests/tests.go index 8f8f2bd10..f01965739 100644 --- a/pkg/protocol/engine/mempool/tests/tests.go +++ b/pkg/protocol/engine/mempool/tests/tests.go @@ -232,7 +232,8 @@ func TestSetTxOrphanageMultipleAttachments(t *testing.T, tf *TestFramework) { require.False(t, tx2Metadata.IsAccepted()) require.False(t, tx3Metadata.IsAccepted()) - tf.Instance.Evict(1) + tf.CommitSlot(1) + require.False(t, lo.Return2(tx1Metadata.OrphanedSlot())) require.False(t, lo.Return2(tx2Metadata.OrphanedSlot())) require.False(t, lo.Return2(tx3Metadata.OrphanedSlot())) @@ -241,7 +242,7 @@ func TestSetTxOrphanageMultipleAttachments(t *testing.T, tf *TestFramework) { require.True(t, lo.Return2(tf.SpendDAG.SpendSets(tf.TransactionID("tx2")))) require.True(t, lo.Return2(tf.SpendDAG.SpendSets(tf.TransactionID("tx3")))) - tf.Instance.Evict(2) + tf.CommitSlot(2) require.True(t, lo.Return2(tx1Metadata.OrphanedSlot())) require.True(t, lo.Return2(tx2Metadata.OrphanedSlot())) @@ -303,7 +304,7 @@ func TestStoreAttachmentInEvictedSlot(t *testing.T, tf *TestFramework) { debug.SetEnabled(true) defer debug.SetEnabled(false) - tf.Instance.Evict(iotago.SlotIndex(5)) + tf.CommitSlot(5) tf.CreateSignedTransaction("tx1", []string{"genesis"}, 1, true) require.Error(t, tf.AttachTransaction("tx1-signed", "tx1", "block2", 1)) diff --git a/pkg/protocol/engine/mempool/v1/mempool.go b/pkg/protocol/engine/mempool/v1/mempool.go index 7adbf9115..114f2071a 100644 --- a/pkg/protocol/engine/mempool/v1/mempool.go +++ b/pkg/protocol/engine/mempool/v1/mempool.go @@ -63,11 +63,11 @@ type MemPool[VoteRank spenddag.VoteRankType[VoteRank]] struct { // executionWorkers is the worker pool that is used to execute the state transitions of transactions. executionWorkers *workerpool.WorkerPool - // lastEvictedSlot is the last slot that was evicted from the MemPool. - lastEvictedSlot iotago.SlotIndex + // lastCommittedSlot is the last slot that was committed in the MemPool. + lastCommittedSlot iotago.SlotIndex - // evictionMutex is used to synchronize the eviction of slots. - evictionMutex syncutils.RWMutex + // commitmentMutex is used to synchronize commitment of slots. + commitmentMutex syncutils.RWMutex signedTransactionAttached *event.Event1[mempool.SignedTransactionMetadata] @@ -174,17 +174,24 @@ func (m *MemPool[VoteRank]) TransactionMetadataByAttachment(blockID iotago.Block return m.transactionByAttachment(blockID) } -// StateDiff returns the state diff for the given slot. -func (m *MemPool[VoteRank]) StateDiff(slot iotago.SlotIndex) (mempool.StateDiff, error) { - m.evictionMutex.RLock() - defer m.evictionMutex.RUnlock() +// CommitStateDiff commits the state diff for the given slot so that it cannot be modified anymore and returns it. +func (m *MemPool[VoteRank]) CommitStateDiff(slot iotago.SlotIndex) (mempool.StateDiff, error) { + m.commitmentMutex.Lock() + defer m.commitmentMutex.Unlock() - return m.stateDiff(slot) + stateDiff, err := m.stateDiff(slot) + if err != nil { + return nil, ierrors.Wrapf(err, "failed to retrieve StateDiff instance for slot %d", slot) + } + + m.lastCommittedSlot = slot + + return stateDiff, nil } func (m *MemPool[VoteRank]) stateDiff(slot iotago.SlotIndex) (*StateDiff, error) { - if m.lastEvictedSlot >= slot { - return nil, ierrors.Errorf("slot %d is older than last evicted slot %d", slot, m.lastEvictedSlot) + if m.lastCommittedSlot >= slot { + return nil, ierrors.Errorf("slot %d is older than last evicted slot %d", slot, m.lastCommittedSlot) } kv, err := m.mutationsFunc(slot) @@ -198,7 +205,7 @@ func (m *MemPool[VoteRank]) stateDiff(slot iotago.SlotIndex) (*StateDiff, error) // Reset resets the component to a clean state as if it was created at the last commitment. func (m *MemPool[VoteRank]) Reset() { m.stateDiffs.ForEachKey(func(slot iotago.SlotIndex) bool { - if slot > m.lastEvictedSlot { + if slot > m.lastCommittedSlot { if stateDiff, deleted := m.stateDiffs.DeleteAndReturn(slot); deleted { if err := stateDiff.Reset(); err != nil { m.errorHandler(ierrors.Wrapf(err, "failed to reset state diff for slot %d", slot)) @@ -211,7 +218,7 @@ func (m *MemPool[VoteRank]) Reset() { attachmentsToDelete := make([]iotago.SlotIndex, 0) m.attachments.ForEach(func(slot iotago.SlotIndex, _ *shrinkingmap.ShrinkingMap[iotago.BlockID, *SignedTransactionMetadata]) { - if slot > m.lastEvictedSlot { + if slot > m.lastCommittedSlot { attachmentsToDelete = append(attachmentsToDelete, slot) } }) @@ -229,10 +236,8 @@ func (m *MemPool[VoteRank]) Reset() { // Evict evicts the slot with the given slot from the MemPool. func (m *MemPool[VoteRank]) Evict(slot iotago.SlotIndex) { if evictedAttachments := func() *shrinkingmap.ShrinkingMap[iotago.BlockID, *SignedTransactionMetadata] { - m.evictionMutex.Lock() - defer m.evictionMutex.Unlock() - - m.lastEvictedSlot = slot + m.commitmentMutex.Lock() + defer m.commitmentMutex.Unlock() m.stateDiffs.Delete(slot) @@ -247,6 +252,9 @@ func (m *MemPool[VoteRank]) Evict(slot iotago.SlotIndex) { protocolParams := m.apiProvider.APIForSlot(slot).ProtocolParameters() genesisSlot := protocolParams.GenesisSlot() maxCommittableAge := protocolParams.MaxCommittableAge() + + // No need to evict delayed eviction slot if the committed slot is below maxCommittableAge, + // as there is nothing to evict anyway at this point. if slot <= genesisSlot+maxCommittableAge { return } @@ -272,12 +280,12 @@ func (m *MemPool[VoteRank]) Evict(slot iotago.SlotIndex) { } func (m *MemPool[VoteRank]) storeTransaction(signedTransaction mempool.SignedTransaction, transaction mempool.Transaction, blockID iotago.BlockID) (storedSignedTransaction *SignedTransactionMetadata, isNewSignedTransaction bool, isNewTransaction bool, err error) { - m.evictionMutex.RLock() - defer m.evictionMutex.RUnlock() + m.commitmentMutex.RLock() + defer m.commitmentMutex.RUnlock() - if m.lastEvictedSlot >= blockID.Slot() { + if m.lastCommittedSlot >= blockID.Slot() { // block will be retained as invalid, we do not store tx failure as it was block's fault - return nil, false, false, ierrors.Errorf("blockID %d is older than last evicted slot %d", blockID.Slot(), m.lastEvictedSlot) + return nil, false, false, ierrors.Errorf("blockID %d is older than last evicted slot %d", blockID.Slot(), m.lastCommittedSlot) } inputReferences, err := m.vm.Inputs(transaction) @@ -409,10 +417,10 @@ func (m *MemPool[VoteRank]) requestState(stateRef mempool.StateReference, waitIf } func (m *MemPool[VoteRank]) updateAttachment(blockID iotago.BlockID, updateFunc func(transaction *TransactionMetadata, blockID iotago.BlockID) bool) bool { - m.evictionMutex.RLock() - defer m.evictionMutex.RUnlock() + m.commitmentMutex.RLock() + defer m.commitmentMutex.RUnlock() - if m.lastEvictedSlot < blockID.Slot() { + if m.lastCommittedSlot < blockID.Slot() { if transaction, exists := m.transactionByAttachment(blockID); exists { return updateFunc(transaction, blockID) } @@ -461,8 +469,8 @@ func (m *MemPool[VoteRank]) updateStateDiffs(transaction *TransactionMetadata, p func (m *MemPool[VoteRank]) setup() { m.spendDAG.Events().SpenderAccepted.Hook(func(id iotago.TransactionID) { if transaction, exists := m.cachedTransactions.Get(id); exists { - m.evictionMutex.RLock() - defer m.evictionMutex.RUnlock() + m.commitmentMutex.RLock() + defer m.commitmentMutex.RUnlock() transaction.setConflictAccepted() } diff --git a/pkg/protocol/engine/mempool/v1/mempool_test.go b/pkg/protocol/engine/mempool/v1/mempool_test.go index 957ff0208..8f15b7926 100644 --- a/pkg/protocol/engine/mempool/v1/mempool_test.go +++ b/pkg/protocol/engine/mempool/v1/mempool_test.go @@ -60,7 +60,6 @@ func TestMempoolV1_ResourceCleanup(t *testing.T) { prevStateAlias = fmt.Sprintf("tx%d:0", index) tf.CommitSlot(iotago.SlotIndex(index)) - tf.Instance.Evict(iotago.SlotIndex(index)) require.Nil(t, memPoolInstance.attachments.Get(iotago.SlotIndex(index), false)) @@ -111,7 +110,6 @@ func TestMempoolV1_ResourceCleanup(t *testing.T) { for index := txIndex; index <= txIndex+20; index++ { tf.CommitSlot(iotago.SlotIndex(index)) - tf.Instance.Evict(iotago.SlotIndex(index)) } // Genesis output is also finally evicted thanks to the fact that all its spenders got evicted. diff --git a/pkg/protocol/engine/mempool/v1/state_diff.go b/pkg/protocol/engine/mempool/v1/state_diff.go index 305eeef1e..3a47088f6 100644 --- a/pkg/protocol/engine/mempool/v1/state_diff.go +++ b/pkg/protocol/engine/mempool/v1/state_diff.go @@ -125,7 +125,7 @@ func (s *StateDiff) Reset() error { } } - return nil + return s.mutations.Commit() } func (s *StateDiff) compactStateChanges(stateMetadata *StateMetadata, usageCounter int) { diff --git a/pkg/protocol/engine/notarization/events.go b/pkg/protocol/engine/notarization/events.go index 81aaa5e8a..9490e1def 100644 --- a/pkg/protocol/engine/notarization/events.go +++ b/pkg/protocol/engine/notarization/events.go @@ -31,4 +31,5 @@ type SlotCommittedDetails struct { ActiveValidatorsCount int OutputsCreated utxoledger.Outputs OutputsConsumed utxoledger.Spents + Mutations []*iotago.Transaction } diff --git a/pkg/protocol/engine/notarization/slotnotarization/manager.go b/pkg/protocol/engine/notarization/slotnotarization/manager.go index d776ea50d..c4bdf586b 100644 --- a/pkg/protocol/engine/notarization/slotnotarization/manager.go +++ b/pkg/protocol/engine/notarization/slotnotarization/manager.go @@ -122,6 +122,8 @@ func (m *Manager) tryCommitUntil(commitUntilSlot iotago.SlotIndex) { } func (m *Manager) ForceCommit(slot iotago.SlotIndex) (*model.Commitment, error) { + m.LogInfof("Force commit slot %d", slot) + if m.WasStopped() { return nil, ierrors.New("notarization manager was stopped") } @@ -144,6 +146,8 @@ func (m *Manager) ForceCommit(slot iotago.SlotIndex) (*model.Commitment, error) } func (m *Manager) ForceCommitUntil(commitUntilSlot iotago.SlotIndex) error { + m.LogInfof("Force commit until slot %d", commitUntilSlot) + for i := m.storage.Settings().LatestCommitment().Slot() + 1; i <= commitUntilSlot; i++ { if _, err := m.ForceCommit(i); err != nil { return ierrors.Wrapf(err, "failed to force commit slot %d", i) @@ -208,10 +212,9 @@ func (m *Manager) createCommitment(slot iotago.SlotIndex) (*model.Commitment, er return nil, ierrors.Errorf("cannot create commitment for slot %d, latest commitment is for slot %d", slot, latestCommitment.Slot()) } - // Set createIfMissing to true to make sure that this is never nil. Will get evicted later on anyway. - acceptedBlocks := m.slotMutations.AcceptedBlocks(slot, true) - if err := acceptedBlocks.Commit(); err != nil { - return nil, ierrors.Wrap(err, "failed to commit accepted blocks") + acceptedBlocksSet, err := m.slotMutations.Commit(slot) + if err != nil { + return nil, ierrors.Wrap(err, "failed to commit acceptedBlocksSet") } cumulativeWeight, attestationsRoot, err := m.attestation.Commit(slot) @@ -219,7 +222,7 @@ func (m *Manager) createCommitment(slot iotago.SlotIndex) (*model.Commitment, er return nil, ierrors.Wrap(err, "failed to commit attestations") } - stateRoot, mutationRoot, accountRoot, created, consumed, err := m.ledger.CommitSlot(slot) + stateRoot, mutationRoot, accountRoot, created, consumed, mutations, err := m.ledger.CommitSlot(slot) if err != nil { return nil, ierrors.Wrap(err, "failed to commit ledger") } @@ -236,7 +239,7 @@ func (m *Manager) createCommitment(slot iotago.SlotIndex) (*model.Commitment, er } roots := iotago.NewRoots( - acceptedBlocks.Root(), + acceptedBlocksSet.Root(), mutationRoot, attestationsRoot, stateRoot, @@ -282,10 +285,11 @@ func (m *Manager) createCommitment(slot iotago.SlotIndex) (*model.Commitment, er m.events.SlotCommitted.Trigger(¬arization.SlotCommittedDetails{ Commitment: newModelCommitment, - AcceptedBlocks: acceptedBlocks, + AcceptedBlocks: acceptedBlocksSet, ActiveValidatorsCount: 0, OutputsCreated: created, OutputsConsumed: consumed, + Mutations: mutations, }) if err = m.storage.Settings().SetLatestCommitment(newModelCommitment); err != nil { @@ -294,10 +298,6 @@ func (m *Manager) createCommitment(slot iotago.SlotIndex) (*model.Commitment, er m.events.LatestCommitmentUpdated.Trigger(newModelCommitment) - if err = m.slotMutations.Evict(slot); err != nil { - m.errorHandler(ierrors.Wrapf(err, "failed to evict slotMutations at slot: %d", slot)) - } - return newModelCommitment, nil } diff --git a/pkg/protocol/engine/notarization/slotnotarization/slotmutations.go b/pkg/protocol/engine/notarization/slotnotarization/slotmutations.go index aedee8fec..af17fa755 100644 --- a/pkg/protocol/engine/notarization/slotnotarization/slotmutations.go +++ b/pkg/protocol/engine/notarization/slotnotarization/slotmutations.go @@ -16,8 +16,8 @@ type SlotMutations struct { // acceptedBlocksBySlot stores the accepted blocks per slot. acceptedBlocksBySlot *shrinkingmap.ShrinkingMap[iotago.SlotIndex, ads.Set[iotago.Identifier, iotago.BlockID]] - // latestCommittedIndex stores the index of the latest committed slot. - latestCommittedIndex iotago.SlotIndex + // latestCommittedSlot stores the index of the latest committed slot. + latestCommittedSlot iotago.SlotIndex evictionMutex syncutils.RWMutex } @@ -26,7 +26,7 @@ type SlotMutations struct { func NewSlotMutations(lastCommittedSlot iotago.SlotIndex) *SlotMutations { return &SlotMutations{ acceptedBlocksBySlot: shrinkingmap.New[iotago.SlotIndex, ads.Set[iotago.Identifier, iotago.BlockID]](), - latestCommittedIndex: lastCommittedSlot, + latestCommittedSlot: lastCommittedSlot, } } @@ -36,40 +36,43 @@ func (m *SlotMutations) AddAcceptedBlock(block *blocks.Block) (err error) { defer m.evictionMutex.RUnlock() blockID := block.ID() - if blockID.Slot() <= m.latestCommittedIndex { + if blockID.Slot() <= m.latestCommittedSlot { return ierrors.Errorf("cannot add block %s: slot with %d is already committed", blockID, blockID.Slot()) } - if err := m.AcceptedBlocks(blockID.Slot(), true).Add(blockID); err != nil { + if err := m.acceptedBlocks(blockID.Slot(), true).Add(blockID); err != nil { return ierrors.Wrapf(err, "failed to add block to accepted blocks, blockID: %s", blockID.ToHex()) } return } -// Evict evicts the given slot. -func (m *SlotMutations) Evict(index iotago.SlotIndex) error { +// Reset resets the component to a clean state as if it was created at the last commitment. +func (m *SlotMutations) Reset() { + m.acceptedBlocksBySlot.Clear() +} + +func (m *SlotMutations) Commit(slot iotago.SlotIndex) (ads.Set[iotago.Identifier, iotago.BlockID], error) { m.evictionMutex.Lock() defer m.evictionMutex.Unlock() - if index <= m.latestCommittedIndex { - return ierrors.Errorf("cannot commit slot %d: already committed", index) + // If for whatever reason no blocks exist (empty slot), then we still want to create the ads.Set and get the root from it. + acceptedBlocksSet := m.acceptedBlocks(slot, true) + + if err := acceptedBlocksSet.Commit(); err != nil { + return nil, ierrors.Wrap(err, "failed to commit accepted blocks") } - m.evictUntil(index) + m.acceptedBlocksBySlot.Delete(slot) + m.latestCommittedSlot = slot - return nil + return acceptedBlocksSet, nil } -// Reset resets the component to a clean state as if it was created at the last commitment. -func (m *SlotMutations) Reset() { - m.acceptedBlocksBySlot.Clear() -} - -// AcceptedBlocks returns the set of accepted blocks for the given slot. -func (m *SlotMutations) AcceptedBlocks(index iotago.SlotIndex, createIfMissing ...bool) ads.Set[iotago.Identifier, iotago.BlockID] { +// acceptedBlocks returns the set of accepted blocks for the given slot. +func (m *SlotMutations) acceptedBlocks(slot iotago.SlotIndex, createIfMissing ...bool) ads.Set[iotago.Identifier, iotago.BlockID] { if len(createIfMissing) > 0 && createIfMissing[0] { - return lo.Return1(m.acceptedBlocksBySlot.GetOrCreate(index, func() ads.Set[iotago.Identifier, iotago.BlockID] { + return lo.Return1(m.acceptedBlocksBySlot.GetOrCreate(slot, func() ads.Set[iotago.Identifier, iotago.BlockID] { return ads.NewSet[iotago.Identifier]( mapdb.NewMapDB(), iotago.Identifier.Bytes, @@ -80,7 +83,7 @@ func (m *SlotMutations) AcceptedBlocks(index iotago.SlotIndex, createIfMissing . })) } - return lo.Return1(m.acceptedBlocksBySlot.Get(index)) + return lo.Return1(m.acceptedBlocksBySlot.Get(slot)) } func (m *SlotMutations) AcceptedBlocksCount(index iotago.SlotIndex) int { @@ -91,12 +94,3 @@ func (m *SlotMutations) AcceptedBlocksCount(index iotago.SlotIndex) int { return acceptedBlocks.Size() } - -// evictUntil removes all data for slots that are older than the given slot. -func (m *SlotMutations) evictUntil(index iotago.SlotIndex) { - for i := m.latestCommittedIndex + 1; i <= index; i++ { - m.acceptedBlocksBySlot.Delete(i) - } - - m.latestCommittedIndex = index -} diff --git a/pkg/protocol/engine/tipmanager/v1/provider.go b/pkg/protocol/engine/tipmanager/v1/provider.go index 0a824884a..e1c8a84af 100644 --- a/pkg/protocol/engine/tipmanager/v1/provider.go +++ b/pkg/protocol/engine/tipmanager/v1/provider.go @@ -21,7 +21,7 @@ func NewProvider() module.Provider[*engine.Engine, tipmanager.TipManager] { e.Events.Scheduler.BlockScheduled.Hook(lo.Void(t.AddBlock), event.WithWorkerPool(tipWorker)) e.Events.Scheduler.BlockSkipped.Hook(lo.Void(t.AddBlock), event.WithWorkerPool(tipWorker)) - e.BlockCache.Evict.Hook(t.Evict) + e.Events.Evict.Hook(t.Evict) e.Events.SeatManager.OnlineCommitteeSeatAdded.Hook(func(index account.SeatIndex, _ iotago.AccountID) { t.AddSeat(index) diff --git a/pkg/protocol/warp_sync.go b/pkg/protocol/warp_sync.go index 7eff35cf2..84f06fa51 100644 --- a/pkg/protocol/warp_sync.go +++ b/pkg/protocol/warp_sync.go @@ -150,7 +150,7 @@ func (w *WarpSync) ProcessResponse(commitmentID iotago.CommitmentID, blockIDsByS } if !iotago.VerifyProof(mutationProof, acceptedTransactionIDs.Root(), commitment.RootsID()) { - w.LogError("failed to verify mutations proof", "commitment", commitment.LogName(), "transactionIDs", transactionIDs, "proof", mutationProof, "fromPeer", from) + w.LogError("failed to verify mutations proof", "commitment", commitment.ID(), commitment.Commitment.Commitment().String(), "acceptedTransactionIDsRoot", acceptedTransactionIDs.Root(), "transactionIDs len()", len(transactionIDs), "proof", mutationProof, "fromPeer", from) return blocksToWarpSync } diff --git a/tools/docker-network/.env b/tools/docker-network/.env index 0410a179a..025ff1d9e 100644 --- a/tools/docker-network/.env +++ b/tools/docker-network/.env @@ -1,6 +1,6 @@ COMMON_CONFIG=" -c config.json ---logger.level=info +--logger.level=debug --logger.outputPaths=stdout --p2p.db.path=/app/data/peerdb --profiling.enabled=true diff --git a/tools/docker-network/grafana/provisioning/dashboards/global_metrics.json b/tools/docker-network/grafana/provisioning/dashboards/commitments-overview.json similarity index 65% rename from tools/docker-network/grafana/provisioning/dashboards/global_metrics.json rename to tools/docker-network/grafana/provisioning/dashboards/commitments-overview.json index b0ea4b97c..bf5f45e6f 100644 --- a/tools/docker-network/grafana/provisioning/dashboards/global_metrics.json +++ b/tools/docker-network/grafana/provisioning/dashboards/commitments-overview.json @@ -24,7 +24,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 1, + "id": 2, "links": [], "liveNow": false, "panels": [ @@ -65,7 +65,7 @@ }, "gridPos": { "h": 6, - "w": 4.8, + "w": 4, "x": 0, "y": 1 }, @@ -240,7 +240,7 @@ "mode": "fixed" }, "custom": { - "align": "auto", + "align": "left", "cellOptions": { "type": "auto" }, @@ -257,34 +257,25 @@ } ] }, - "unit": "SI" + "unit": "none" }, "overrides": [ { "matcher": { "id": "byName", - "options": "slot" + "options": "Slot" }, "properties": [ { - "id": "custom.cellOptions", - "value": { - "type": "color-text" - } - }, - { - "id": "color", - "value": { - "fixedColor": "dark-green", - "mode": "fixed" - } + "id": "custom.width", + "value": 40 } ] }, { "matcher": { "id": "byName", - "options": "commitment" + "options": "Slot" }, "properties": [ { @@ -307,7 +298,7 @@ }, "gridPos": { "h": 8, - "w": 4.8, + "w": 4, "x": 0, "y": 15 }, @@ -382,8 +373,16 @@ "instance": true, "job": true }, - "indexByName": {}, + "indexByName": { + "Time": 0, + "Value": 2, + "__name__": 1, + "commitment": 3, + "instance": 4, + "job": 5 + }, "renameByName": { + "Value": "Slot", "commitment": "Commitment", "slot": "Slot Index" } @@ -392,6 +391,229 @@ ], "transparent": true, "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 140, + "panels": [], + "title": "Slot Commitment - Details", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed" + }, + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Slot" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "basic", + "type": "color-background" + } + }, + { + "id": "color", + "value": { + "fixedColor": "#143003", + "mode": "fixed" + } + }, + { + "id": "custom.width", + "value": 52 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "# Blocks" + }, + "properties": [ + { + "id": "custom.width", + "value": 50 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "# TX" + }, + "properties": [ + { + "id": "custom.width", + "value": 40 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 0, + "y": 24 + }, + "id": 151, + "maxPerRow": 6, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": false, + "fields": "", + "reducer": [ + "last" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.5.6", + "repeat": "instance", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "commitments_accepted_blocks{instance=~\"$instance\"}", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "commitments_accepted_transactions{instance=~\"$instance\"}", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "B" + } + ], + "title": "Details $instance", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "slot", + "mode": "outer" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "number", + "targetField": "slot" + } + ], + "fields": {} + } + }, + { + "id": "limit", + "options": { + "limitField": 100 + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": false, + "__name__": true, + "instance": true, + "job": true + }, + "indexByName": { + "Time": 0, + "Value": 3, + "__name__": 1, + "instance": 4, + "job": 5, + "slot": 2 + }, + "renameByName": { + "Value": "Blocks", + "Value #A": "# Blocks", + "Value #B": "# TX", + "commitment": "Commitment", + "slot": "Slot" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": true, + "field": "Slot" + } + ] + } + } + ], + "transparent": true, + "type": "table" } ], "refresh": "5s", @@ -452,7 +674,7 @@ ] }, "time": { - "from": "now-5m", + "from": "now-15m", "to": "now" }, "timepicker": { @@ -470,8 +692,8 @@ ] }, "timezone": "", - "title": "Global Nodes View", + "title": "Commitments Overview", "uid": "JKcc02v4d", - "version": 2, + "version": 1, "weekStart": "" } diff --git a/tools/docker-network/prometheus.yml b/tools/docker-network/prometheus.yml index 07728e80f..c6458b27f 100644 --- a/tools/docker-network/prometheus.yml +++ b/tools/docker-network/prometheus.yml @@ -6,6 +6,7 @@ scrape_configs: - node-1-validator:9311 - node-2-validator:9311 - node-3-validator:9311 + - node-4-validator:9311 - node-4:9311 - node-5:9311 dns_sd_configs: diff --git a/tools/genesis-snapshot/presets/presets.go b/tools/genesis-snapshot/presets/presets.go index d6392c53c..97934ebc0 100644 --- a/tools/genesis-snapshot/presets/presets.go +++ b/tools/genesis-snapshot/presets/presets.go @@ -24,7 +24,8 @@ var ( // use defaults from iota.go protocolParamsDocker = iotago.NewV3SnapshotProtocolParameters( iotago.WithNetworkOptions("docker", iotago.PrefixTestnet), - iotago.WithTimeProviderOptions(5, time.Now().Unix(), 10, 13), + iotago.WithTimeProviderOptions(5, time.Now().Unix(), 10, 7), + iotago.WithLivenessOptions(10, 15, 3, 6, 8), ) // use defaults from iota.go