From e5aeb47af19d2e59507104754e946227865d5bab Mon Sep 17 00:00:00 2001 From: Andrew Date: Thu, 31 Aug 2023 17:56:41 +0100 Subject: [PATCH 01/24] start implementing validation block scheduler --- .../scheduler/drr/scheduler.go | 119 +++++++++++++++--- .../scheduler/drr/validatorqueue.go | 61 +++++++++ .../congestioncontrol/scheduler/scheduler.go | 2 +- 3 files changed, 166 insertions(+), 16 deletions(-) create mode 100644 pkg/protocol/engine/congestioncontrol/scheduler/drr/validatorqueue.go diff --git a/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go b/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go index df7bd39e7..6f3ac0f6d 100644 --- a/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go +++ b/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go @@ -20,6 +20,8 @@ import ( type Deficit int64 +type SubSlotIndex int + type Scheduler struct { events *scheduler.Events @@ -29,8 +31,9 @@ type Scheduler struct { apiProvider api.Provider - buffer *BufferQueue - bufferMutex syncutils.RWMutex + buffer *BufferQueue + validatorBuffer map[iotago.AccountID]*ValidatorQueue + bufferMutex syncutils.RWMutex deficits *shrinkingmap.ShrinkingMap[iotago.AccountID, Deficit] @@ -43,6 +46,12 @@ type Scheduler struct { blockCache *blocks.Blocks + validationBlocksPerSlot SubSlotIndex // TODO: make this a protocol parameter + + blacklist map[iotago.AccountID]struct{} + + validationBlocks map[iotago.SlotIndex]map[SubSlotIndex]*blocks.Block + errorHandler func(error) module.Module @@ -78,18 +87,9 @@ func NewProvider(opts ...options.Option[Scheduler]) module.Provider[*engine.Engi }) s.TriggerConstructed() e.Events.Booker.BlockBooked.Hook(func(block *blocks.Block) { - if _, isBasic := block.BasicBlock(); isBasic { - s.AddBlock(block) - s.selectBlockToScheduleWithLocking() - } else { // immediately schedule validator blocks for now. TODO: implement scheduling for validator blocks issue #236 - block.SetEnqueued() - block.SetScheduled() - // check for another block ready to schedule - s.updateChildrenWithLocking(block) - s.selectBlockToScheduleWithLocking() - - s.events.BlockScheduled.Trigger(block) - } + s.AddBlock(block) + s.selectBlockToScheduleWithLocking() + }) e.Events.Ledger.AccountCreated.Hook(func(accountID iotago.AccountID) { s.bufferMutex.Lock() @@ -122,6 +122,18 @@ func New(apiProvider api.Provider, opts ...options.Option[Scheduler]) *Scheduler ) } +func (s *Scheduler) subSlotIndex(block *blocks.Block) SubSlotIndex { + apiForBlock, err := s.apiProvider.APIForVersion(block.ProtocolBlock().ProtocolVersion) + if err != nil { + s.errorHandler(ierrors.Wrapf(err, "failed to get API for version %d", block.ProtocolBlock().ProtocolVersion)) + } + blockSlot := apiForBlock.TimeProvider().SlotFromTime(block.IssuingTime()) + apiForBlock.TimeProvider().SlotStartTime(blockSlot) + timeSinceSlotStart := block.IssuingTime().Sub(apiForBlock.TimeProvider().SlotStartTime(blockSlot)) + + return SubSlotIndex(timeSinceSlotStart.Seconds() * float64(s.validationBlocksPerSlot) / float64(apiForBlock.TimeProvider().SlotDurationSeconds())) +} + func (s *Scheduler) Shutdown() { close(s.shutdownSignal) s.TriggerStopped() @@ -204,6 +216,16 @@ func (s *Scheduler) IsBlockIssuerReady(accountID iotago.AccountID, blocks ...*bl } func (s *Scheduler) AddBlock(block *blocks.Block) { + if _, isBasic := block.BasicBlock(); isBasic { + s.enqueueBasicBlock(block) + } else if _, isValidation := block.ValidationBlock(); isValidation { + s.enqueueValidationBlock(block) + } else { + panic("invalid block type") + } +} + +func (s *Scheduler) enqueueBasicBlock(block *blocks.Block) { s.bufferMutex.Lock() defer s.bufferMutex.Unlock() @@ -246,6 +268,43 @@ func (s *Scheduler) AddBlock(block *blocks.Block) { } } +func (s *Scheduler) enqueueValidationBlock(block *blocks.Block) { + s.bufferMutex.Lock() + defer s.bufferMutex.Unlock() + + if _, exists := s.blacklist[block.ProtocolBlock().IssuerID]; exists { + // this validator is blacklisted, so skip this block + return + } + + // TODO: double check that the validator is in the committee for this epoch? + validatorQueue, exists := s.validatorBuffer[block.ProtocolBlock().IssuerID] + if !exists { + validatorQueue = NewValidatorQueue(block.ProtocolBlock().IssuerID) + s.validatorBuffer[block.ProtocolBlock().IssuerID] = validatorQueue + } + validatorQueue.Submit(block) + + if slotValidationBlocks := s.validationBlocks[block.ID().Index()]; slotValidationBlocks != nil { + subSlot := s.subSlotIndex(block) + if _, has := slotValidationBlocks[subSlot]; has { + // the validator has already issued a block for this subslot, so we need to blacklist them. + s.blacklist[block.ProtocolBlock().IssuerID] = struct{}{} + // TODO: retrieve the block that has already been issued for the proof of equivocation + // continue with enqueueing this block as we want to enqueue the first two blocks for the validator for the subslot + } + slotValidationBlocks[subSlot] = block + } + + if block.SetEnqueued() { + s.events.BlockEnqueued.Trigger(block) + } + + if s.tryReadyValidationBlock(block) { + s.scheduleValidationBlock(block) + } +} + func (s *Scheduler) mainLoop() { var blockToSchedule *blocks.Block loop: @@ -287,6 +346,13 @@ func (s *Scheduler) scheduleBlock(block *blocks.Block) { } } +func (s *Scheduler) scheduleValidationBlock(block *blocks.Block) { + if block.SetScheduled() { + s.events.BlockScheduled.Trigger(block) + s.updateChildrenWithoutLocking(block) + } +} + func (s *Scheduler) selectBlockToScheduleWithLocking() { s.bufferMutex.Lock() defer s.bufferMutex.Unlock() @@ -531,10 +597,25 @@ func (s *Scheduler) tryReady(block *blocks.Block) { } } +// tryReadyValidator tries to set the given validation block as ready. +func (s *Scheduler) tryReadyValidationBlock(block *blocks.Block) bool { + if s.isReady(block) { + s.readyValidationBlock(block) + + return true + } + + return false +} + func (s *Scheduler) ready(block *blocks.Block) { s.buffer.Ready(block) } +func (s *Scheduler) readyValidationBlock(block *blocks.Block) { + s.validatorBuffer[block.ProtocolBlock().IssuerID].Unsubmit(block) +} + // updateChildrenWithLocking locks the buffer mutex and iterates over the direct children of the given blockID and // tries to mark them as ready. func (s *Scheduler) updateChildrenWithLocking(block *blocks.Block) { @@ -549,7 +630,15 @@ func (s *Scheduler) updateChildrenWithLocking(block *blocks.Block) { func (s *Scheduler) updateChildrenWithoutLocking(block *blocks.Block) { for _, childBlock := range block.Children() { if _, childBlockExists := s.blockCache.Block(childBlock.ID()); childBlockExists && childBlock.IsEnqueued() { - s.tryReady(childBlock) + if _, isBasic := childBlock.BasicBlock(); isBasic { + s.tryReady(childBlock) + } else if _, isValidation := childBlock.ValidationBlock(); isValidation { + if s.tryReadyValidationBlock(childBlock) { + s.scheduleValidationBlock(childBlock) + } + } else { + panic("invalid block type") + } } } } diff --git a/pkg/protocol/engine/congestioncontrol/scheduler/drr/validatorqueue.go b/pkg/protocol/engine/congestioncontrol/scheduler/drr/validatorqueue.go new file mode 100644 index 000000000..cbf3d9b49 --- /dev/null +++ b/pkg/protocol/engine/congestioncontrol/scheduler/drr/validatorqueue.go @@ -0,0 +1,61 @@ +package drr + +import ( + "fmt" + + "github.com/iotaledger/hive.go/ds/shrinkingmap" + "github.com/iotaledger/iota-core/pkg/protocol/engine/blocks" + iotago "github.com/iotaledger/iota.go/v4" + "go.uber.org/atomic" +) + +type ValidatorQueue struct { + accountID iotago.AccountID + submitted *shrinkingmap.ShrinkingMap[iotago.BlockID, *blocks.Block] + size atomic.Int64 +} + +func NewValidatorQueue(accountID iotago.AccountID) *ValidatorQueue { + return &ValidatorQueue{ + accountID: accountID, + submitted: shrinkingmap.New[iotago.BlockID, *blocks.Block](), + } +} + +func (q *ValidatorQueue) Size() int { + if q == nil { + return 0 + } + + return int(q.size.Load()) +} + +func (q *ValidatorQueue) AccountID() iotago.AccountID { + return q.accountID +} + +func (q *ValidatorQueue) Submit(block *blocks.Block) bool { + if blkAccountID := block.ProtocolBlock().IssuerID; q.accountID != blkAccountID { + panic(fmt.Sprintf("issuerqueue: queue issuer ID(%x) and issuer ID(%x) does not match.", q.accountID, blkAccountID)) + } + + if _, submitted := q.submitted.Get(block.ID()); submitted { + return false + } + + q.submitted.Set(block.ID(), block) + q.size.Inc() + + return true +} + +func (q *ValidatorQueue) Unsubmit(block *blocks.Block) bool { + if _, submitted := q.submitted.Get(block.ID()); !submitted { + return false + } + + q.submitted.Delete(block.ID()) + q.size.Dec() + + return true +} diff --git a/pkg/protocol/engine/congestioncontrol/scheduler/scheduler.go b/pkg/protocol/engine/congestioncontrol/scheduler/scheduler.go index 28e81f71d..197fe87ce 100644 --- a/pkg/protocol/engine/congestioncontrol/scheduler/scheduler.go +++ b/pkg/protocol/engine/congestioncontrol/scheduler/scheduler.go @@ -7,7 +7,7 @@ import ( ) type Scheduler interface { - // AddBlock adds a block to the scheduling buffer. + // AddBasicBlock adds a basic block to the scheduling buffer. AddBlock(*blocks.Block) // IsBlockIssuerReady returns true if the block issuer is ready to issuer a block, i.e., if the block issuer were to add a block to the scheduler, would it be scheduled. IsBlockIssuerReady(iotago.AccountID, ...*blocks.Block) bool From cb2c5311fbb81399e8fc8a107f024b9f6f4a4370 Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 6 Sep 2023 11:02:46 +0100 Subject: [PATCH 02/24] switch to simplified validation scheduler --- go.mod | 2 +- go.sum | 7 + .../scheduler/drr/drrbuffer.go | 34 ++- .../scheduler/drr/scheduler.go | 225 +++++++++--------- .../scheduler/drr/validatorqueue.go | 110 ++++++++- .../congestioncontrol/scheduler/scheduler.go | 2 +- pkg/testsuite/testsuite.go | 2 +- 7 files changed, 267 insertions(+), 115 deletions(-) diff --git a/go.mod b/go.mod index 8ee963cad..9097dbb0f 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/iotaledger/hive.go/stringify v0.0.0-20230829152614-7afc7a4d89b3 github.com/iotaledger/inx-app v1.0.0-rc.3.0.20230829161228-3f4eb50a4d14 github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230829160617-69b96c7c9f9b - github.com/iotaledger/iota.go/v4 v4.0.0-20230829160021-46cad51e89d1 + github.com/iotaledger/iota.go/v4 v4.0.0-20230905174229-dfb67db9ea20 github.com/labstack/echo/v4 v4.11.1 github.com/labstack/gommon v0.4.0 github.com/libp2p/go-libp2p v0.30.0 diff --git a/go.sum b/go.sum index 0b10673ac..c5e17fcd1 100644 --- a/go.sum +++ b/go.sum @@ -129,6 +129,7 @@ github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -294,6 +295,10 @@ github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230829160617-69b96c7c9f9b h1:EPB/+i github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230829160617-69b96c7c9f9b/go.mod h1:B7gyJP6GshCSlEmY3CxEk5TZdsMs3UNz5U92hkFDdMs= github.com/iotaledger/iota.go/v4 v4.0.0-20230829160021-46cad51e89d1 h1:10hGLm62uQ2V2HgqCR6FV0fvgpXo5GqlL/SIJ/t4VmY= github.com/iotaledger/iota.go/v4 v4.0.0-20230829160021-46cad51e89d1/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= +github.com/iotaledger/iota.go/v4 v4.0.0-20230905160612-e6a89d4bccc9 h1:72n+obvx7Ph6D0Z5ygEq5x47hLSX61eRTkDejWZAzxw= +github.com/iotaledger/iota.go/v4 v4.0.0-20230905160612-e6a89d4bccc9/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= +github.com/iotaledger/iota.go/v4 v4.0.0-20230905174229-dfb67db9ea20 h1:ZfZBm80qbrTDKrHyrvg2x5IO54RkMJwInnpvg2qckpc= +github.com/iotaledger/iota.go/v4 v4.0.0-20230905174229-dfb67db9ea20/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= @@ -313,6 +318,7 @@ github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= @@ -423,6 +429,7 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= diff --git a/pkg/protocol/engine/congestioncontrol/scheduler/drr/drrbuffer.go b/pkg/protocol/engine/congestioncontrol/scheduler/drr/drrbuffer.go index 572743971..88672c58b 100644 --- a/pkg/protocol/engine/congestioncontrol/scheduler/drr/drrbuffer.go +++ b/pkg/protocol/engine/congestioncontrol/scheduler/drr/drrbuffer.go @@ -3,9 +3,11 @@ package drr import ( "container/ring" "math" + "time" "github.com/iotaledger/hive.go/ds/shrinkingmap" "github.com/iotaledger/hive.go/ierrors" + "github.com/iotaledger/hive.go/lo" "github.com/iotaledger/iota-core/pkg/protocol/engine/blocks" iotago "github.com/iotaledger/iota.go/v4" @@ -25,14 +27,22 @@ type BufferQueue struct { ring *ring.Ring // size is the number of blocks in the buffer. size int + + tokenBucket float64 + lastScheduleTime time.Time + + blockChan chan *blocks.Block + timer *time.Timer } // NewBufferQueue returns a new BufferQueue. func NewBufferQueue(maxBuffer int) *BufferQueue { return &BufferQueue{ - maxBuffer: maxBuffer, - activeIssuers: shrinkingmap.New[iotago.AccountID, *ring.Ring](), - ring: nil, + maxBuffer: maxBuffer, + activeIssuers: shrinkingmap.New[iotago.AccountID, *ring.Ring](), + ring: nil, + lastScheduleTime: time.Now(), + blockChan: make(chan *blocks.Block, 1), } } @@ -320,4 +330,22 @@ func (b *BufferQueue) ringInsert(v interface{}) *ring.Ring { return p.Link(b.ring) } +func (b *BufferQueue) waitTime(rate float64, block *blocks.Block) time.Duration { + tokensRequired := float64(block.WorkScore()) - (b.tokenBucket + rate*time.Since(b.lastScheduleTime).Seconds()) + + return lo.Max(0, time.Duration(tokensRequired/rate)) +} + +func (b *BufferQueue) updateTokenBucket(rate float64, tokenBucketSize float64) { + b.tokenBucket = lo.Min( + tokenBucketSize, + b.tokenBucket+rate*time.Since(b.lastScheduleTime).Seconds(), + ) + b.lastScheduleTime = time.Now() +} + +func (b *BufferQueue) deductTokens(tokens float64) { + b.tokenBucket -= tokens +} + // endregion /////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go b/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go index 6f3ac0f6d..925f0eda5 100644 --- a/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go +++ b/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go @@ -11,9 +11,11 @@ import ( "github.com/iotaledger/hive.go/runtime/module" "github.com/iotaledger/hive.go/runtime/options" "github.com/iotaledger/hive.go/runtime/syncutils" + "github.com/iotaledger/iota-core/pkg/model" "github.com/iotaledger/iota-core/pkg/protocol/engine" "github.com/iotaledger/iota-core/pkg/protocol/engine/blocks" "github.com/iotaledger/iota-core/pkg/protocol/engine/congestioncontrol/scheduler" + "github.com/iotaledger/iota-core/pkg/protocol/sybilprotection/seatmanager" iotago "github.com/iotaledger/iota.go/v4" "github.com/iotaledger/iota.go/v4/api" ) @@ -31,27 +33,18 @@ type Scheduler struct { apiProvider api.Provider - buffer *BufferQueue + seatManager seatmanager.SeatManager + + basicBuffer *BufferQueue validatorBuffer map[iotago.AccountID]*ValidatorQueue bufferMutex syncutils.RWMutex deficits *shrinkingmap.ShrinkingMap[iotago.AccountID, Deficit] - tokenBucket float64 - lastScheduleTime time.Time - shutdownSignal chan struct{} - blockChan chan *blocks.Block - blockCache *blocks.Blocks - validationBlocksPerSlot SubSlotIndex // TODO: make this a protocol parameter - - blacklist map[iotago.AccountID]struct{} - - validationBlocks map[iotago.SlotIndex]map[SubSlotIndex]*blocks.Block - errorHandler func(error) module.Module @@ -61,11 +54,27 @@ func NewProvider(opts ...options.Option[Scheduler]) module.Provider[*engine.Engi return module.Provide(func(e *engine.Engine) scheduler.Scheduler { s := New(e, opts...) s.errorHandler = e.ErrorHandler("scheduler") - s.buffer = NewBufferQueue(int(s.apiProvider.CurrentAPI().ProtocolParameters().CongestionControlParameters().MaxBufferSize)) + s.basicBuffer = NewBufferQueue(int(s.apiProvider.CurrentAPI().ProtocolParameters().CongestionControlParameters().MaxBufferSize)) e.HookConstructed(func() { + s.latestCommittedSlot = func() iotago.SlotIndex { + return e.Storage.Settings().LatestCommitment().Index() + } s.blockCache = e.BlockCache e.Events.Scheduler.LinkTo(s.events) + e.SybilProtection.HookInitialized(func() { + s.seatManager = e.SybilProtection.SeatManager() + }) + e.Events.Notarization.LatestCommitmentUpdated.Hook(func(commitment *model.Commitment) { + // when the last slot of an epoch is committed, remove the queues of validators that are no longer in the committee. + if e.CurrentAPI().TimeProvider().SlotsBeforeNextEpoch(commitment.Index()) == 0 { + for accountID, validatorQueue := range s.validatorBuffer { + if !s.seatManager.Committee(commitment.Index() + 1).HasAccount(accountID) { + s.removeValidator(validatorQueue) + } + } + } + }) e.Ledger.HookInitialized(func() { // quantum retrieve function gets the account's Mana and returns the quantum for that account s.quantumFunc = func(accountID iotago.AccountID, manaSlot iotago.SlotIndex) (Deficit, error) { @@ -81,9 +90,6 @@ func NewProvider(opts ...options.Option[Scheduler]) module.Provider[*engine.Engi return Deficit(mana / minMana), nil } - s.latestCommittedSlot = func() iotago.SlotIndex { - return e.Storage.Settings().LatestCommitment().Index() - } }) s.TriggerConstructed() e.Events.Booker.BlockBooked.Hook(func(block *blocks.Block) { @@ -114,26 +120,13 @@ func NewProvider(opts ...options.Option[Scheduler]) module.Provider[*engine.Engi func New(apiProvider api.Provider, opts ...options.Option[Scheduler]) *Scheduler { return options.Apply( &Scheduler{ - events: scheduler.NewEvents(), - lastScheduleTime: time.Now(), - deficits: shrinkingmap.New[iotago.AccountID, Deficit](), - apiProvider: apiProvider, + events: scheduler.NewEvents(), + deficits: shrinkingmap.New[iotago.AccountID, Deficit](), + apiProvider: apiProvider, }, opts, ) } -func (s *Scheduler) subSlotIndex(block *blocks.Block) SubSlotIndex { - apiForBlock, err := s.apiProvider.APIForVersion(block.ProtocolBlock().ProtocolVersion) - if err != nil { - s.errorHandler(ierrors.Wrapf(err, "failed to get API for version %d", block.ProtocolBlock().ProtocolVersion)) - } - blockSlot := apiForBlock.TimeProvider().SlotFromTime(block.IssuingTime()) - apiForBlock.TimeProvider().SlotStartTime(blockSlot) - timeSinceSlotStart := block.IssuingTime().Sub(apiForBlock.TimeProvider().SlotStartTime(blockSlot)) - - return SubSlotIndex(timeSinceSlotStart.Seconds() * float64(s.validationBlocksPerSlot) / float64(apiForBlock.TimeProvider().SlotDurationSeconds())) -} - func (s *Scheduler) Shutdown() { close(s.shutdownSignal) s.TriggerStopped() @@ -142,8 +135,7 @@ func (s *Scheduler) Shutdown() { // Start starts the scheduler. func (s *Scheduler) Start() { s.shutdownSignal = make(chan struct{}, 1) - s.blockChan = make(chan *blocks.Block, 1) - go s.mainLoop() + go s.basicBlockLoop() s.TriggerInitialized() } @@ -158,7 +150,7 @@ func (s *Scheduler) IssuerQueueBlockCount(issuerID iotago.AccountID) int { s.bufferMutex.RLock() defer s.bufferMutex.RUnlock() - return s.buffer.IssuerQueue(issuerID).Size() + return s.basicBuffer.IssuerQueue(issuerID).Size() } // IssuerQueueWork returns the queue size of the given issuer in work units. @@ -166,7 +158,7 @@ func (s *Scheduler) IssuerQueueWork(issuerID iotago.AccountID) iotago.WorkScore s.bufferMutex.RLock() defer s.bufferMutex.RUnlock() - return s.buffer.IssuerQueue(issuerID).Work() + return s.basicBuffer.IssuerQueue(issuerID).Work() } // BufferSize returns the current buffer size of the Scheduler as block count. @@ -174,7 +166,7 @@ func (s *Scheduler) BufferSize() int { s.bufferMutex.RLock() defer s.bufferMutex.RUnlock() - return s.buffer.Size() + return s.basicBuffer.Size() } // MaxBufferSize returns the max buffer size of the Scheduler as block count. @@ -187,7 +179,7 @@ func (s *Scheduler) ReadyBlocksCount() int { s.bufferMutex.RLock() defer s.bufferMutex.RUnlock() - return s.buffer.ReadyBlocksCount() + return s.basicBuffer.ReadyBlocksCount() } func (s *Scheduler) IsBlockIssuerReady(accountID iotago.AccountID, blocks ...*blocks.Block) bool { @@ -195,7 +187,7 @@ func (s *Scheduler) IsBlockIssuerReady(accountID iotago.AccountID, blocks ...*bl defer s.bufferMutex.RUnlock() // if the buffer is completely empty, any issuer can issue a block. - if s.buffer.Size() == 0 { + if s.basicBuffer.Size() == 0 { return true } work := iotago.WorkScore(0) @@ -212,16 +204,15 @@ func (s *Scheduler) IsBlockIssuerReady(accountID iotago.AccountID, blocks ...*bl return false } - return deficit >= s.deficitFromWork(work+s.buffer.IssuerQueue(accountID).Work()) + return deficit >= s.deficitFromWork(work+s.basicBuffer.IssuerQueue(accountID).Work()) } func (s *Scheduler) AddBlock(block *blocks.Block) { - if _, isBasic := block.BasicBlock(); isBasic { - s.enqueueBasicBlock(block) - } else if _, isValidation := block.ValidationBlock(); isValidation { + // TODO: can we remove the check for committee membership here? It should be checked before this in the accounts filter perhaps. + if _, isValidation := block.ValidationBlock(); isValidation && s.seatManager.Committee(block.ID().Index()).HasAccount(block.ProtocolBlock().IssuerID) { s.enqueueValidationBlock(block) - } else { - panic("invalid block type") + } else if _, isBasic := block.BasicBlock(); isBasic { + s.enqueueBasicBlock(block) } } @@ -232,7 +223,7 @@ func (s *Scheduler) enqueueBasicBlock(block *blocks.Block) { slotIndex := s.latestCommittedSlot() issuerID := block.ProtocolBlock().IssuerID - issuerQueue, err := s.buffer.GetIssuerQueue(issuerID) + issuerQueue, err := s.basicBuffer.GetIssuerQueue(issuerID) if err != nil { // this should only ever happen if the issuer has been removed due to insufficient Mana. // if Mana is now sufficient again, we can add the issuer again. @@ -244,7 +235,7 @@ func (s *Scheduler) enqueueBasicBlock(block *blocks.Block) { issuerQueue = s.createIssuer(issuerID) } - droppedBlocks, err := s.buffer.Submit(block, issuerQueue, func(issuerID iotago.AccountID) Deficit { + droppedBlocks, err := s.basicBuffer.Submit(block, issuerQueue, func(issuerID iotago.AccountID) Deficit { quantum, quantumErr := s.quantumFunc(issuerID, slotIndex) if quantumErr != nil { s.errorHandler(ierrors.Wrapf(quantumErr, "failed to retrieve deficit for issuerID %d in slot %d when submitting a block", issuerID, slotIndex)) @@ -272,40 +263,22 @@ func (s *Scheduler) enqueueValidationBlock(block *blocks.Block) { s.bufferMutex.Lock() defer s.bufferMutex.Unlock() - if _, exists := s.blacklist[block.ProtocolBlock().IssuerID]; exists { - // this validator is blacklisted, so skip this block - return - } - - // TODO: double check that the validator is in the committee for this epoch? validatorQueue, exists := s.validatorBuffer[block.ProtocolBlock().IssuerID] if !exists { - validatorQueue = NewValidatorQueue(block.ProtocolBlock().IssuerID) - s.validatorBuffer[block.ProtocolBlock().IssuerID] = validatorQueue + validatorQueue = s.addValidator(block.ProtocolBlock().IssuerID) } validatorQueue.Submit(block) - if slotValidationBlocks := s.validationBlocks[block.ID().Index()]; slotValidationBlocks != nil { - subSlot := s.subSlotIndex(block) - if _, has := slotValidationBlocks[subSlot]; has { - // the validator has already issued a block for this subslot, so we need to blacklist them. - s.blacklist[block.ProtocolBlock().IssuerID] = struct{}{} - // TODO: retrieve the block that has already been issued for the proof of equivocation - // continue with enqueueing this block as we want to enqueue the first two blocks for the validator for the subslot - } - slotValidationBlocks[subSlot] = block - } - if block.SetEnqueued() { s.events.BlockEnqueued.Trigger(block) } - if s.tryReadyValidationBlock(block) { - s.scheduleValidationBlock(block) - } + s.tryReadyValidationBlock(block) + + //s.scheduleValidationBlock(block) } -func (s *Scheduler) mainLoop() { +func (s *Scheduler) basicBlockLoop() { var blockToSchedule *blocks.Block loop: for { @@ -314,29 +287,54 @@ loop: case <-s.shutdownSignal: break loop // when a block is pushed by the buffer - case blockToSchedule = <-s.blockChan: + case blockToSchedule = <-s.basicBuffer.blockChan: currentAPI := s.apiProvider.CurrentAPI() rate := currentAPI.ProtocolParameters().CongestionControlParameters().SchedulerRate - tokensRequired := float64(blockToSchedule.WorkScore()) - (s.tokenBucket + float64(rate)*time.Since(s.lastScheduleTime).Seconds()) - if tokensRequired > 0 { - // wait until sufficient tokens in token bucket - timer := time.NewTimer(time.Duration(tokensRequired/float64(rate)) * time.Second) + if waitTime := s.basicBuffer.waitTime(float64(rate), blockToSchedule); waitTime > 0 { + timer := time.NewTimer(waitTime) <-timer.C } - s.tokenBucket = lo.Min( - float64(currentAPI.MaxBlockWork()), - s.tokenBucket+float64(rate)*time.Since(s.lastScheduleTime).Seconds(), - ) - s.lastScheduleTime = time.Now() - s.scheduleBlock(blockToSchedule) + s.basicBuffer.updateTokenBucket(float64(rate), float64(currentAPI.MaxBlockWork())) + + s.scheduleBasicBlock(blockToSchedule) } } } -func (s *Scheduler) scheduleBlock(block *blocks.Block) { +func (s *Scheduler) ValidatorLoop(validatorQueue *ValidatorQueue) { + var blockToSchedule *blocks.Block +loop: + for { + select { + // on close, exit the loop + case <-validatorQueue.shutdownSignal: + s.bufferMutex.Lock() + defer s.bufferMutex.Unlock() + + delete(s.validatorBuffer, validatorQueue.accountID) + + break loop + // when a block is pushed by this validator queue. + case blockToSchedule = <-validatorQueue.blockChan: + currentAPI := s.apiProvider.CurrentAPI() + validationBlocksPerSlot := float64(currentAPI.ProtocolParameters().ValidationBlocksPerSlot()) + rate := validationBlocksPerSlot / float64(currentAPI.TimeProvider().SlotDurationSeconds()) + if waitTime := validatorQueue.waitTime(rate); waitTime > 0 { + timer := time.NewTimer(waitTime) + <-timer.C + } + // allow a maximum burst of validationBlocksPerSlot by setting this as max token bucket size. + validatorQueue.updateTokenBucket(rate, validationBlocksPerSlot) + + s.scheduleValidationBlock(blockToSchedule) + } + } +} + +func (s *Scheduler) scheduleBasicBlock(block *blocks.Block) { if block.SetScheduled() { // deduct tokens from the token bucket according to the scheduled block's work. - s.tokenBucket -= float64(block.WorkScore()) + s.basicBuffer.deductTokens(float64(block.WorkScore())) // check for another block ready to schedule s.updateChildrenWithLocking(block) @@ -348,8 +346,14 @@ func (s *Scheduler) scheduleBlock(block *blocks.Block) { func (s *Scheduler) scheduleValidationBlock(block *blocks.Block) { if block.SetScheduled() { + // deduct 1 token from the token bucket of this validator's queue. + s.validatorBuffer[block.ProtocolBlock().IssuerID].deductTokens(1) + + // check for another block ready to schedule + s.updateChildrenWithLocking(block) + s.selectBlockToScheduleWithLocking() + s.events.BlockScheduled.Trigger(block) - s.updateChildrenWithoutLocking(block) } } @@ -360,10 +364,10 @@ func (s *Scheduler) selectBlockToScheduleWithLocking() { slotIndex := s.latestCommittedSlot() // already a block selected to be scheduled. - if len(s.blockChan) > 0 { + if len(s.basicBuffer.blockChan) > 0 { return } - start := s.buffer.Current() + start := s.basicBuffer.Current() // no blocks submitted if start == nil { return @@ -384,9 +388,9 @@ func (s *Scheduler) selectBlockToScheduleWithLocking() { s.errorHandler(ierrors.Wrapf(err, "failed to increment deficit for issuerID %s in slot %d", issuerID, slotIndex)) s.removeIssuer(issuerID, err) - q = s.buffer.Current() + q = s.basicBuffer.Current() } else { - q = s.buffer.Next() + q = s.basicBuffer.Next() } if q == nil { return @@ -397,7 +401,7 @@ func (s *Scheduler) selectBlockToScheduleWithLocking() { } } // increment the deficit for all issuers before schedulingIssuer one more time - for q := start; q != schedulingIssuer; q = s.buffer.Next() { + for q := start; q != schedulingIssuer; q = s.basicBuffer.Next() { issuerID := q.IssuerID() if err := s.incrementDeficit(issuerID, 1, slotIndex); err != nil { s.errorHandler(ierrors.Wrapf(err, "failed to increment deficit for issuerID %s in slot %d", issuerID, slotIndex)) @@ -408,7 +412,7 @@ func (s *Scheduler) selectBlockToScheduleWithLocking() { } // remove the block from the buffer and adjust issuer's deficit - block := s.buffer.PopFront() + block := s.basicBuffer.PopFront() issuerID := block.ProtocolBlock().IssuerID err := s.updateDeficit(issuerID, -s.deficitFromWork(block.WorkScore())) @@ -419,7 +423,7 @@ func (s *Scheduler) selectBlockToScheduleWithLocking() { return } - s.blockChan <- block + s.basicBuffer.blockChan <- block } func (s *Scheduler) selectIssuer(start *IssuerQueue, slotIndex iotago.SlotIndex) (Deficit, *IssuerQueue) { @@ -438,7 +442,7 @@ func (s *Scheduler) selectIssuer(start *IssuerQueue, slotIndex iotago.SlotIndex) s.events.BlockSkipped.Trigger(block) } - s.buffer.PopFront() + s.basicBuffer.PopFront() block = q.Front() @@ -485,9 +489,9 @@ func (s *Scheduler) selectIssuer(start *IssuerQueue, slotIndex iotago.SlotIndex) } if issuerRemoved { - q = s.buffer.Current() + q = s.basicBuffer.Current() } else { - q = s.buffer.Next() + q = s.basicBuffer.Next() } if q == start || q == nil { break @@ -498,7 +502,7 @@ func (s *Scheduler) selectIssuer(start *IssuerQueue, slotIndex iotago.SlotIndex) } func (s *Scheduler) removeIssuer(issuerID iotago.AccountID, err error) { - q := s.buffer.IssuerQueue(issuerID) + q := s.basicBuffer.IssuerQueue(issuerID) q.submitted.ForEach(func(id iotago.BlockID, block *blocks.Block) bool { block.SetDropped() s.events.BlockDropped.Trigger(block, err) @@ -514,11 +518,11 @@ func (s *Scheduler) removeIssuer(issuerID iotago.AccountID, err error) { s.deficits.Delete(issuerID) - s.buffer.RemoveIssuer(issuerID) + s.basicBuffer.RemoveIssuer(issuerID) } func (s *Scheduler) createIssuer(accountID iotago.AccountID) *IssuerQueue { - issuerQueue := s.buffer.CreateIssuerQueue(accountID) + issuerQueue := s.basicBuffer.CreateIssuerQueue(accountID) s.deficits.Set(accountID, 0) return issuerQueue @@ -598,22 +602,20 @@ func (s *Scheduler) tryReady(block *blocks.Block) { } // tryReadyValidator tries to set the given validation block as ready. -func (s *Scheduler) tryReadyValidationBlock(block *blocks.Block) bool { +func (s *Scheduler) tryReadyValidationBlock(block *blocks.Block) { if s.isReady(block) { s.readyValidationBlock(block) - - return true } - - return false } func (s *Scheduler) ready(block *blocks.Block) { - s.buffer.Ready(block) + s.basicBuffer.Ready(block) } func (s *Scheduler) readyValidationBlock(block *blocks.Block) { - s.validatorBuffer[block.ProtocolBlock().IssuerID].Unsubmit(block) + if validatorQueue, exists := s.validatorBuffer[block.ProtocolBlock().IssuerID]; exists { + validatorQueue.Ready(block) + } } // updateChildrenWithLocking locks the buffer mutex and iterates over the direct children of the given blockID and @@ -633,9 +635,7 @@ func (s *Scheduler) updateChildrenWithoutLocking(block *blocks.Block) { if _, isBasic := childBlock.BasicBlock(); isBasic { s.tryReady(childBlock) } else if _, isValidation := childBlock.ValidationBlock(); isValidation { - if s.tryReadyValidationBlock(childBlock) { - s.scheduleValidationBlock(childBlock) - } + s.tryReadyValidationBlock(childBlock) } else { panic("invalid block type") } @@ -652,3 +652,14 @@ func (s *Scheduler) deficitFromWork(work iotago.WorkScore) Deficit { deficitScaleFactor := s.maxDeficit() / Deficit(s.apiProvider.CurrentAPI().MaxBlockWork()) return Deficit(work) * deficitScaleFactor } + +func (s *Scheduler) addValidator(accountID iotago.AccountID) *ValidatorQueue { + s.validatorBuffer[accountID] = NewValidatorQueue(accountID) + go s.ValidatorLoop(s.validatorBuffer[accountID]) + + return s.validatorBuffer[accountID] +} + +func (s *Scheduler) removeValidator(validatorQueue *ValidatorQueue) { + validatorQueue.shutdownSignal <- struct{}{} +} diff --git a/pkg/protocol/engine/congestioncontrol/scheduler/drr/validatorqueue.go b/pkg/protocol/engine/congestioncontrol/scheduler/drr/validatorqueue.go index cbf3d9b49..3a4d56d1c 100644 --- a/pkg/protocol/engine/congestioncontrol/scheduler/drr/validatorqueue.go +++ b/pkg/protocol/engine/congestioncontrol/scheduler/drr/validatorqueue.go @@ -1,9 +1,14 @@ package drr import ( + "container/heap" "fmt" + "time" + "github.com/iotaledger/hive.go/ds/generalheap" "github.com/iotaledger/hive.go/ds/shrinkingmap" + "github.com/iotaledger/hive.go/lo" + "github.com/iotaledger/hive.go/runtime/timed" "github.com/iotaledger/iota-core/pkg/protocol/engine/blocks" iotago "github.com/iotaledger/iota.go/v4" "go.uber.org/atomic" @@ -12,13 +17,22 @@ import ( type ValidatorQueue struct { accountID iotago.AccountID submitted *shrinkingmap.ShrinkingMap[iotago.BlockID, *blocks.Block] + inbox generalheap.Heap[timed.HeapKey, *blocks.Block] size atomic.Int64 + + tokenBucket float64 + lastScheduleTime time.Time + + blockChan chan *blocks.Block + shutdownSignal chan struct{} } func NewValidatorQueue(accountID iotago.AccountID) *ValidatorQueue { return &ValidatorQueue{ - accountID: accountID, - submitted: shrinkingmap.New[iotago.BlockID, *blocks.Block](), + accountID: accountID, + submitted: shrinkingmap.New[iotago.BlockID, *blocks.Block](), + blockChan: make(chan *blocks.Block, 1), + shutdownSignal: make(chan struct{}), } } @@ -59,3 +73,95 @@ func (q *ValidatorQueue) Unsubmit(block *blocks.Block) bool { return true } + +func (q *ValidatorQueue) Ready(block *blocks.Block) bool { + if _, submitted := q.submitted.Get(block.ID()); !submitted { + return false + } + + q.submitted.Delete(block.ID()) + heap.Push(&q.inbox, &generalheap.HeapElement[timed.HeapKey, *blocks.Block]{Value: block, Key: timed.HeapKey(block.IssuingTime())}) + + return true +} + +// PopFront removes the first ready block from the queue. +func (q *ValidatorQueue) PopFront() *blocks.Block { + if q.inbox.Len() == 0 { + return nil + } + + heapElement, isHeapElement := heap.Pop(&q.inbox).(*generalheap.HeapElement[timed.HeapKey, *blocks.Block]) + if !isHeapElement { + return nil + } + blk := heapElement.Value + q.size.Dec() + + return blk +} + +func (q *ValidatorQueue) RemoveTail() *blocks.Block { + var oldestSubmittedBlock *blocks.Block + q.submitted.ForEach(func(_ iotago.BlockID, block *blocks.Block) bool { + if oldestSubmittedBlock == nil || oldestSubmittedBlock.IssuingTime().After(block.IssuingTime()) { + oldestSubmittedBlock = block + } + + return true + }) + + tail := q.tail() + // if heap tail does not exist or tail is newer than oldest submitted block, unsubmit oldest block + if oldestSubmittedBlock != nil && (tail < 0 || q.inbox[tail].Key.CompareTo(timed.HeapKey(oldestSubmittedBlock.IssuingTime())) > 0) { + q.Unsubmit(oldestSubmittedBlock) + + return oldestSubmittedBlock + } else if tail < 0 { + // should never happen that the oldest submitted block does not exist and the tail does not exist. + return nil + } + + // if the tail exists and is older than the oldest submitted block, drop it + heapElement, isHeapElement := heap.Remove(&q.inbox, tail).(*generalheap.HeapElement[timed.HeapKey, *blocks.Block]) + if !isHeapElement { + return nil + } + blk := heapElement.Value + q.size.Dec() + + return blk +} + +func (q *ValidatorQueue) tail() int { + h := q.inbox + if h.Len() <= 0 { + return -1 + } + tail := 0 + for i := range h { + if !h.Less(i, tail) { // less means older issue time + tail = i + } + } + + return tail +} + +func (q *ValidatorQueue) waitTime(rate float64) time.Duration { + tokensRequired := 1 - (q.tokenBucket + rate*time.Since(q.lastScheduleTime).Seconds()) + + return lo.Max(0, time.Duration(tokensRequired/rate)) +} + +func (q *ValidatorQueue) updateTokenBucket(rate float64, tokenBucketSize float64) { + q.tokenBucket = lo.Min( + tokenBucketSize, + q.tokenBucket+rate*time.Since(q.lastScheduleTime).Seconds(), + ) + q.lastScheduleTime = time.Now() +} + +func (q *ValidatorQueue) deductTokens(tokens float64) { + q.tokenBucket -= tokens +} diff --git a/pkg/protocol/engine/congestioncontrol/scheduler/scheduler.go b/pkg/protocol/engine/congestioncontrol/scheduler/scheduler.go index 197fe87ce..28e81f71d 100644 --- a/pkg/protocol/engine/congestioncontrol/scheduler/scheduler.go +++ b/pkg/protocol/engine/congestioncontrol/scheduler/scheduler.go @@ -7,7 +7,7 @@ import ( ) type Scheduler interface { - // AddBasicBlock adds a basic block to the scheduling buffer. + // AddBlock adds a block to the scheduling buffer. AddBlock(*blocks.Block) // IsBlockIssuerReady returns true if the block issuer is ready to issuer a block, i.e., if the block issuer were to add a block to the scheduler, would it be scheduled. IsBlockIssuerReady(iotago.AccountID, ...*blocks.Block) bool diff --git a/pkg/testsuite/testsuite.go b/pkg/testsuite/testsuite.go index 49f2029d1..922936c79 100644 --- a/pkg/testsuite/testsuite.go +++ b/pkg/testsuite/testsuite.go @@ -136,7 +136,7 @@ func NewTestSuite(testingT *testing.T, opts ...options.Option[TestSuite]) *TestS t.optsMinMana, t.optsMaxBufferSize, ), - iotago.WithStakingOptions(1), + iotago.WithStakingOptions(1, 1, 1), ), ) From 084c422079567d95168ee86458e658fe3058badc Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 6 Sep 2023 11:53:00 +0100 Subject: [PATCH 03/24] complete simplified scheduler --- .../scheduler/drr/scheduler.go | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go b/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go index 925f0eda5..c14bbc0fe 100644 --- a/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go +++ b/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go @@ -120,9 +120,10 @@ func NewProvider(opts ...options.Option[Scheduler]) module.Provider[*engine.Engi func New(apiProvider api.Provider, opts ...options.Option[Scheduler]) *Scheduler { return options.Apply( &Scheduler{ - events: scheduler.NewEvents(), - deficits: shrinkingmap.New[iotago.AccountID, Deficit](), - apiProvider: apiProvider, + events: scheduler.NewEvents(), + deficits: shrinkingmap.New[iotago.AccountID, Deficit](), + apiProvider: apiProvider, + validatorBuffer: make(map[iotago.AccountID]*ValidatorQueue), }, opts, ) } @@ -271,10 +272,9 @@ func (s *Scheduler) enqueueValidationBlock(block *blocks.Block) { if block.SetEnqueued() { s.events.BlockEnqueued.Trigger(block) + s.tryReadyValidationBlock(block) } - s.tryReadyValidationBlock(block) - //s.scheduleValidationBlock(block) } @@ -361,6 +361,25 @@ func (s *Scheduler) selectBlockToScheduleWithLocking() { s.bufferMutex.Lock() defer s.bufferMutex.Unlock() + for _, validatorQueue := range s.validatorBuffer { + s.selectValidationBlockWithoutLocking(validatorQueue) + } + s.selectBasicBlockWithoutLocking() + +} + +func (s *Scheduler) selectValidationBlockWithoutLocking(validatorQueue *ValidatorQueue) { + // already a block selected to be scheduled. + if len(validatorQueue.blockChan) > 0 { + return + } + + if blockToSchedule := validatorQueue.PopFront(); blockToSchedule != nil { + validatorQueue.blockChan <- blockToSchedule + } +} + +func (s *Scheduler) selectBasicBlockWithoutLocking() { slotIndex := s.latestCommittedSlot() // already a block selected to be scheduled. From 9f43ad2a5b6ebbb2a475a84db32f0559e638f13e Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 6 Sep 2023 12:00:14 +0100 Subject: [PATCH 04/24] good doggy --- .../engine/congestioncontrol/scheduler/drr/drrbuffer.go | 1 - .../engine/congestioncontrol/scheduler/drr/validatorqueue.go | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/protocol/engine/congestioncontrol/scheduler/drr/drrbuffer.go b/pkg/protocol/engine/congestioncontrol/scheduler/drr/drrbuffer.go index 88672c58b..6680a1568 100644 --- a/pkg/protocol/engine/congestioncontrol/scheduler/drr/drrbuffer.go +++ b/pkg/protocol/engine/congestioncontrol/scheduler/drr/drrbuffer.go @@ -32,7 +32,6 @@ type BufferQueue struct { lastScheduleTime time.Time blockChan chan *blocks.Block - timer *time.Timer } // NewBufferQueue returns a new BufferQueue. diff --git a/pkg/protocol/engine/congestioncontrol/scheduler/drr/validatorqueue.go b/pkg/protocol/engine/congestioncontrol/scheduler/drr/validatorqueue.go index 3a4d56d1c..6f8b757f0 100644 --- a/pkg/protocol/engine/congestioncontrol/scheduler/drr/validatorqueue.go +++ b/pkg/protocol/engine/congestioncontrol/scheduler/drr/validatorqueue.go @@ -5,13 +5,15 @@ import ( "fmt" "time" + "go.uber.org/atomic" + "github.com/iotaledger/hive.go/ds/generalheap" "github.com/iotaledger/hive.go/ds/shrinkingmap" "github.com/iotaledger/hive.go/lo" "github.com/iotaledger/hive.go/runtime/timed" "github.com/iotaledger/iota-core/pkg/protocol/engine/blocks" + iotago "github.com/iotaledger/iota.go/v4" - "go.uber.org/atomic" ) type ValidatorQueue struct { From 87d6cbff3843c1e07e2d9c1b1d3aa704b6ee77a0 Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 6 Sep 2023 16:57:24 +0100 Subject: [PATCH 05/24] shutdown correctly --- .../engine/congestioncontrol/scheduler/drr/scheduler.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go b/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go index c14bbc0fe..cbbbd1206 100644 --- a/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go +++ b/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go @@ -129,6 +129,9 @@ func New(apiProvider api.Provider, opts ...options.Option[Scheduler]) *Scheduler } func (s *Scheduler) Shutdown() { + for _, validatorQueue := range s.validatorBuffer { + s.removeValidator(validatorQueue) + } close(s.shutdownSignal) s.TriggerStopped() } From 9a7c072abd1894e7adb69951625b05b9d506b4e1 Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 6 Sep 2023 18:17:33 +0100 Subject: [PATCH 06/24] race debugging --- .../engine/congestioncontrol/scheduler/drr/scheduler.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go b/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go index cbbbd1206..86809b7e8 100644 --- a/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go +++ b/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go @@ -329,7 +329,7 @@ loop: // allow a maximum burst of validationBlocksPerSlot by setting this as max token bucket size. validatorQueue.updateTokenBucket(rate, validationBlocksPerSlot) - s.scheduleValidationBlock(blockToSchedule) + s.scheduleValidationBlock(blockToSchedule, validatorQueue) } } } @@ -347,10 +347,10 @@ func (s *Scheduler) scheduleBasicBlock(block *blocks.Block) { } } -func (s *Scheduler) scheduleValidationBlock(block *blocks.Block) { +func (s *Scheduler) scheduleValidationBlock(block *blocks.Block, validatorQueue *ValidatorQueue) { if block.SetScheduled() { // deduct 1 token from the token bucket of this validator's queue. - s.validatorBuffer[block.ProtocolBlock().IssuerID].deductTokens(1) + validatorQueue.deductTokens(1) // check for another block ready to schedule s.updateChildrenWithLocking(block) From e31a4f51e14c5091bd9a97ce7007db540f84e922 Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 6 Sep 2023 18:39:13 +0100 Subject: [PATCH 07/24] race debugging --- .../congestioncontrol/scheduler/drr/scheduler.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go b/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go index 86809b7e8..0d72be983 100644 --- a/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go +++ b/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go @@ -68,9 +68,13 @@ func NewProvider(opts ...options.Option[Scheduler]) module.Provider[*engine.Engi e.Events.Notarization.LatestCommitmentUpdated.Hook(func(commitment *model.Commitment) { // when the last slot of an epoch is committed, remove the queues of validators that are no longer in the committee. if e.CurrentAPI().TimeProvider().SlotsBeforeNextEpoch(commitment.Index()) == 0 { + s.bufferMutex.Lock() + defer s.bufferMutex.Unlock() + for accountID, validatorQueue := range s.validatorBuffer { if !s.seatManager.Committee(commitment.Index() + 1).HasAccount(accountID) { - s.removeValidator(validatorQueue) + s.shutdownValidatorQueue(validatorQueue) + delete(s.validatorBuffer, accountID) } } } @@ -130,7 +134,7 @@ func New(apiProvider api.Provider, opts ...options.Option[Scheduler]) *Scheduler func (s *Scheduler) Shutdown() { for _, validatorQueue := range s.validatorBuffer { - s.removeValidator(validatorQueue) + s.shutdownValidatorQueue(validatorQueue) } close(s.shutdownSignal) s.TriggerStopped() @@ -311,11 +315,6 @@ loop: select { // on close, exit the loop case <-validatorQueue.shutdownSignal: - s.bufferMutex.Lock() - defer s.bufferMutex.Unlock() - - delete(s.validatorBuffer, validatorQueue.accountID) - break loop // when a block is pushed by this validator queue. case blockToSchedule = <-validatorQueue.blockChan: @@ -682,6 +681,6 @@ func (s *Scheduler) addValidator(accountID iotago.AccountID) *ValidatorQueue { return s.validatorBuffer[accountID] } -func (s *Scheduler) removeValidator(validatorQueue *ValidatorQueue) { +func (s *Scheduler) shutdownValidatorQueue(validatorQueue *ValidatorQueue) { validatorQueue.shutdownSignal <- struct{}{} } From 8729aedbf0d46b33ec6b5cc71fda9a28a7604a78 Mon Sep 17 00:00:00 2001 From: Andrew Date: Thu, 7 Sep 2023 10:04:36 +0100 Subject: [PATCH 08/24] charge maxBlockWork for over-issued validation blocks --- .../engine/accounts/accountsledger/manager.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pkg/protocol/engine/accounts/accountsledger/manager.go b/pkg/protocol/engine/accounts/accountsledger/manager.go index 35114d498..ae9268584 100644 --- a/pkg/protocol/engine/accounts/accountsledger/manager.go +++ b/pkg/protocol/engine/accounts/accountsledger/manager.go @@ -412,8 +412,8 @@ func (m *Manager) preserveDestroyedAccountData(accountID iotago.AccountID) (acco func (m *Manager) computeBlockBurnsForSlot(slotIndex iotago.SlotIndex, rmc iotago.Mana) (burns map[iotago.AccountID]iotago.Mana, err error) { burns = make(map[iotago.AccountID]iotago.Mana) + validationBlockCount := make(map[iotago.AccountID]int) if set, exists := m.blockBurns.Get(slotIndex); exists { - // Get RMC for this slot for it := set.Iterator(); it.HasNext(); { blockID := it.Next() block, blockLoaded := m.block(blockID) @@ -422,6 +422,15 @@ func (m *Manager) computeBlockBurnsForSlot(slotIndex iotago.SlotIndex, rmc iotag } if _, isBasicBlock := block.BasicBlock(); isBasicBlock { burns[block.ProtocolBlock().IssuerID] += iotago.Mana(block.WorkScore()) * rmc + } else if _, isValidationBlock := block.ValidationBlock(); isValidationBlock { + validationBlockCount[block.ProtocolBlock().IssuerID]++ + } + } + validationBlocksPerSlot := int(m.apiProvider.APIForSlot(slotIndex).ProtocolParameters().ValidationBlocksPerSlot()) + for accountID, count := range validationBlockCount { + if count > validationBlocksPerSlot { + // penalize over-issuance by charging for a maximum work score block for each validation block over the quota + burns[accountID] += iotago.Mana(count-validationBlocksPerSlot) * iotago.Mana(m.apiProvider.CurrentAPI().MaxBlockWork()) * rmc } } } From c7453b64155f26ee7497270dce48b8fb17e3a9e8 Mon Sep 17 00:00:00 2001 From: Andrew Date: Thu, 7 Sep 2023 10:09:21 +0100 Subject: [PATCH 09/24] go mod tidy --- go.mod | 2 +- go.sum | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 559c7d458..865ec0dd0 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/iotaledger/hive.go/stringify v0.0.0-20230906114834-b50190b9f9c2 github.com/iotaledger/inx-app v1.0.0-rc.3.0.20230829161228-3f4eb50a4d14 github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230829160617-69b96c7c9f9b - github.com/iotaledger/iota.go/v4 v4.0.0-20230905174229-dfb67db9ea20 + github.com/iotaledger/iota.go/v4 v4.0.0-20230907080856-d287a036fa11 github.com/labstack/echo/v4 v4.11.1 github.com/labstack/gommon v0.4.0 github.com/libp2p/go-libp2p v0.30.0 diff --git a/go.sum b/go.sum index 16540da45..330940c7f 100644 --- a/go.sum +++ b/go.sum @@ -129,6 +129,7 @@ github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -307,6 +308,8 @@ github.com/iotaledger/inx-app v1.0.0-rc.3.0.20230829161228-3f4eb50a4d14 h1:BkDuQ github.com/iotaledger/inx-app v1.0.0-rc.3.0.20230829161228-3f4eb50a4d14/go.mod h1:ADBXzdHXTldP0NB2Vf+KbhDxkYciGRjzQVXT6Rdne1g= github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230829160617-69b96c7c9f9b h1:EPB/+iWeSx/WgJlzaXl8yjinxuD8CCOdi2ZPMLeeMVY= github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230829160617-69b96c7c9f9b/go.mod h1:B7gyJP6GshCSlEmY3CxEk5TZdsMs3UNz5U92hkFDdMs= +github.com/iotaledger/iota.go/v4 v4.0.0-20230907080856-d287a036fa11 h1:7e7SkoQ4crUOZkDjSm0iC/nuMZvCSgRiePwyiR9KcA8= +github.com/iotaledger/iota.go/v4 v4.0.0-20230907080856-d287a036fa11/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= github.com/ipfs/boxo v0.10.0 h1:tdDAxq8jrsbRkYoF+5Rcqyeb91hgWe2hp7iLu7ORZLY= github.com/ipfs/boxo v0.10.0/go.mod h1:Fg+BnfxZ0RPzR0nOodzdIq3A7KgoWAOWsEIImrIQdBM= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= From e50578db5a99cc1e3601e966245cfab6ffffb45a Mon Sep 17 00:00:00 2001 From: Andrew Date: Thu, 7 Sep 2023 10:14:31 +0100 Subject: [PATCH 10/24] go mod tidy --- go.mod | 2 +- go.sum | 2 ++ pkg/testsuite/testsuite.go | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 865ec0dd0..ed1600cba 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/iotaledger/hive.go/stringify v0.0.0-20230906114834-b50190b9f9c2 github.com/iotaledger/inx-app v1.0.0-rc.3.0.20230829161228-3f4eb50a4d14 github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230829160617-69b96c7c9f9b - github.com/iotaledger/iota.go/v4 v4.0.0-20230907080856-d287a036fa11 + github.com/iotaledger/iota.go/v4 v4.0.0-20230907091304-e54ea4c46dfd github.com/labstack/echo/v4 v4.11.1 github.com/labstack/gommon v0.4.0 github.com/libp2p/go-libp2p v0.30.0 diff --git a/go.sum b/go.sum index 330940c7f..209c92b93 100644 --- a/go.sum +++ b/go.sum @@ -310,6 +310,8 @@ github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230829160617-69b96c7c9f9b h1:EPB/+i github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230829160617-69b96c7c9f9b/go.mod h1:B7gyJP6GshCSlEmY3CxEk5TZdsMs3UNz5U92hkFDdMs= github.com/iotaledger/iota.go/v4 v4.0.0-20230907080856-d287a036fa11 h1:7e7SkoQ4crUOZkDjSm0iC/nuMZvCSgRiePwyiR9KcA8= github.com/iotaledger/iota.go/v4 v4.0.0-20230907080856-d287a036fa11/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= +github.com/iotaledger/iota.go/v4 v4.0.0-20230907091304-e54ea4c46dfd h1:xwddq0tkXnBv600Pw1vyhF81+bMwwbksoGFsc4yb3jk= +github.com/iotaledger/iota.go/v4 v4.0.0-20230907091304-e54ea4c46dfd/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= github.com/ipfs/boxo v0.10.0 h1:tdDAxq8jrsbRkYoF+5Rcqyeb91hgWe2hp7iLu7ORZLY= github.com/ipfs/boxo v0.10.0/go.mod h1:Fg+BnfxZ0RPzR0nOodzdIq3A7KgoWAOWsEIImrIQdBM= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= diff --git a/pkg/testsuite/testsuite.go b/pkg/testsuite/testsuite.go index fb2c76414..11cb292ee 100644 --- a/pkg/testsuite/testsuite.go +++ b/pkg/testsuite/testsuite.go @@ -136,7 +136,7 @@ func NewTestSuite(testingT *testing.T, opts ...options.Option[TestSuite]) *TestS t.optsMinMana, t.optsMaxBufferSize, ), - iotago.WithStakingOptions(1, 1, 1), + iotago.WithStakingOptions(1, 1), ), ) From 9ae344773e2151d666441551151a5c84ed83105a Mon Sep 17 00:00:00 2001 From: Andrew Date: Thu, 7 Sep 2023 13:35:47 +0100 Subject: [PATCH 11/24] defer penalizing over-issuer validators to issue #338 --- .../engine/accounts/accountsledger/manager.go | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/pkg/protocol/engine/accounts/accountsledger/manager.go b/pkg/protocol/engine/accounts/accountsledger/manager.go index ae9268584..11d26003a 100644 --- a/pkg/protocol/engine/accounts/accountsledger/manager.go +++ b/pkg/protocol/engine/accounts/accountsledger/manager.go @@ -412,7 +412,6 @@ func (m *Manager) preserveDestroyedAccountData(accountID iotago.AccountID) (acco func (m *Manager) computeBlockBurnsForSlot(slotIndex iotago.SlotIndex, rmc iotago.Mana) (burns map[iotago.AccountID]iotago.Mana, err error) { burns = make(map[iotago.AccountID]iotago.Mana) - validationBlockCount := make(map[iotago.AccountID]int) if set, exists := m.blockBurns.Get(slotIndex); exists { for it := set.Iterator(); it.HasNext(); { blockID := it.Next() @@ -422,17 +421,22 @@ func (m *Manager) computeBlockBurnsForSlot(slotIndex iotago.SlotIndex, rmc iotag } if _, isBasicBlock := block.BasicBlock(); isBasicBlock { burns[block.ProtocolBlock().IssuerID] += iotago.Mana(block.WorkScore()) * rmc - } else if _, isValidationBlock := block.ValidationBlock(); isValidationBlock { - validationBlockCount[block.ProtocolBlock().IssuerID]++ - } - } - validationBlocksPerSlot := int(m.apiProvider.APIForSlot(slotIndex).ProtocolParameters().ValidationBlocksPerSlot()) - for accountID, count := range validationBlockCount { - if count > validationBlocksPerSlot { - // penalize over-issuance by charging for a maximum work score block for each validation block over the quota - burns[accountID] += iotago.Mana(count-validationBlocksPerSlot) * iotago.Mana(m.apiProvider.CurrentAPI().MaxBlockWork()) * rmc } } + + // TODO: issue #338 enable this block of code and fix the tests to issue correct rate of validation blocks. + // validationBlockCount := make(map[iotago.AccountID]int) + // else if _, isValidationBlock := block.ValidationBlock(); isValidationBlock { + // validationBlockCount[block.ProtocolBlock().IssuerID]++ + // } + // } + // validationBlocksPerSlot := int(m.apiProvider.APIForSlot(slotIndex).ProtocolParameters().ValidationBlocksPerSlot()) + // for accountID, count := range validationBlockCount { + // if count > validationBlocksPerSlot { + // // penalize over-issuance by charging for a maximum work score block for each validation block over the quota + // burns[accountID] += iotago.Mana(count-validationBlocksPerSlot) * iotago.Mana(m.apiProvider.CurrentAPI().MaxBlockWork()) * rmc + // } + // } } return burns, nil From a9ff82fdfb47d7c3e992c1111821983588987448 Mon Sep 17 00:00:00 2001 From: jkrvivian Date: Wed, 6 Sep 2023 10:03:09 +0800 Subject: [PATCH 12/24] Update to new protocol parameters --- .../attestation/slotattestation/testframework_test.go | 2 +- pkg/testsuite/testsuite.go | 1 + tools/genesis-snapshot/presets/presets.go | 6 +++--- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/protocol/engine/attestation/slotattestation/testframework_test.go b/pkg/protocol/engine/attestation/slotattestation/testframework_test.go index 4362ed8d3..8e756c266 100644 --- a/pkg/protocol/engine/attestation/slotattestation/testframework_test.go +++ b/pkg/protocol/engine/attestation/slotattestation/testframework_test.go @@ -70,7 +70,7 @@ func NewTestFramework(test *testing.T) *TestFramework { testAPI := iotago.V3API( iotago.NewV3ProtocolParameters( iotago.WithNetworkOptions("TestJungle", "tgl"), - iotago.WithSupplyOptions(10000, 0, 0, 0, 0, 0), + iotago.WithSupplyOptions(10000, 0, 0, 0, 0, 0, 0), iotago.WithLivenessOptions(1, 1, 2, 8), ), ) diff --git a/pkg/testsuite/testsuite.go b/pkg/testsuite/testsuite.go index 11cb292ee..9a351fccf 100644 --- a/pkg/testsuite/testsuite.go +++ b/pkg/testsuite/testsuite.go @@ -114,6 +114,7 @@ func NewTestSuite(testingT *testing.T, opts ...options.Option[TestSuite]) *TestS 10, 100, 100, + 100, ), iotago.WithTimeProviderOptions( time.Now().Truncate(10*time.Second).Unix()-t.optsGenesisTimestampOffset, diff --git a/tools/genesis-snapshot/presets/presets.go b/tools/genesis-snapshot/presets/presets.go index 55b143742..f9aa42da1 100644 --- a/tools/genesis-snapshot/presets/presets.go +++ b/tools/genesis-snapshot/presets/presets.go @@ -22,7 +22,7 @@ var Base = []options.Option[snapshotcreator.Options]{ snapshotcreator.WithProtocolParameters( iotago.NewV3ProtocolParameters( iotago.WithNetworkOptions("default", "rms"), - iotago.WithSupplyOptions(10_000_000_000, 100, 1, 10, 100, 100), + iotago.WithSupplyOptions(10_000_000_000, 100, 1, 10, 100, 100, 100), iotago.WithTimeProviderOptions(time.Now().Unix(), 10, 13), iotago.WithLivenessOptions(5, 7, 14, 30), // increase/decrease threshold = fraction * slotDurationInSeconds * schedulerRate @@ -96,7 +96,7 @@ var Docker = []options.Option[snapshotcreator.Options]{ snapshotcreator.WithProtocolParameters( iotago.NewV3ProtocolParameters( iotago.WithNetworkOptions("docker", "rms"), - iotago.WithSupplyOptions(10_000_000_000, 1, 1, 10, 100, 100), + iotago.WithSupplyOptions(10_000_000_000, 1, 1, 10, 100, 100, 100), iotago.WithTimeProviderOptions(time.Now().Unix(), 10, 13), iotago.WithLivenessOptions(5, 7, 14, 30), // increase/decrease threshold = fraction * slotDurationInSeconds * schedulerRate @@ -150,7 +150,7 @@ var Feature = []options.Option[snapshotcreator.Options]{ snapshotcreator.WithProtocolParameters( iotago.NewV3ProtocolParameters( iotago.WithNetworkOptions("feature", "rms"), - iotago.WithSupplyOptions(10_000_000_000, 100, 1, 10, 100, 100), + iotago.WithSupplyOptions(10_000_000_000, 100, 1, 10, 100, 100, 100), iotago.WithTimeProviderOptions(1689848996, 10, 13), iotago.WithLivenessOptions(5, 10, 20, 30), // increase/decrease threshold = fraction * slotDurationInSeconds * schedulerRate From 4bf14175a5e111227fbe80adf18573b97ca33805 Mon Sep 17 00:00:00 2001 From: jkrvivian Date: Fri, 8 Sep 2023 15:25:54 +0800 Subject: [PATCH 13/24] Update iota.go version --- go.mod | 2 +- go.sum | 9 ++------- tools/evil-spammer/go.mod | 2 +- tools/evil-spammer/go.sum | 4 ++-- tools/gendoc/go.mod | 2 +- tools/gendoc/go.sum | 4 ++-- tools/genesis-snapshot/go.mod | 2 +- tools/genesis-snapshot/go.sum | 4 ++-- 8 files changed, 12 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index ed1600cba..579947976 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/iotaledger/hive.go/stringify v0.0.0-20230906114834-b50190b9f9c2 github.com/iotaledger/inx-app v1.0.0-rc.3.0.20230829161228-3f4eb50a4d14 github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230829160617-69b96c7c9f9b - github.com/iotaledger/iota.go/v4 v4.0.0-20230907091304-e54ea4c46dfd + github.com/iotaledger/iota.go/v4 v4.0.0-20230908070236-ae553965e1a3 github.com/labstack/echo/v4 v4.11.1 github.com/labstack/gommon v0.4.0 github.com/libp2p/go-libp2p v0.30.0 diff --git a/go.sum b/go.sum index 209c92b93..38dc69b03 100644 --- a/go.sum +++ b/go.sum @@ -129,7 +129,6 @@ github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -308,10 +307,8 @@ github.com/iotaledger/inx-app v1.0.0-rc.3.0.20230829161228-3f4eb50a4d14 h1:BkDuQ github.com/iotaledger/inx-app v1.0.0-rc.3.0.20230829161228-3f4eb50a4d14/go.mod h1:ADBXzdHXTldP0NB2Vf+KbhDxkYciGRjzQVXT6Rdne1g= github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230829160617-69b96c7c9f9b h1:EPB/+iWeSx/WgJlzaXl8yjinxuD8CCOdi2ZPMLeeMVY= github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230829160617-69b96c7c9f9b/go.mod h1:B7gyJP6GshCSlEmY3CxEk5TZdsMs3UNz5U92hkFDdMs= -github.com/iotaledger/iota.go/v4 v4.0.0-20230907080856-d287a036fa11 h1:7e7SkoQ4crUOZkDjSm0iC/nuMZvCSgRiePwyiR9KcA8= -github.com/iotaledger/iota.go/v4 v4.0.0-20230907080856-d287a036fa11/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= -github.com/iotaledger/iota.go/v4 v4.0.0-20230907091304-e54ea4c46dfd h1:xwddq0tkXnBv600Pw1vyhF81+bMwwbksoGFsc4yb3jk= -github.com/iotaledger/iota.go/v4 v4.0.0-20230907091304-e54ea4c46dfd/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= +github.com/iotaledger/iota.go/v4 v4.0.0-20230908070236-ae553965e1a3 h1:VnS5NOlgtmL9bEXHRKD9oYOzMSKYg8c9LiOUbd/CHN0= +github.com/iotaledger/iota.go/v4 v4.0.0-20230908070236-ae553965e1a3/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= github.com/ipfs/boxo v0.10.0 h1:tdDAxq8jrsbRkYoF+5Rcqyeb91hgWe2hp7iLu7ORZLY= github.com/ipfs/boxo v0.10.0/go.mod h1:Fg+BnfxZ0RPzR0nOodzdIq3A7KgoWAOWsEIImrIQdBM= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= @@ -345,7 +342,6 @@ github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= @@ -466,7 +462,6 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= diff --git a/tools/evil-spammer/go.mod b/tools/evil-spammer/go.mod index 354edeb7a..351eeee10 100644 --- a/tools/evil-spammer/go.mod +++ b/tools/evil-spammer/go.mod @@ -17,7 +17,7 @@ require ( github.com/iotaledger/hive.go/runtime v0.0.0-20230906114834-b50190b9f9c2 github.com/iotaledger/iota-core v0.0.0-00010101000000-000000000000 github.com/iotaledger/iota-core/tools/genesis-snapshot v0.0.0-00010101000000-000000000000 - github.com/iotaledger/iota.go/v4 v4.0.0-20230906064556-75f5be378284 + github.com/iotaledger/iota.go/v4 v4.0.0-20230908070236-ae553965e1a3 github.com/mr-tron/base58 v1.2.0 go.uber.org/atomic v1.11.0 ) diff --git a/tools/evil-spammer/go.sum b/tools/evil-spammer/go.sum index 0f58ae65e..9f3ede384 100644 --- a/tools/evil-spammer/go.sum +++ b/tools/evil-spammer/go.sum @@ -197,8 +197,8 @@ github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20230906114834-b50190b github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20230906114834-b50190b9f9c2/go.mod h1:IJgaaxbgKCsNat18jlJJEAxCY2oVYR3F30B+M4vJ89I= github.com/iotaledger/hive.go/stringify v0.0.0-20230906114834-b50190b9f9c2 h1:exATYMLT/d8fgMuVNO6kMDsFn9DUJEcyCuoBv9sP13g= github.com/iotaledger/hive.go/stringify v0.0.0-20230906114834-b50190b9f9c2/go.mod h1:FTo/UWzNYgnQ082GI9QVM9HFDERqf9rw9RivNpqrnTs= -github.com/iotaledger/iota.go/v4 v4.0.0-20230906064556-75f5be378284 h1:SmPyo08i2/ymMZkmIQX/7EexuWWFl7CKDZjJr47zn8Q= -github.com/iotaledger/iota.go/v4 v4.0.0-20230906064556-75f5be378284/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= +github.com/iotaledger/iota.go/v4 v4.0.0-20230908070236-ae553965e1a3 h1:VnS5NOlgtmL9bEXHRKD9oYOzMSKYg8c9LiOUbd/CHN0= +github.com/iotaledger/iota.go/v4 v4.0.0-20230908070236-ae553965e1a3/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= diff --git a/tools/gendoc/go.mod b/tools/gendoc/go.mod index 5d90d3eb9..4c3d33b64 100644 --- a/tools/gendoc/go.mod +++ b/tools/gendoc/go.mod @@ -72,7 +72,7 @@ require ( github.com/iotaledger/hive.go/stringify v0.0.0-20230906114834-b50190b9f9c2 // indirect github.com/iotaledger/inx-app v1.0.0-rc.3.0.20230829161228-3f4eb50a4d14 // indirect github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230829160617-69b96c7c9f9b // indirect - github.com/iotaledger/iota.go/v4 v4.0.0-20230906064556-75f5be378284 // indirect + github.com/iotaledger/iota.go/v4 v4.0.0-20230908070236-ae553965e1a3 // indirect github.com/ipfs/boxo v0.10.0 // indirect github.com/ipfs/go-cid v0.4.1 // indirect github.com/ipfs/go-datastore v0.6.0 // indirect diff --git a/tools/gendoc/go.sum b/tools/gendoc/go.sum index 5f4967123..3ad9cc1fe 100644 --- a/tools/gendoc/go.sum +++ b/tools/gendoc/go.sum @@ -311,8 +311,8 @@ github.com/iotaledger/inx-app v1.0.0-rc.3.0.20230829161228-3f4eb50a4d14 h1:BkDuQ github.com/iotaledger/inx-app v1.0.0-rc.3.0.20230829161228-3f4eb50a4d14/go.mod h1:ADBXzdHXTldP0NB2Vf+KbhDxkYciGRjzQVXT6Rdne1g= github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230829160617-69b96c7c9f9b h1:EPB/+iWeSx/WgJlzaXl8yjinxuD8CCOdi2ZPMLeeMVY= github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230829160617-69b96c7c9f9b/go.mod h1:B7gyJP6GshCSlEmY3CxEk5TZdsMs3UNz5U92hkFDdMs= -github.com/iotaledger/iota.go/v4 v4.0.0-20230906064556-75f5be378284 h1:SmPyo08i2/ymMZkmIQX/7EexuWWFl7CKDZjJr47zn8Q= -github.com/iotaledger/iota.go/v4 v4.0.0-20230906064556-75f5be378284/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= +github.com/iotaledger/iota.go/v4 v4.0.0-20230908070236-ae553965e1a3 h1:VnS5NOlgtmL9bEXHRKD9oYOzMSKYg8c9LiOUbd/CHN0= +github.com/iotaledger/iota.go/v4 v4.0.0-20230908070236-ae553965e1a3/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= github.com/ipfs/boxo v0.10.0 h1:tdDAxq8jrsbRkYoF+5Rcqyeb91hgWe2hp7iLu7ORZLY= github.com/ipfs/boxo v0.10.0/go.mod h1:Fg+BnfxZ0RPzR0nOodzdIq3A7KgoWAOWsEIImrIQdBM= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= diff --git a/tools/genesis-snapshot/go.mod b/tools/genesis-snapshot/go.mod index 096b9ea87..cc07b9b50 100644 --- a/tools/genesis-snapshot/go.mod +++ b/tools/genesis-snapshot/go.mod @@ -10,7 +10,7 @@ require ( github.com/iotaledger/hive.go/lo v0.0.0-20230906114834-b50190b9f9c2 github.com/iotaledger/hive.go/runtime v0.0.0-20230906114834-b50190b9f9c2 github.com/iotaledger/iota-core v0.0.0-00010101000000-000000000000 - github.com/iotaledger/iota.go/v4 v4.0.0-20230906064556-75f5be378284 + github.com/iotaledger/iota.go/v4 v4.0.0-20230908070236-ae553965e1a3 github.com/mr-tron/base58 v1.2.0 github.com/spf13/pflag v1.0.5 golang.org/x/crypto v0.13.0 diff --git a/tools/genesis-snapshot/go.sum b/tools/genesis-snapshot/go.sum index 9b20fdc88..2aefe5875 100644 --- a/tools/genesis-snapshot/go.sum +++ b/tools/genesis-snapshot/go.sum @@ -50,8 +50,8 @@ github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20230906114834-b50190b github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20230906114834-b50190b9f9c2/go.mod h1:IJgaaxbgKCsNat18jlJJEAxCY2oVYR3F30B+M4vJ89I= github.com/iotaledger/hive.go/stringify v0.0.0-20230906114834-b50190b9f9c2 h1:exATYMLT/d8fgMuVNO6kMDsFn9DUJEcyCuoBv9sP13g= github.com/iotaledger/hive.go/stringify v0.0.0-20230906114834-b50190b9f9c2/go.mod h1:FTo/UWzNYgnQ082GI9QVM9HFDERqf9rw9RivNpqrnTs= -github.com/iotaledger/iota.go/v4 v4.0.0-20230906064556-75f5be378284 h1:SmPyo08i2/ymMZkmIQX/7EexuWWFl7CKDZjJr47zn8Q= -github.com/iotaledger/iota.go/v4 v4.0.0-20230906064556-75f5be378284/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= +github.com/iotaledger/iota.go/v4 v4.0.0-20230908070236-ae553965e1a3 h1:VnS5NOlgtmL9bEXHRKD9oYOzMSKYg8c9LiOUbd/CHN0= +github.com/iotaledger/iota.go/v4 v4.0.0-20230908070236-ae553965e1a3/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= From 64dec2dbccc751d1e2256152ed18cd23e048ee3b Mon Sep 17 00:00:00 2001 From: Andrew Date: Sat, 9 Sep 2023 15:40:54 +0100 Subject: [PATCH 14/24] move committee check for validator to the filter --- .../scheduler/drr/scheduler.go | 40 ++++++++------ .../engine/filter/blockfilter/filter.go | 32 +++++++++++- .../engine/filter/blockfilter/filter_test.go | 52 +++++++++++++++++++ 3 files changed, 106 insertions(+), 18 deletions(-) diff --git a/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go b/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go index 0d72be983..bbeeaff8a 100644 --- a/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go +++ b/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go @@ -36,7 +36,7 @@ type Scheduler struct { seatManager seatmanager.SeatManager basicBuffer *BufferQueue - validatorBuffer map[iotago.AccountID]*ValidatorQueue + validatorBuffer *shrinkingmap.ShrinkingMap[iotago.AccountID, *ValidatorQueue] bufferMutex syncutils.RWMutex deficits *shrinkingmap.ShrinkingMap[iotago.AccountID, Deficit] @@ -71,12 +71,14 @@ func NewProvider(opts ...options.Option[Scheduler]) module.Provider[*engine.Engi s.bufferMutex.Lock() defer s.bufferMutex.Unlock() - for accountID, validatorQueue := range s.validatorBuffer { + s.validatorBuffer.ForEach(func(accountID iotago.AccountID, validatorQueue *ValidatorQueue) bool { if !s.seatManager.Committee(commitment.Index() + 1).HasAccount(accountID) { s.shutdownValidatorQueue(validatorQueue) - delete(s.validatorBuffer, accountID) + s.validatorBuffer.Delete(accountID) } - } + + return true + }) } }) e.Ledger.HookInitialized(func() { @@ -127,15 +129,17 @@ func New(apiProvider api.Provider, opts ...options.Option[Scheduler]) *Scheduler events: scheduler.NewEvents(), deficits: shrinkingmap.New[iotago.AccountID, Deficit](), apiProvider: apiProvider, - validatorBuffer: make(map[iotago.AccountID]*ValidatorQueue), + validatorBuffer: shrinkingmap.New[iotago.AccountID, *ValidatorQueue](), }, opts, ) } func (s *Scheduler) Shutdown() { - for _, validatorQueue := range s.validatorBuffer { + s.validatorBuffer.ForEach(func(_ iotago.AccountID, validatorQueue *ValidatorQueue) bool { s.shutdownValidatorQueue(validatorQueue) - } + + return true + }) close(s.shutdownSignal) s.TriggerStopped() } @@ -216,8 +220,7 @@ func (s *Scheduler) IsBlockIssuerReady(accountID iotago.AccountID, blocks ...*bl } func (s *Scheduler) AddBlock(block *blocks.Block) { - // TODO: can we remove the check for committee membership here? It should be checked before this in the accounts filter perhaps. - if _, isValidation := block.ValidationBlock(); isValidation && s.seatManager.Committee(block.ID().Index()).HasAccount(block.ProtocolBlock().IssuerID) { + if _, isValidation := block.ValidationBlock(); isValidation { s.enqueueValidationBlock(block) } else if _, isBasic := block.BasicBlock(); isBasic { s.enqueueBasicBlock(block) @@ -271,7 +274,7 @@ func (s *Scheduler) enqueueValidationBlock(block *blocks.Block) { s.bufferMutex.Lock() defer s.bufferMutex.Unlock() - validatorQueue, exists := s.validatorBuffer[block.ProtocolBlock().IssuerID] + validatorQueue, exists := s.validatorBuffer.Get(block.ProtocolBlock().IssuerID) if !exists { validatorQueue = s.addValidator(block.ProtocolBlock().IssuerID) } @@ -308,7 +311,7 @@ loop: } } -func (s *Scheduler) ValidatorLoop(validatorQueue *ValidatorQueue) { +func (s *Scheduler) validatorLoop(validatorQueue *ValidatorQueue) { var blockToSchedule *blocks.Block loop: for { @@ -363,9 +366,11 @@ func (s *Scheduler) selectBlockToScheduleWithLocking() { s.bufferMutex.Lock() defer s.bufferMutex.Unlock() - for _, validatorQueue := range s.validatorBuffer { + s.validatorBuffer.ForEach(func(accountID iotago.AccountID, validatorQueue *ValidatorQueue) bool { s.selectValidationBlockWithoutLocking(validatorQueue) - } + + return true + }) s.selectBasicBlockWithoutLocking() } @@ -634,7 +639,7 @@ func (s *Scheduler) ready(block *blocks.Block) { } func (s *Scheduler) readyValidationBlock(block *blocks.Block) { - if validatorQueue, exists := s.validatorBuffer[block.ProtocolBlock().IssuerID]; exists { + if validatorQueue, exists := s.validatorBuffer.Get(block.ProtocolBlock().IssuerID); exists { validatorQueue.Ready(block) } } @@ -675,10 +680,11 @@ func (s *Scheduler) deficitFromWork(work iotago.WorkScore) Deficit { } func (s *Scheduler) addValidator(accountID iotago.AccountID) *ValidatorQueue { - s.validatorBuffer[accountID] = NewValidatorQueue(accountID) - go s.ValidatorLoop(s.validatorBuffer[accountID]) + validatorQueue := NewValidatorQueue(accountID) + s.validatorBuffer.Set(accountID, validatorQueue) + go s.validatorLoop(validatorQueue) - return s.validatorBuffer[accountID] + return validatorQueue } func (s *Scheduler) shutdownValidatorQueue(validatorQueue *ValidatorQueue) { diff --git a/pkg/protocol/engine/filter/blockfilter/filter.go b/pkg/protocol/engine/filter/blockfilter/filter.go index f3e01bf2e..1e99a7da1 100644 --- a/pkg/protocol/engine/filter/blockfilter/filter.go +++ b/pkg/protocol/engine/filter/blockfilter/filter.go @@ -8,13 +8,16 @@ import ( "github.com/iotaledger/hive.go/ierrors" "github.com/iotaledger/hive.go/runtime/module" "github.com/iotaledger/hive.go/runtime/options" + "github.com/iotaledger/iota-core/pkg/core/account" "github.com/iotaledger/iota-core/pkg/model" "github.com/iotaledger/iota-core/pkg/protocol/engine" "github.com/iotaledger/iota-core/pkg/protocol/engine/filter" + iotago "github.com/iotaledger/iota.go/v4" "github.com/iotaledger/iota.go/v4/api" ) var ErrBlockTimeTooFarAheadInFuture = ierrors.New("a block cannot be too far ahead in the future") +var ErrValidatorNotInCommittee = ierrors.New("validation block issuer is not in the committee") // Filter filters blocks. type Filter struct { @@ -24,6 +27,8 @@ type Filter struct { optsMaxAllowedWallClockDrift time.Duration + committeeFunc func(iotago.SlotIndex) *account.SeatedAccounts + module.Module } @@ -34,7 +39,9 @@ func NewProvider(opts ...options.Option[Filter]) module.Provider[*engine.Engine, e.HookConstructed(func() { e.Events.Filter.LinkTo(f.events) - + e.SybilProtection.HookInitialized(func() { + f.committeeFunc = e.SybilProtection.SeatManager().Committee + }) f.TriggerInitialized() }) @@ -69,6 +76,29 @@ func (f *Filter) ProcessReceivedBlock(block *model.Block, source peer.ID) { return } + if _, isValidation := block.ValidationBlock(); isValidation { + blockAPI, err := f.apiProvider.APIForVersion(block.ProtocolBlock().ProtocolVersion) + if err != nil { + f.events.BlockPreFiltered.Trigger(&filter.BlockPreFilteredEvent{ + Block: block, + Reason: ierrors.Wrapf(err, "could not get API for version %d", block.ProtocolBlock().ProtocolVersion), + Source: source, + }) + + return + } + blockSlot := blockAPI.TimeProvider().SlotFromTime(block.ProtocolBlock().IssuingTime) + if !f.committeeFunc(blockSlot).HasAccount(block.ProtocolBlock().IssuerID) { + f.events.BlockPreFiltered.Trigger(&filter.BlockPreFilteredEvent{ + Block: block, + Reason: ierrors.Wrapf(ErrValidatorNotInCommittee, "validation block issuer %s is not part of the committee for slot %d", block.ProtocolBlock().IssuerID, blockSlot), + Source: source, + }) + + return + } + } + f.events.BlockPreAllowed.Trigger(block) } diff --git a/pkg/protocol/engine/filter/blockfilter/filter_test.go b/pkg/protocol/engine/filter/blockfilter/filter_test.go index 772394143..18ba4e9b1 100644 --- a/pkg/protocol/engine/filter/blockfilter/filter_test.go +++ b/pkg/protocol/engine/filter/blockfilter/filter_test.go @@ -9,6 +9,7 @@ import ( "github.com/iotaledger/hive.go/ierrors" "github.com/iotaledger/hive.go/runtime/options" "github.com/iotaledger/hive.go/serializer/v2/serix" + "github.com/iotaledger/iota-core/pkg/core/account" "github.com/iotaledger/iota-core/pkg/model" "github.com/iotaledger/iota-core/pkg/protocol/engine/filter" iotago "github.com/iotaledger/iota.go/v4" @@ -68,6 +69,30 @@ func (t *TestFramework) IssueUnsignedBlockAtTime(alias string, issuingTime time. return t.processBlock(alias, block) } +func (t *TestFramework) IssueValidationBlockAtTime(alias string, issuingTime time.Time, validatorAccountID iotago.AccountID) error { + version := t.apiProvider.LatestAPI().ProtocolParameters().Version() + block, err := builder.NewValidationBlockBuilder(t.apiProvider.LatestAPI()). + StrongParents(iotago.BlockIDs{tpkg.RandBlockID()}). + HighestSupportedVersion(version). + Sign(validatorAccountID, tpkg.RandEd25519PrivateKey()). + IssuingTime(issuingTime). + Build() + require.NoError(t.Test, err) + + return t.processBlock(alias, block) +} + +func mockedCommitteeFunc(validatorAccountID iotago.AccountID) func(iotago.SlotIndex) *account.SeatedAccounts { + mockedAccounts := account.NewAccounts() + mockedAccounts.Set(validatorAccountID, new(account.Pool)) + seatedAccounts := account.NewSeatedAccounts(mockedAccounts) + seatedAccounts.Set(account.SeatIndex(0), validatorAccountID) + + return func(slotIndex iotago.SlotIndex) *account.SeatedAccounts { + return seatedAccounts + } +} + func TestFilter_WithMaxAllowedWallClockDrift(t *testing.T) { allowedDrift := 3 * time.Second @@ -92,3 +117,30 @@ func TestFilter_WithMaxAllowedWallClockDrift(t *testing.T) { require.NoError(t, tf.IssueUnsignedBlockAtTime("acceptedFuture", time.Now().Add(allowedDrift))) require.NoError(t, tf.IssueUnsignedBlockAtTime("tooFarAheadFuture", time.Now().Add(allowedDrift).Add(1*time.Second))) } + +func TestFilter_ValidationBlocks(t *testing.T) { + testAPI := tpkg.TestAPI + + tf := NewTestFramework(t, + api.SingleVersionProvider(testAPI), + ) + + validatorAccountID := tpkg.RandAccountID() + nonValidatorAccountID := tpkg.RandAccountID() + + tf.Filter.committeeFunc = mockedCommitteeFunc(validatorAccountID) + + tf.Filter.events.BlockPreAllowed.Hook(func(block *model.Block) { + require.Equal(t, "validator", block.ID().Alias()) + require.NotEqual(t, "nonValidator", block.ID().Alias()) + }) + + tf.Filter.events.BlockPreFiltered.Hook(func(event *filter.BlockPreFilteredEvent) { + require.NotEqual(t, "validator", event.Block.ID().Alias()) + require.Equal(t, "nonValidator", event.Block.ID().Alias()) + require.True(t, ierrors.Is(event.Reason, ErrValidatorNotInCommittee)) + }) + + require.NoError(t, tf.IssueValidationBlockAtTime("validator", time.Now(), validatorAccountID)) + require.NoError(t, tf.IssueValidationBlockAtTime("nonValidator", time.Now(), nonValidatorAccountID)) +} From b21e9f203c1c97b6b7414a6b80139c542b99ad33 Mon Sep 17 00:00:00 2001 From: Andrew Date: Sat, 9 Sep 2023 16:58:43 +0100 Subject: [PATCH 15/24] add buffer drop policy to validator queues --- go.mod | 2 +- go.sum | 5 +++ .../scheduler/drr/drrbuffer.go | 25 ++++-------- .../scheduler/drr/scheduler.go | 38 ++++++++++++------- .../scheduler/drr/validatorqueue.go | 20 ++++++---- pkg/testsuite/testsuite.go | 1 + 6 files changed, 52 insertions(+), 39 deletions(-) diff --git a/go.mod b/go.mod index 579947976..e0d4ffd53 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/iotaledger/hive.go/stringify v0.0.0-20230906114834-b50190b9f9c2 github.com/iotaledger/inx-app v1.0.0-rc.3.0.20230829161228-3f4eb50a4d14 github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230829160617-69b96c7c9f9b - github.com/iotaledger/iota.go/v4 v4.0.0-20230908070236-ae553965e1a3 + github.com/iotaledger/iota.go/v4 v4.0.0-20230909152630-c52155893297 github.com/labstack/echo/v4 v4.11.1 github.com/labstack/gommon v0.4.0 github.com/libp2p/go-libp2p v0.30.0 diff --git a/go.sum b/go.sum index 38dc69b03..7f286d904 100644 --- a/go.sum +++ b/go.sum @@ -129,6 +129,7 @@ github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -309,6 +310,8 @@ github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230829160617-69b96c7c9f9b h1:EPB/+i github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230829160617-69b96c7c9f9b/go.mod h1:B7gyJP6GshCSlEmY3CxEk5TZdsMs3UNz5U92hkFDdMs= github.com/iotaledger/iota.go/v4 v4.0.0-20230908070236-ae553965e1a3 h1:VnS5NOlgtmL9bEXHRKD9oYOzMSKYg8c9LiOUbd/CHN0= github.com/iotaledger/iota.go/v4 v4.0.0-20230908070236-ae553965e1a3/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= +github.com/iotaledger/iota.go/v4 v4.0.0-20230909152630-c52155893297 h1:8lClNmnzKbV4Gb1K+yYBskLBO6bSHJx9LsoDzo+DxRs= +github.com/iotaledger/iota.go/v4 v4.0.0-20230909152630-c52155893297/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= github.com/ipfs/boxo v0.10.0 h1:tdDAxq8jrsbRkYoF+5Rcqyeb91hgWe2hp7iLu7ORZLY= github.com/ipfs/boxo v0.10.0/go.mod h1:Fg+BnfxZ0RPzR0nOodzdIq3A7KgoWAOWsEIImrIQdBM= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= @@ -342,6 +345,7 @@ github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= @@ -462,6 +466,7 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= diff --git a/pkg/protocol/engine/congestioncontrol/scheduler/drr/drrbuffer.go b/pkg/protocol/engine/congestioncontrol/scheduler/drr/drrbuffer.go index 6680a1568..0fa45bc4d 100644 --- a/pkg/protocol/engine/congestioncontrol/scheduler/drr/drrbuffer.go +++ b/pkg/protocol/engine/congestioncontrol/scheduler/drr/drrbuffer.go @@ -20,9 +20,6 @@ var ErrInsufficientMana = ierrors.New("insufficient issuer's mana to schedule th // BufferQueue represents a buffer of IssuerQueue. type BufferQueue struct { - // maxBuffer is the maximum buffer size in number of blocks. - maxBuffer int - activeIssuers *shrinkingmap.ShrinkingMap[iotago.AccountID, *ring.Ring] ring *ring.Ring // size is the number of blocks in the buffer. @@ -35,9 +32,8 @@ type BufferQueue struct { } // NewBufferQueue returns a new BufferQueue. -func NewBufferQueue(maxBuffer int) *BufferQueue { +func NewBufferQueue() *BufferQueue { return &BufferQueue{ - maxBuffer: maxBuffer, activeIssuers: shrinkingmap.New[iotago.AccountID, *ring.Ring](), ring: nil, lastScheduleTime: time.Now(), @@ -50,11 +46,6 @@ func (b *BufferQueue) NumActiveIssuers() int { return b.activeIssuers.Size() } -// MaxSize returns the max number of blocks in BufferQueue. -func (b *BufferQueue) MaxSize() int { - return b.maxBuffer -} - // Size returns the total number of blocks in BufferQueue. func (b *BufferQueue) Size() int { return b.size @@ -97,28 +88,28 @@ func (b *BufferQueue) GetIssuerQueue(issuerID iotago.AccountID) (*IssuerQueue, e // Submit submits a block. Return blocks dropped from the scheduler to make room for the submitted block. // The submitted block can also be returned as dropped if the issuer does not have enough mana. -func (b *BufferQueue) Submit(blk *blocks.Block, issuerQueue *IssuerQueue, quantumFunc func(iotago.AccountID) Deficit) (elements []*blocks.Block, err error) { +func (b *BufferQueue) Submit(blk *blocks.Block, issuerQueue *IssuerQueue, quantumFunc func(iotago.AccountID) Deficit, maxBuffer int) ([]*blocks.Block, bool) { // first we submit the block, and if it turns out that the issuer doesn't have enough bandwidth to submit, it will be removed by dropTail if !issuerQueue.Submit(blk) { - return nil, ierrors.Errorf("block already submitted %s", blk) + return nil, false } b.size++ // if max buffer size exceeded, drop from tail of the longest mana-scaled queue - if b.Size() > b.maxBuffer { - return b.dropTail(quantumFunc), nil + if b.Size() > maxBuffer { + return b.dropTail(quantumFunc, maxBuffer), true } - return nil, nil + return nil, true } -func (b *BufferQueue) dropTail(quantumFunc func(iotago.AccountID) Deficit) (droppedBlocks []*blocks.Block) { +func (b *BufferQueue) dropTail(quantumFunc func(iotago.AccountID) Deficit, maxBuffer int) (droppedBlocks []*blocks.Block) { start := b.Current() ringStart := b.ring // remove as many blocks as necessary to stay within max buffer size - for b.Size() > b.maxBuffer { + for b.Size() > maxBuffer { // TODO: extract to util func // find longest mana-scaled queue maxScale := math.Inf(-1) diff --git a/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go b/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go index bbeeaff8a..211e160bd 100644 --- a/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go +++ b/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go @@ -54,7 +54,7 @@ func NewProvider(opts ...options.Option[Scheduler]) module.Provider[*engine.Engi return module.Provide(func(e *engine.Engine) scheduler.Scheduler { s := New(e, opts...) s.errorHandler = e.ErrorHandler("scheduler") - s.basicBuffer = NewBufferQueue(int(s.apiProvider.CurrentAPI().ProtocolParameters().CongestionControlParameters().MaxBufferSize)) + s.basicBuffer = NewBufferQueue() e.HookConstructed(func() { s.latestCommittedSlot = func() iotago.SlotIndex { @@ -246,23 +246,28 @@ func (s *Scheduler) enqueueBasicBlock(block *blocks.Block) { issuerQueue = s.createIssuer(issuerID) } - droppedBlocks, err := s.basicBuffer.Submit(block, issuerQueue, func(issuerID iotago.AccountID) Deficit { - quantum, quantumErr := s.quantumFunc(issuerID, slotIndex) - if quantumErr != nil { - s.errorHandler(ierrors.Wrapf(quantumErr, "failed to retrieve deficit for issuerID %d in slot %d when submitting a block", issuerID, slotIndex)) + droppedBlocks, submitted := s.basicBuffer.Submit( + block, + issuerQueue, + func(issuerID iotago.AccountID) Deficit { + quantum, quantumErr := s.quantumFunc(issuerID, slotIndex) + if quantumErr != nil { + s.errorHandler(ierrors.Wrapf(quantumErr, "failed to retrieve deficit for issuerID %d in slot %d when submitting a block", issuerID, slotIndex)) - return 0 - } + return 0 + } - return quantum - }) + return quantum + }, + int(s.apiProvider.CurrentAPI().ProtocolParameters().CongestionControlParameters().MaxBufferSize), + ) // error submitting indicates that the block was already submitted so we do nothing else. - if err != nil { + if !submitted { return } for _, b := range droppedBlocks { b.SetDropped() - s.events.BlockDropped.Trigger(b, ierrors.New("block dropped from buffer")) + s.events.BlockDropped.Trigger(b, ierrors.New("basic block dropped from buffer")) } if block.SetEnqueued() { s.events.BlockEnqueued.Trigger(block) @@ -278,14 +283,19 @@ func (s *Scheduler) enqueueValidationBlock(block *blocks.Block) { if !exists { validatorQueue = s.addValidator(block.ProtocolBlock().IssuerID) } - validatorQueue.Submit(block) + droppedBlock, submitted := validatorQueue.Submit(block, int(s.apiProvider.CurrentAPI().ProtocolParameters().CongestionControlParameters().MaxValidationBufferSize)) + if !submitted { + return + } + if droppedBlock != nil { + droppedBlock.SetDropped() + s.events.BlockDropped.Trigger(droppedBlock, ierrors.New("validation block dropped from buffer")) + } if block.SetEnqueued() { s.events.BlockEnqueued.Trigger(block) s.tryReadyValidationBlock(block) } - - //s.scheduleValidationBlock(block) } func (s *Scheduler) basicBlockLoop() { diff --git a/pkg/protocol/engine/congestioncontrol/scheduler/drr/validatorqueue.go b/pkg/protocol/engine/congestioncontrol/scheduler/drr/validatorqueue.go index 6f8b757f0..1d941c55c 100644 --- a/pkg/protocol/engine/congestioncontrol/scheduler/drr/validatorqueue.go +++ b/pkg/protocol/engine/congestioncontrol/scheduler/drr/validatorqueue.go @@ -31,10 +31,12 @@ type ValidatorQueue struct { func NewValidatorQueue(accountID iotago.AccountID) *ValidatorQueue { return &ValidatorQueue{ - accountID: accountID, - submitted: shrinkingmap.New[iotago.BlockID, *blocks.Block](), - blockChan: make(chan *blocks.Block, 1), - shutdownSignal: make(chan struct{}), + accountID: accountID, + submitted: shrinkingmap.New[iotago.BlockID, *blocks.Block](), + blockChan: make(chan *blocks.Block, 1), + shutdownSignal: make(chan struct{}), + tokenBucket: 1, + lastScheduleTime: time.Now(), } } @@ -50,19 +52,23 @@ func (q *ValidatorQueue) AccountID() iotago.AccountID { return q.accountID } -func (q *ValidatorQueue) Submit(block *blocks.Block) bool { +func (q *ValidatorQueue) Submit(block *blocks.Block, maxBuffer int) (*blocks.Block, bool) { if blkAccountID := block.ProtocolBlock().IssuerID; q.accountID != blkAccountID { panic(fmt.Sprintf("issuerqueue: queue issuer ID(%x) and issuer ID(%x) does not match.", q.accountID, blkAccountID)) } if _, submitted := q.submitted.Get(block.ID()); submitted { - return false + return nil, false } q.submitted.Set(block.ID(), block) q.size.Inc() - return true + if int(q.size.Load()) > maxBuffer { + return q.RemoveTail(), true + } + + return nil, true } func (q *ValidatorQueue) Unsubmit(block *blocks.Block) bool { diff --git a/pkg/testsuite/testsuite.go b/pkg/testsuite/testsuite.go index 9a351fccf..22f45372d 100644 --- a/pkg/testsuite/testsuite.go +++ b/pkg/testsuite/testsuite.go @@ -136,6 +136,7 @@ func NewTestSuite(testingT *testing.T, opts ...options.Option[TestSuite]) *TestS t.optsSchedulerRate, t.optsMinMana, t.optsMaxBufferSize, + t.optsMaxBufferSize, ), iotago.WithStakingOptions(1, 1), ), From 91f713220e54e2bb7bbba0547be49a875147e696 Mon Sep 17 00:00:00 2001 From: Andrew Date: Sat, 9 Sep 2023 17:04:43 +0100 Subject: [PATCH 16/24] update presets with buffer params --- tools/genesis-snapshot/presets/presets.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/genesis-snapshot/presets/presets.go b/tools/genesis-snapshot/presets/presets.go index f9aa42da1..6115b4203 100644 --- a/tools/genesis-snapshot/presets/presets.go +++ b/tools/genesis-snapshot/presets/presets.go @@ -26,7 +26,7 @@ var Base = []options.Option[snapshotcreator.Options]{ iotago.WithTimeProviderOptions(time.Now().Unix(), 10, 13), iotago.WithLivenessOptions(5, 7, 14, 30), // increase/decrease threshold = fraction * slotDurationInSeconds * schedulerRate - iotago.WithCongestionControlOptions(500, 500, 500, 800000, 500000, 100000, 1, 100*iotago.MaxBlockSize), + iotago.WithCongestionControlOptions(500, 500, 500, 800000, 500000, 100000, 1, 1000, 100), iotago.WithWorkScoreOptions(25, 1, 10, 100, 50, 10, 10, 50, 1, 10, 250, 2), ), ), @@ -100,7 +100,7 @@ var Docker = []options.Option[snapshotcreator.Options]{ iotago.WithTimeProviderOptions(time.Now().Unix(), 10, 13), iotago.WithLivenessOptions(5, 7, 14, 30), // increase/decrease threshold = fraction * slotDurationInSeconds * schedulerRate - iotago.WithCongestionControlOptions(500, 500, 500, 800000, 500000, 100000, 1, 100*iotago.MaxBlockSize), + iotago.WithCongestionControlOptions(500, 500, 500, 800000, 500000, 100000, 1, 1000, 100), iotago.WithWorkScoreOptions(25, 1, 10, 100, 50, 10, 10, 50, 1, 10, 250, 2), ), ), @@ -154,7 +154,7 @@ var Feature = []options.Option[snapshotcreator.Options]{ iotago.WithTimeProviderOptions(1689848996, 10, 13), iotago.WithLivenessOptions(5, 10, 20, 30), // increase/decrease threshold = fraction * slotDurationInSeconds * schedulerRate - iotago.WithCongestionControlOptions(500, 500, 500, 800000, 500000, 100000, 1, 100*iotago.MaxBlockSize), + iotago.WithCongestionControlOptions(500, 500, 500, 800000, 500000, 100000, 1, 1000, 100), iotago.WithWorkScoreOptions(25, 1, 10, 100, 50, 10, 10, 50, 1, 10, 250, 2), ), ), From f53f638d267a47f1fcbc1afb3ca8d260bd4783d8 Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 11 Sep 2023 16:32:51 +0100 Subject: [PATCH 17/24] Add validator buffer metrics --- components/metrics/metrics_scheduler.go | 109 +++++++++++++----- .../scheduler/drr/scheduler.go | 57 ++++++--- .../scheduler/drr/validatorqueue.go | 53 +++++++++ .../scheduler/passthrough/scheduler.go | 10 +- .../congestioncontrol/scheduler/scheduler.go | 10 +- 5 files changed, 182 insertions(+), 57 deletions(-) diff --git a/components/metrics/metrics_scheduler.go b/components/metrics/metrics_scheduler.go index 4bf22e2af..0d81472b6 100644 --- a/components/metrics/metrics_scheduler.go +++ b/components/metrics/metrics_scheduler.go @@ -13,18 +13,21 @@ import ( const ( schedulerNamespace = "scheduler" - queueSizePerNodeWork = "queue_size_per_node_work" //nolint:gosec - queueSizePerNodeCount = "queue_size_per_node_count" - schedulerProcessedBlocks = "processed_blocks" - manaAmountPerNode = "mana_per_node" - scheduledBlockLabel = "scheduled" - skippedBlockLabel = "skipped" - droppedBlockLabel = "dropped" - enqueuedBlockLabel = "enqueued" - bufferReadyBlockCount = "buffer_ready_block_total" //nolint:gosec - bufferTotalSize = "buffer_size_block_total" - bufferMaxSize = "buffer_max_size" - rate = "rate" + queueSizePerNodeWork = "queue_size_per_node_work" //nolint:gosec + queueSizePerNodeCount = "queue_size_per_node_count" + validatorQueueSizePerNodeCount = "validator_queue_size_per_node_count" + schedulerProcessedBlocks = "processed_blocks" + manaAmountPerNode = "mana_per_node" + scheduledBlockLabel = "scheduled" + skippedBlockLabel = "skipped" + droppedBlockLabel = "dropped" + enqueuedBlockLabel = "enqueued" + basicBufferReadyBlockCount = "buffer_ready_block_total" //nolint:gosec + basicBufferTotalSize = "buffer_size_block_total" + basicBufferMaxSize = "buffer_max_size" + rate = "rate" + validatorBufferTotalSize = "validator_buffer_size_block_total" + validatorQueueMaxSize = "validator_buffer_max_size" ) var SchedulerMetrics = collector.NewCollection(schedulerNamespace, @@ -55,7 +58,6 @@ var SchedulerMetrics = collector.NewCollection(schedulerNamespace, }, event.WithWorkerPool(Component.WorkerPool)) }), )), - collector.WithMetric(collector.NewMetric(queueSizePerNodeCount, collector.WithType(collector.Gauge), collector.WithLabels("issuer_id"), @@ -63,23 +65,58 @@ var SchedulerMetrics = collector.NewCollection(schedulerNamespace, collector.WithHelp("Current size of each node's queue (as block count)."), collector.WithInitFunc(func() { deps.Protocol.Events.Engine.Scheduler.BlockEnqueued.Hook(func(block *blocks.Block) { - deps.Collector.Update(schedulerNamespace, queueSizePerNodeCount, float64(deps.Protocol.MainEngineInstance().Scheduler.IssuerQueueBlockCount(block.ProtocolBlock().IssuerID)), block.ProtocolBlock().IssuerID.String()) - + if _, isBasic := block.BasicBlock(); isBasic { + deps.Collector.Update(schedulerNamespace, queueSizePerNodeCount, float64(deps.Protocol.MainEngineInstance().Scheduler.IssuerQueueBlockCount(block.ProtocolBlock().IssuerID)), block.ProtocolBlock().IssuerID.String()) + } }, event.WithWorkerPool(Component.WorkerPool)) deps.Protocol.Events.Engine.Scheduler.BlockSkipped.Hook(func(block *blocks.Block) { - deps.Collector.Update(schedulerNamespace, queueSizePerNodeCount, float64(deps.Protocol.MainEngineInstance().Scheduler.IssuerQueueBlockCount(block.ProtocolBlock().IssuerID)), block.ProtocolBlock().IssuerID.String()) - + if _, isBasic := block.BasicBlock(); isBasic { + deps.Collector.Update(schedulerNamespace, queueSizePerNodeCount, float64(deps.Protocol.MainEngineInstance().Scheduler.IssuerQueueBlockCount(block.ProtocolBlock().IssuerID)), block.ProtocolBlock().IssuerID.String()) + } }, event.WithWorkerPool(Component.WorkerPool)) deps.Protocol.Events.Engine.Scheduler.BlockDropped.Hook(func(block *blocks.Block, _ error) { - deps.Collector.Update(schedulerNamespace, queueSizePerNodeCount, float64(deps.Protocol.MainEngineInstance().Scheduler.IssuerQueueBlockCount(block.ProtocolBlock().IssuerID)), block.ProtocolBlock().IssuerID.String()) - + if _, isBasic := block.BasicBlock(); isBasic { + deps.Collector.Update(schedulerNamespace, queueSizePerNodeCount, float64(deps.Protocol.MainEngineInstance().Scheduler.IssuerQueueBlockCount(block.ProtocolBlock().IssuerID)), block.ProtocolBlock().IssuerID.String()) + } }, event.WithWorkerPool(Component.WorkerPool)) deps.Protocol.Events.Engine.Scheduler.BlockScheduled.Hook(func(block *blocks.Block) { - deps.Collector.Update(schedulerNamespace, queueSizePerNodeCount, float64(deps.Protocol.MainEngineInstance().Scheduler.IssuerQueueBlockCount(block.ProtocolBlock().IssuerID)), block.ProtocolBlock().IssuerID.String()) + if _, isBasic := block.BasicBlock(); isBasic { + deps.Collector.Update(schedulerNamespace, queueSizePerNodeCount, float64(deps.Protocol.MainEngineInstance().Scheduler.IssuerQueueBlockCount(block.ProtocolBlock().IssuerID)), block.ProtocolBlock().IssuerID.String()) + } + }, event.WithWorkerPool(Component.WorkerPool)) + }), + )), + collector.WithMetric(collector.NewMetric(validatorQueueSizePerNodeCount, + collector.WithType(collector.Gauge), + collector.WithLabels("issuer_id"), + collector.WithPruningDelay(10*time.Minute), + collector.WithHelp("Current number of validation blocks in each validator's queue."), + collector.WithInitFunc(func() { + deps.Protocol.Events.Engine.Scheduler.BlockEnqueued.Hook(func(block *blocks.Block) { + if _, isValidation := block.ValidationBlock(); isValidation { + deps.Collector.Update(schedulerNamespace, validatorQueueSizePerNodeCount, float64(deps.Protocol.MainEngineInstance().Scheduler.ValidatorQueueBlockCount(block.ProtocolBlock().IssuerID)), block.ProtocolBlock().IssuerID.String()) + } + }, event.WithWorkerPool(Component.WorkerPool)) + deps.Protocol.Events.Engine.Scheduler.BlockSkipped.Hook(func(block *blocks.Block) { + if _, isValidation := block.ValidationBlock(); isValidation { + deps.Collector.Update(schedulerNamespace, validatorQueueSizePerNodeCount, float64(deps.Protocol.MainEngineInstance().Scheduler.ValidatorQueueBlockCount(block.ProtocolBlock().IssuerID)), block.ProtocolBlock().IssuerID.String()) + } + }, event.WithWorkerPool(Component.WorkerPool)) + + deps.Protocol.Events.Engine.Scheduler.BlockDropped.Hook(func(block *blocks.Block, _ error) { + if _, isValidation := block.ValidationBlock(); isValidation { + deps.Collector.Update(schedulerNamespace, validatorQueueSizePerNodeCount, float64(deps.Protocol.MainEngineInstance().Scheduler.ValidatorQueueBlockCount(block.ProtocolBlock().IssuerID)), block.ProtocolBlock().IssuerID.String()) + } + }, event.WithWorkerPool(Component.WorkerPool)) + + deps.Protocol.Events.Engine.Scheduler.BlockScheduled.Hook(func(block *blocks.Block) { + if _, isValidation := block.ValidationBlock(); isValidation { + deps.Collector.Update(schedulerNamespace, validatorQueueSizePerNodeCount, float64(deps.Protocol.MainEngineInstance().Scheduler.ValidatorQueueBlockCount(block.ProtocolBlock().IssuerID)), block.ProtocolBlock().IssuerID.String()) + } }, event.WithWorkerPool(Component.WorkerPool)) }), )), @@ -127,32 +164,46 @@ var SchedulerMetrics = collector.NewCollection(schedulerNamespace, }, event.WithWorkerPool(Component.WorkerPool)) }), )), - collector.WithMetric(collector.NewMetric(bufferMaxSize, + collector.WithMetric(collector.NewMetric(basicBufferMaxSize, collector.WithType(collector.Gauge), - collector.WithHelp("Maximum number of blocks that can be stored in the buffer."), + collector.WithHelp("Maximum number of basic blocks that can be stored in the buffer."), collector.WithCollectFunc(func() (float64, []string) { - return float64(deps.Protocol.MainEngineInstance().Scheduler.MaxBufferSize()), []string{} + return float64(deps.Protocol.MainEngineInstance().CurrentAPI().ProtocolParameters().CongestionControlParameters().MaxBufferSize), []string{} }), )), - collector.WithMetric(collector.NewMetric(bufferReadyBlockCount, + collector.WithMetric(collector.NewMetric(basicBufferReadyBlockCount, collector.WithType(collector.Gauge), collector.WithHelp("Number of ready blocks in the scheduler buffer."), collector.WithCollectFunc(func() (float64, []string) { return float64(deps.Protocol.MainEngineInstance().Scheduler.ReadyBlocksCount()), []string{} }), )), - collector.WithMetric(collector.NewMetric(bufferTotalSize, + collector.WithMetric(collector.NewMetric(basicBufferTotalSize, collector.WithType(collector.Gauge), - collector.WithHelp("Current size of the scheduler buffer (in bytes)."), + collector.WithHelp("Current number of basic blocks in the scheduler buffer."), collector.WithCollectFunc(func() (float64, []string) { - return float64(deps.Protocol.MainEngineInstance().Scheduler.BufferSize()), []string{} + return float64(deps.Protocol.MainEngineInstance().Scheduler.BasicBufferSize()), []string{} }), )), collector.WithMetric(collector.NewMetric(rate, collector.WithType(collector.Gauge), - collector.WithHelp("Current rate of the scheduler."), + collector.WithHelp("Current scheduling rate of basic blocks."), + collector.WithCollectFunc(func() (float64, []string) { + return float64(deps.Protocol.MainEngineInstance().CurrentAPI().ProtocolParameters().CongestionControlParameters().SchedulerRate), []string{} + }), + )), + collector.WithMetric(collector.NewMetric(validatorBufferTotalSize, + collector.WithType(collector.Gauge), + collector.WithHelp("Current number of validation blocks in the scheduling buffer."), + collector.WithCollectFunc(func() (float64, []string) { + return float64(deps.Protocol.MainEngineInstance().Scheduler.ValidatorBufferSize()), []string{} + }), + )), + collector.WithMetric(collector.NewMetric(validatorQueueMaxSize, + collector.WithType(collector.Gauge), + collector.WithHelp("Maximum number of validation blocks that can be stored in each validator queue."), collector.WithCollectFunc(func() (float64, []string) { - return float64(deps.Protocol.MainEngineInstance().Scheduler.Rate()), []string{} + return float64(deps.Protocol.MainEngineInstance().CurrentAPI().ProtocolParameters().CongestionControlParameters().MaxValidationBufferSize), []string{} }), )), ) diff --git a/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go b/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go index 211e160bd..bc698b39e 100644 --- a/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go +++ b/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go @@ -36,7 +36,7 @@ type Scheduler struct { seatManager seatmanager.SeatManager basicBuffer *BufferQueue - validatorBuffer *shrinkingmap.ShrinkingMap[iotago.AccountID, *ValidatorQueue] + validatorBuffer *ValidatorBuffer bufferMutex syncutils.RWMutex deficits *shrinkingmap.ShrinkingMap[iotago.AccountID, Deficit] @@ -71,7 +71,7 @@ func NewProvider(opts ...options.Option[Scheduler]) module.Provider[*engine.Engi s.bufferMutex.Lock() defer s.bufferMutex.Unlock() - s.validatorBuffer.ForEach(func(accountID iotago.AccountID, validatorQueue *ValidatorQueue) bool { + s.validatorBuffer.buffer.ForEach(func(accountID iotago.AccountID, validatorQueue *ValidatorQueue) bool { if !s.seatManager.Committee(commitment.Index() + 1).HasAccount(accountID) { s.shutdownValidatorQueue(validatorQueue) s.validatorBuffer.Delete(accountID) @@ -129,13 +129,13 @@ func New(apiProvider api.Provider, opts ...options.Option[Scheduler]) *Scheduler events: scheduler.NewEvents(), deficits: shrinkingmap.New[iotago.AccountID, Deficit](), apiProvider: apiProvider, - validatorBuffer: shrinkingmap.New[iotago.AccountID, *ValidatorQueue](), + validatorBuffer: NewValidatorBuffer(), }, opts, ) } func (s *Scheduler) Shutdown() { - s.validatorBuffer.ForEach(func(_ iotago.AccountID, validatorQueue *ValidatorQueue) bool { + s.validatorBuffer.buffer.ForEach(func(_ iotago.AccountID, validatorQueue *ValidatorQueue) bool { s.shutdownValidatorQueue(validatorQueue) return true @@ -152,12 +152,7 @@ func (s *Scheduler) Start() { s.TriggerInitialized() } -// Rate gets the rate of the scheduler in units of work per second. -func (s *Scheduler) Rate() iotago.WorkScore { - return s.apiProvider.CurrentAPI().ProtocolParameters().CongestionControlParameters().SchedulerRate -} - -// IssuerQueueSizeCount returns the queue size of the given issuer as block count. +// IssuerQueueSizeCount returns the number of blocks in the queue of the given issuer. func (s *Scheduler) IssuerQueueBlockCount(issuerID iotago.AccountID) int { s.bufferMutex.RLock() defer s.bufferMutex.RUnlock() @@ -173,14 +168,34 @@ func (s *Scheduler) IssuerQueueWork(issuerID iotago.AccountID) iotago.WorkScore return s.basicBuffer.IssuerQueue(issuerID).Work() } +// ValidatorQueueBlockCount returns the number of validation blocks in the validator queue of the given issuer. +func (s *Scheduler) ValidatorQueueBlockCount(issuerID iotago.AccountID) int { + s.bufferMutex.RLock() + defer s.bufferMutex.RUnlock() + + validatorQueue, exists := s.validatorBuffer.Get(issuerID) + if !exists { + return 0 + } + + return validatorQueue.Size() +} + // BufferSize returns the current buffer size of the Scheduler as block count. -func (s *Scheduler) BufferSize() int { +func (s *Scheduler) BasicBufferSize() int { s.bufferMutex.RLock() defer s.bufferMutex.RUnlock() return s.basicBuffer.Size() } +func (s *Scheduler) ValidatorBufferSize() int { + s.bufferMutex.RLock() + defer s.bufferMutex.RUnlock() + + return s.validatorBuffer.Size() +} + // MaxBufferSize returns the max buffer size of the Scheduler as block count. func (s *Scheduler) MaxBufferSize() int { return int(s.apiProvider.CurrentAPI().ProtocolParameters().CongestionControlParameters().MaxBufferSize) @@ -279,11 +294,11 @@ func (s *Scheduler) enqueueValidationBlock(block *blocks.Block) { s.bufferMutex.Lock() defer s.bufferMutex.Unlock() - validatorQueue, exists := s.validatorBuffer.Get(block.ProtocolBlock().IssuerID) + _, exists := s.validatorBuffer.Get(block.ProtocolBlock().IssuerID) if !exists { - validatorQueue = s.addValidator(block.ProtocolBlock().IssuerID) + s.addValidator(block.ProtocolBlock().IssuerID) } - droppedBlock, submitted := validatorQueue.Submit(block, int(s.apiProvider.CurrentAPI().ProtocolParameters().CongestionControlParameters().MaxValidationBufferSize)) + droppedBlock, submitted := s.validatorBuffer.Submit(block, int(s.apiProvider.CurrentAPI().ProtocolParameters().CongestionControlParameters().MaxValidationBufferSize)) if !submitted { return } @@ -376,8 +391,10 @@ func (s *Scheduler) selectBlockToScheduleWithLocking() { s.bufferMutex.Lock() defer s.bufferMutex.Unlock() - s.validatorBuffer.ForEach(func(accountID iotago.AccountID, validatorQueue *ValidatorQueue) bool { - s.selectValidationBlockWithoutLocking(validatorQueue) + s.validatorBuffer.buffer.ForEach(func(accountID iotago.AccountID, validatorQueue *ValidatorQueue) bool { + if s.selectValidationBlockWithoutLocking(validatorQueue) { + s.validatorBuffer.size-- + } return true }) @@ -385,15 +402,19 @@ func (s *Scheduler) selectBlockToScheduleWithLocking() { } -func (s *Scheduler) selectValidationBlockWithoutLocking(validatorQueue *ValidatorQueue) { +func (s *Scheduler) selectValidationBlockWithoutLocking(validatorQueue *ValidatorQueue) bool { // already a block selected to be scheduled. if len(validatorQueue.blockChan) > 0 { - return + return false } if blockToSchedule := validatorQueue.PopFront(); blockToSchedule != nil { validatorQueue.blockChan <- blockToSchedule + + return true } + + return false } func (s *Scheduler) selectBasicBlockWithoutLocking() { diff --git a/pkg/protocol/engine/congestioncontrol/scheduler/drr/validatorqueue.go b/pkg/protocol/engine/congestioncontrol/scheduler/drr/validatorqueue.go index 1d941c55c..8d2fd0aaf 100644 --- a/pkg/protocol/engine/congestioncontrol/scheduler/drr/validatorqueue.go +++ b/pkg/protocol/engine/congestioncontrol/scheduler/drr/validatorqueue.go @@ -173,3 +173,56 @@ func (q *ValidatorQueue) updateTokenBucket(rate float64, tokenBucketSize float64 func (q *ValidatorQueue) deductTokens(tokens float64) { q.tokenBucket -= tokens } + +type ValidatorBuffer struct { + buffer *shrinkingmap.ShrinkingMap[iotago.AccountID, *ValidatorQueue] + size int +} + +func NewValidatorBuffer() *ValidatorBuffer { + return &ValidatorBuffer{ + buffer: shrinkingmap.New[iotago.AccountID, *ValidatorQueue](), + } +} + +func (b *ValidatorBuffer) Size() int { + if b == nil { + return 0 + } + + return b.size +} + +func (b *ValidatorBuffer) Get(accountID iotago.AccountID) (*ValidatorQueue, bool) { + return b.buffer.Get(accountID) +} + +func (b *ValidatorBuffer) Set(accountID iotago.AccountID, validatorQueue *ValidatorQueue) bool { + return b.buffer.Set(accountID, validatorQueue) +} + +func (b *ValidatorBuffer) Submit(block *blocks.Block, maxBuffer int) (*blocks.Block, bool) { + validatorQueue, exists := b.buffer.Get(block.ProtocolBlock().IssuerID) + if !exists { + return nil, false + } + droppedBlock, submitted := validatorQueue.Submit(block, maxBuffer) + if submitted { + b.size++ + } + if droppedBlock != nil { + b.size-- + } + + return droppedBlock, submitted +} + +func (b *ValidatorBuffer) Delete(accountID iotago.AccountID) { + validatorQueue, exists := b.buffer.Get(accountID) + if !exists { + return + } + b.size -= validatorQueue.Size() + + b.buffer.Delete(accountID) +} diff --git a/pkg/protocol/engine/congestioncontrol/scheduler/passthrough/scheduler.go b/pkg/protocol/engine/congestioncontrol/scheduler/passthrough/scheduler.go index 716288b5f..1c034ac7b 100644 --- a/pkg/protocol/engine/congestioncontrol/scheduler/passthrough/scheduler.go +++ b/pkg/protocol/engine/congestioncontrol/scheduler/passthrough/scheduler.go @@ -42,23 +42,23 @@ func (s *Scheduler) IsBlockIssuerReady(_ iotago.AccountID, _ ...*blocks.Block) b return true } -func (s *Scheduler) Rate() iotago.WorkScore { +func (s *Scheduler) BasicBufferSize() int { return 0 } -func (s *Scheduler) BufferSize() int { +func (s *Scheduler) ValidatorBufferSize() int { return 0 } -func (s *Scheduler) MaxBufferSize() int { +func (s *Scheduler) ReadyBlocksCount() int { return 0 } -func (s *Scheduler) ReadyBlocksCount() int { +func (s *Scheduler) IssuerQueueBlockCount(_ iotago.AccountID) int { return 0 } -func (s *Scheduler) IssuerQueueBlockCount(_ iotago.AccountID) int { +func (s *Scheduler) ValidatorQueueBlockCount(_ iotago.AccountID) int { return 0 } diff --git a/pkg/protocol/engine/congestioncontrol/scheduler/scheduler.go b/pkg/protocol/engine/congestioncontrol/scheduler/scheduler.go index 28e81f71d..997106554 100644 --- a/pkg/protocol/engine/congestioncontrol/scheduler/scheduler.go +++ b/pkg/protocol/engine/congestioncontrol/scheduler/scheduler.go @@ -11,18 +11,18 @@ type Scheduler interface { AddBlock(*blocks.Block) // IsBlockIssuerReady returns true if the block issuer is ready to issuer a block, i.e., if the block issuer were to add a block to the scheduler, would it be scheduled. IsBlockIssuerReady(iotago.AccountID, ...*blocks.Block) bool - // Rate returns the specified rate of the Scheduler in work units. - Rate() iotago.WorkScore // BufferSize returns the current buffer size of the Scheduler as block count. - BufferSize() int - // MaxBufferSize returns the max buffer size of the Scheduler as block count. - MaxBufferSize() int + BasicBufferSize() int + // ValidatorBufferSize returns the current buffer size of the Scheduler as block count. + ValidatorBufferSize() int // ReadyBlocksCount returns the number of ready blocks. ReadyBlocksCount() int // IssuerQueueBlockCount returns the queue size of the given issuer as block count. IssuerQueueBlockCount(issuerID iotago.AccountID) int // IssuerQueueWork returns the queue size of the given issuer in work units. IssuerQueueWork(issuerID iotago.AccountID) iotago.WorkScore + // ValidatorQueueBlockCount returns the queue size of the given validator as block count. + ValidatorQueueBlockCount(validatorID iotago.AccountID) int module.Interface } From 2b5fa12b4462643eff512e56c011d4037e1715dc Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 11 Sep 2023 17:14:16 +0100 Subject: [PATCH 18/24] go mod tidy --- go.mod | 2 +- go.sum | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index e0d4ffd53..9090d4c1d 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/iotaledger/hive.go/stringify v0.0.0-20230906114834-b50190b9f9c2 github.com/iotaledger/inx-app v1.0.0-rc.3.0.20230829161228-3f4eb50a4d14 github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230829160617-69b96c7c9f9b - github.com/iotaledger/iota.go/v4 v4.0.0-20230909152630-c52155893297 + github.com/iotaledger/iota.go/v4 v4.0.0-20230911155200-b5188e9c8b9e github.com/labstack/echo/v4 v4.11.1 github.com/labstack/gommon v0.4.0 github.com/libp2p/go-libp2p v0.30.0 diff --git a/go.sum b/go.sum index 7f286d904..b66348281 100644 --- a/go.sum +++ b/go.sum @@ -312,6 +312,10 @@ github.com/iotaledger/iota.go/v4 v4.0.0-20230908070236-ae553965e1a3 h1:VnS5NOlgt github.com/iotaledger/iota.go/v4 v4.0.0-20230908070236-ae553965e1a3/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= github.com/iotaledger/iota.go/v4 v4.0.0-20230909152630-c52155893297 h1:8lClNmnzKbV4Gb1K+yYBskLBO6bSHJx9LsoDzo+DxRs= github.com/iotaledger/iota.go/v4 v4.0.0-20230909152630-c52155893297/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= +github.com/iotaledger/iota.go/v4 v4.0.0-20230911154900-76a815a68a07 h1:PaHJQSDZNDXPE2+hbOWy1guiFtJE1t6p3u6rouX2+hs= +github.com/iotaledger/iota.go/v4 v4.0.0-20230911154900-76a815a68a07/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= +github.com/iotaledger/iota.go/v4 v4.0.0-20230911155200-b5188e9c8b9e h1:5Z8Li4OVm+a8xLaFds6v0e8UXbOxVoyJLifQY/x9riE= +github.com/iotaledger/iota.go/v4 v4.0.0-20230911155200-b5188e9c8b9e/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= github.com/ipfs/boxo v0.10.0 h1:tdDAxq8jrsbRkYoF+5Rcqyeb91hgWe2hp7iLu7ORZLY= github.com/ipfs/boxo v0.10.0/go.mod h1:Fg+BnfxZ0RPzR0nOodzdIq3A7KgoWAOWsEIImrIQdBM= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= From 1a898986d8676964d7fd534a7c372c3b3e6525cc Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 11 Sep 2023 17:31:53 +0100 Subject: [PATCH 19/24] go mod tidy --- go.sum | 9 --------- pkg/testsuite/testsuite.go | 2 +- tools/evil-spammer/go.mod | 2 +- tools/evil-spammer/go.sum | 4 ++-- tools/gendoc/go.mod | 2 +- tools/gendoc/go.sum | 4 ++-- tools/genesis-snapshot/go.mod | 2 +- tools/genesis-snapshot/go.sum | 4 ++-- 8 files changed, 10 insertions(+), 19 deletions(-) diff --git a/go.sum b/go.sum index b66348281..ea340c195 100644 --- a/go.sum +++ b/go.sum @@ -129,7 +129,6 @@ github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -308,12 +307,6 @@ github.com/iotaledger/inx-app v1.0.0-rc.3.0.20230829161228-3f4eb50a4d14 h1:BkDuQ github.com/iotaledger/inx-app v1.0.0-rc.3.0.20230829161228-3f4eb50a4d14/go.mod h1:ADBXzdHXTldP0NB2Vf+KbhDxkYciGRjzQVXT6Rdne1g= github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230829160617-69b96c7c9f9b h1:EPB/+iWeSx/WgJlzaXl8yjinxuD8CCOdi2ZPMLeeMVY= github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230829160617-69b96c7c9f9b/go.mod h1:B7gyJP6GshCSlEmY3CxEk5TZdsMs3UNz5U92hkFDdMs= -github.com/iotaledger/iota.go/v4 v4.0.0-20230908070236-ae553965e1a3 h1:VnS5NOlgtmL9bEXHRKD9oYOzMSKYg8c9LiOUbd/CHN0= -github.com/iotaledger/iota.go/v4 v4.0.0-20230908070236-ae553965e1a3/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= -github.com/iotaledger/iota.go/v4 v4.0.0-20230909152630-c52155893297 h1:8lClNmnzKbV4Gb1K+yYBskLBO6bSHJx9LsoDzo+DxRs= -github.com/iotaledger/iota.go/v4 v4.0.0-20230909152630-c52155893297/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= -github.com/iotaledger/iota.go/v4 v4.0.0-20230911154900-76a815a68a07 h1:PaHJQSDZNDXPE2+hbOWy1guiFtJE1t6p3u6rouX2+hs= -github.com/iotaledger/iota.go/v4 v4.0.0-20230911154900-76a815a68a07/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= github.com/iotaledger/iota.go/v4 v4.0.0-20230911155200-b5188e9c8b9e h1:5Z8Li4OVm+a8xLaFds6v0e8UXbOxVoyJLifQY/x9riE= github.com/iotaledger/iota.go/v4 v4.0.0-20230911155200-b5188e9c8b9e/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= github.com/ipfs/boxo v0.10.0 h1:tdDAxq8jrsbRkYoF+5Rcqyeb91hgWe2hp7iLu7ORZLY= @@ -349,7 +342,6 @@ github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= @@ -470,7 +462,6 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= diff --git a/pkg/testsuite/testsuite.go b/pkg/testsuite/testsuite.go index 22f45372d..cd4f56535 100644 --- a/pkg/testsuite/testsuite.go +++ b/pkg/testsuite/testsuite.go @@ -138,7 +138,7 @@ func NewTestSuite(testingT *testing.T, opts ...options.Option[TestSuite]) *TestS t.optsMaxBufferSize, t.optsMaxBufferSize, ), - iotago.WithStakingOptions(1, 1), + iotago.WithStakingOptions(1, 1, 1), ), ) diff --git a/tools/evil-spammer/go.mod b/tools/evil-spammer/go.mod index 351eeee10..46f354ba5 100644 --- a/tools/evil-spammer/go.mod +++ b/tools/evil-spammer/go.mod @@ -17,7 +17,7 @@ require ( github.com/iotaledger/hive.go/runtime v0.0.0-20230906114834-b50190b9f9c2 github.com/iotaledger/iota-core v0.0.0-00010101000000-000000000000 github.com/iotaledger/iota-core/tools/genesis-snapshot v0.0.0-00010101000000-000000000000 - github.com/iotaledger/iota.go/v4 v4.0.0-20230908070236-ae553965e1a3 + github.com/iotaledger/iota.go/v4 v4.0.0-20230911155200-b5188e9c8b9e github.com/mr-tron/base58 v1.2.0 go.uber.org/atomic v1.11.0 ) diff --git a/tools/evil-spammer/go.sum b/tools/evil-spammer/go.sum index 9f3ede384..75372285d 100644 --- a/tools/evil-spammer/go.sum +++ b/tools/evil-spammer/go.sum @@ -197,8 +197,8 @@ github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20230906114834-b50190b github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20230906114834-b50190b9f9c2/go.mod h1:IJgaaxbgKCsNat18jlJJEAxCY2oVYR3F30B+M4vJ89I= github.com/iotaledger/hive.go/stringify v0.0.0-20230906114834-b50190b9f9c2 h1:exATYMLT/d8fgMuVNO6kMDsFn9DUJEcyCuoBv9sP13g= github.com/iotaledger/hive.go/stringify v0.0.0-20230906114834-b50190b9f9c2/go.mod h1:FTo/UWzNYgnQ082GI9QVM9HFDERqf9rw9RivNpqrnTs= -github.com/iotaledger/iota.go/v4 v4.0.0-20230908070236-ae553965e1a3 h1:VnS5NOlgtmL9bEXHRKD9oYOzMSKYg8c9LiOUbd/CHN0= -github.com/iotaledger/iota.go/v4 v4.0.0-20230908070236-ae553965e1a3/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= +github.com/iotaledger/iota.go/v4 v4.0.0-20230911155200-b5188e9c8b9e h1:5Z8Li4OVm+a8xLaFds6v0e8UXbOxVoyJLifQY/x9riE= +github.com/iotaledger/iota.go/v4 v4.0.0-20230911155200-b5188e9c8b9e/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= diff --git a/tools/gendoc/go.mod b/tools/gendoc/go.mod index 4c3d33b64..ea06a6144 100644 --- a/tools/gendoc/go.mod +++ b/tools/gendoc/go.mod @@ -72,7 +72,7 @@ require ( github.com/iotaledger/hive.go/stringify v0.0.0-20230906114834-b50190b9f9c2 // indirect github.com/iotaledger/inx-app v1.0.0-rc.3.0.20230829161228-3f4eb50a4d14 // indirect github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230829160617-69b96c7c9f9b // indirect - github.com/iotaledger/iota.go/v4 v4.0.0-20230908070236-ae553965e1a3 // indirect + github.com/iotaledger/iota.go/v4 v4.0.0-20230911155200-b5188e9c8b9e // indirect github.com/ipfs/boxo v0.10.0 // indirect github.com/ipfs/go-cid v0.4.1 // indirect github.com/ipfs/go-datastore v0.6.0 // indirect diff --git a/tools/gendoc/go.sum b/tools/gendoc/go.sum index 3ad9cc1fe..b899ff357 100644 --- a/tools/gendoc/go.sum +++ b/tools/gendoc/go.sum @@ -311,8 +311,8 @@ github.com/iotaledger/inx-app v1.0.0-rc.3.0.20230829161228-3f4eb50a4d14 h1:BkDuQ github.com/iotaledger/inx-app v1.0.0-rc.3.0.20230829161228-3f4eb50a4d14/go.mod h1:ADBXzdHXTldP0NB2Vf+KbhDxkYciGRjzQVXT6Rdne1g= github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230829160617-69b96c7c9f9b h1:EPB/+iWeSx/WgJlzaXl8yjinxuD8CCOdi2ZPMLeeMVY= github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230829160617-69b96c7c9f9b/go.mod h1:B7gyJP6GshCSlEmY3CxEk5TZdsMs3UNz5U92hkFDdMs= -github.com/iotaledger/iota.go/v4 v4.0.0-20230908070236-ae553965e1a3 h1:VnS5NOlgtmL9bEXHRKD9oYOzMSKYg8c9LiOUbd/CHN0= -github.com/iotaledger/iota.go/v4 v4.0.0-20230908070236-ae553965e1a3/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= +github.com/iotaledger/iota.go/v4 v4.0.0-20230911155200-b5188e9c8b9e h1:5Z8Li4OVm+a8xLaFds6v0e8UXbOxVoyJLifQY/x9riE= +github.com/iotaledger/iota.go/v4 v4.0.0-20230911155200-b5188e9c8b9e/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= github.com/ipfs/boxo v0.10.0 h1:tdDAxq8jrsbRkYoF+5Rcqyeb91hgWe2hp7iLu7ORZLY= github.com/ipfs/boxo v0.10.0/go.mod h1:Fg+BnfxZ0RPzR0nOodzdIq3A7KgoWAOWsEIImrIQdBM= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= diff --git a/tools/genesis-snapshot/go.mod b/tools/genesis-snapshot/go.mod index cc07b9b50..d513dfbcd 100644 --- a/tools/genesis-snapshot/go.mod +++ b/tools/genesis-snapshot/go.mod @@ -10,7 +10,7 @@ require ( github.com/iotaledger/hive.go/lo v0.0.0-20230906114834-b50190b9f9c2 github.com/iotaledger/hive.go/runtime v0.0.0-20230906114834-b50190b9f9c2 github.com/iotaledger/iota-core v0.0.0-00010101000000-000000000000 - github.com/iotaledger/iota.go/v4 v4.0.0-20230908070236-ae553965e1a3 + github.com/iotaledger/iota.go/v4 v4.0.0-20230911155200-b5188e9c8b9e github.com/mr-tron/base58 v1.2.0 github.com/spf13/pflag v1.0.5 golang.org/x/crypto v0.13.0 diff --git a/tools/genesis-snapshot/go.sum b/tools/genesis-snapshot/go.sum index 2aefe5875..d1464e91c 100644 --- a/tools/genesis-snapshot/go.sum +++ b/tools/genesis-snapshot/go.sum @@ -50,8 +50,8 @@ github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20230906114834-b50190b github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20230906114834-b50190b9f9c2/go.mod h1:IJgaaxbgKCsNat18jlJJEAxCY2oVYR3F30B+M4vJ89I= github.com/iotaledger/hive.go/stringify v0.0.0-20230906114834-b50190b9f9c2 h1:exATYMLT/d8fgMuVNO6kMDsFn9DUJEcyCuoBv9sP13g= github.com/iotaledger/hive.go/stringify v0.0.0-20230906114834-b50190b9f9c2/go.mod h1:FTo/UWzNYgnQ082GI9QVM9HFDERqf9rw9RivNpqrnTs= -github.com/iotaledger/iota.go/v4 v4.0.0-20230908070236-ae553965e1a3 h1:VnS5NOlgtmL9bEXHRKD9oYOzMSKYg8c9LiOUbd/CHN0= -github.com/iotaledger/iota.go/v4 v4.0.0-20230908070236-ae553965e1a3/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= +github.com/iotaledger/iota.go/v4 v4.0.0-20230911155200-b5188e9c8b9e h1:5Z8Li4OVm+a8xLaFds6v0e8UXbOxVoyJLifQY/x9riE= +github.com/iotaledger/iota.go/v4 v4.0.0-20230911155200-b5188e9c8b9e/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= From a4ba938575e0ce1768490e3d9e3ee18fca41aaef Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 11 Sep 2023 17:40:42 +0100 Subject: [PATCH 20/24] go mod tidy --- go.mod | 2 +- go.sum | 4 ++-- tools/evil-spammer/go.mod | 2 +- tools/evil-spammer/go.sum | 4 ++-- tools/gendoc/go.mod | 2 +- tools/gendoc/go.sum | 4 ++-- tools/genesis-snapshot/go.mod | 2 +- tools/genesis-snapshot/go.sum | 4 ++-- 8 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 9090d4c1d..1879c0213 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/iotaledger/hive.go/stringify v0.0.0-20230906114834-b50190b9f9c2 github.com/iotaledger/inx-app v1.0.0-rc.3.0.20230829161228-3f4eb50a4d14 github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230829160617-69b96c7c9f9b - github.com/iotaledger/iota.go/v4 v4.0.0-20230911155200-b5188e9c8b9e + github.com/iotaledger/iota.go/v4 v4.0.0-20230911163626-ec5833cb0094 github.com/labstack/echo/v4 v4.11.1 github.com/labstack/gommon v0.4.0 github.com/libp2p/go-libp2p v0.30.0 diff --git a/go.sum b/go.sum index ea340c195..5ecfd51d0 100644 --- a/go.sum +++ b/go.sum @@ -307,8 +307,8 @@ github.com/iotaledger/inx-app v1.0.0-rc.3.0.20230829161228-3f4eb50a4d14 h1:BkDuQ github.com/iotaledger/inx-app v1.0.0-rc.3.0.20230829161228-3f4eb50a4d14/go.mod h1:ADBXzdHXTldP0NB2Vf+KbhDxkYciGRjzQVXT6Rdne1g= github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230829160617-69b96c7c9f9b h1:EPB/+iWeSx/WgJlzaXl8yjinxuD8CCOdi2ZPMLeeMVY= github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230829160617-69b96c7c9f9b/go.mod h1:B7gyJP6GshCSlEmY3CxEk5TZdsMs3UNz5U92hkFDdMs= -github.com/iotaledger/iota.go/v4 v4.0.0-20230911155200-b5188e9c8b9e h1:5Z8Li4OVm+a8xLaFds6v0e8UXbOxVoyJLifQY/x9riE= -github.com/iotaledger/iota.go/v4 v4.0.0-20230911155200-b5188e9c8b9e/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= +github.com/iotaledger/iota.go/v4 v4.0.0-20230911163626-ec5833cb0094 h1:zd3ar9F3j/LJF1fedOdSUc4X876gJCFy1zMFa9yg1X8= +github.com/iotaledger/iota.go/v4 v4.0.0-20230911163626-ec5833cb0094/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= github.com/ipfs/boxo v0.10.0 h1:tdDAxq8jrsbRkYoF+5Rcqyeb91hgWe2hp7iLu7ORZLY= github.com/ipfs/boxo v0.10.0/go.mod h1:Fg+BnfxZ0RPzR0nOodzdIq3A7KgoWAOWsEIImrIQdBM= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= diff --git a/tools/evil-spammer/go.mod b/tools/evil-spammer/go.mod index 46f354ba5..8cbf3a4c3 100644 --- a/tools/evil-spammer/go.mod +++ b/tools/evil-spammer/go.mod @@ -17,7 +17,7 @@ require ( github.com/iotaledger/hive.go/runtime v0.0.0-20230906114834-b50190b9f9c2 github.com/iotaledger/iota-core v0.0.0-00010101000000-000000000000 github.com/iotaledger/iota-core/tools/genesis-snapshot v0.0.0-00010101000000-000000000000 - github.com/iotaledger/iota.go/v4 v4.0.0-20230911155200-b5188e9c8b9e + github.com/iotaledger/iota.go/v4 v4.0.0-20230911163626-ec5833cb0094 github.com/mr-tron/base58 v1.2.0 go.uber.org/atomic v1.11.0 ) diff --git a/tools/evil-spammer/go.sum b/tools/evil-spammer/go.sum index 75372285d..f5d58c49d 100644 --- a/tools/evil-spammer/go.sum +++ b/tools/evil-spammer/go.sum @@ -197,8 +197,8 @@ github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20230906114834-b50190b github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20230906114834-b50190b9f9c2/go.mod h1:IJgaaxbgKCsNat18jlJJEAxCY2oVYR3F30B+M4vJ89I= github.com/iotaledger/hive.go/stringify v0.0.0-20230906114834-b50190b9f9c2 h1:exATYMLT/d8fgMuVNO6kMDsFn9DUJEcyCuoBv9sP13g= github.com/iotaledger/hive.go/stringify v0.0.0-20230906114834-b50190b9f9c2/go.mod h1:FTo/UWzNYgnQ082GI9QVM9HFDERqf9rw9RivNpqrnTs= -github.com/iotaledger/iota.go/v4 v4.0.0-20230911155200-b5188e9c8b9e h1:5Z8Li4OVm+a8xLaFds6v0e8UXbOxVoyJLifQY/x9riE= -github.com/iotaledger/iota.go/v4 v4.0.0-20230911155200-b5188e9c8b9e/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= +github.com/iotaledger/iota.go/v4 v4.0.0-20230911163626-ec5833cb0094 h1:zd3ar9F3j/LJF1fedOdSUc4X876gJCFy1zMFa9yg1X8= +github.com/iotaledger/iota.go/v4 v4.0.0-20230911163626-ec5833cb0094/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= diff --git a/tools/gendoc/go.mod b/tools/gendoc/go.mod index ea06a6144..9b40c1a82 100644 --- a/tools/gendoc/go.mod +++ b/tools/gendoc/go.mod @@ -72,7 +72,7 @@ require ( github.com/iotaledger/hive.go/stringify v0.0.0-20230906114834-b50190b9f9c2 // indirect github.com/iotaledger/inx-app v1.0.0-rc.3.0.20230829161228-3f4eb50a4d14 // indirect github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230829160617-69b96c7c9f9b // indirect - github.com/iotaledger/iota.go/v4 v4.0.0-20230911155200-b5188e9c8b9e // indirect + github.com/iotaledger/iota.go/v4 v4.0.0-20230911163626-ec5833cb0094 // indirect github.com/ipfs/boxo v0.10.0 // indirect github.com/ipfs/go-cid v0.4.1 // indirect github.com/ipfs/go-datastore v0.6.0 // indirect diff --git a/tools/gendoc/go.sum b/tools/gendoc/go.sum index b899ff357..fcfcc3a77 100644 --- a/tools/gendoc/go.sum +++ b/tools/gendoc/go.sum @@ -311,8 +311,8 @@ github.com/iotaledger/inx-app v1.0.0-rc.3.0.20230829161228-3f4eb50a4d14 h1:BkDuQ github.com/iotaledger/inx-app v1.0.0-rc.3.0.20230829161228-3f4eb50a4d14/go.mod h1:ADBXzdHXTldP0NB2Vf+KbhDxkYciGRjzQVXT6Rdne1g= github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230829160617-69b96c7c9f9b h1:EPB/+iWeSx/WgJlzaXl8yjinxuD8CCOdi2ZPMLeeMVY= github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230829160617-69b96c7c9f9b/go.mod h1:B7gyJP6GshCSlEmY3CxEk5TZdsMs3UNz5U92hkFDdMs= -github.com/iotaledger/iota.go/v4 v4.0.0-20230911155200-b5188e9c8b9e h1:5Z8Li4OVm+a8xLaFds6v0e8UXbOxVoyJLifQY/x9riE= -github.com/iotaledger/iota.go/v4 v4.0.0-20230911155200-b5188e9c8b9e/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= +github.com/iotaledger/iota.go/v4 v4.0.0-20230911163626-ec5833cb0094 h1:zd3ar9F3j/LJF1fedOdSUc4X876gJCFy1zMFa9yg1X8= +github.com/iotaledger/iota.go/v4 v4.0.0-20230911163626-ec5833cb0094/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= github.com/ipfs/boxo v0.10.0 h1:tdDAxq8jrsbRkYoF+5Rcqyeb91hgWe2hp7iLu7ORZLY= github.com/ipfs/boxo v0.10.0/go.mod h1:Fg+BnfxZ0RPzR0nOodzdIq3A7KgoWAOWsEIImrIQdBM= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= diff --git a/tools/genesis-snapshot/go.mod b/tools/genesis-snapshot/go.mod index d513dfbcd..e72788fcf 100644 --- a/tools/genesis-snapshot/go.mod +++ b/tools/genesis-snapshot/go.mod @@ -10,7 +10,7 @@ require ( github.com/iotaledger/hive.go/lo v0.0.0-20230906114834-b50190b9f9c2 github.com/iotaledger/hive.go/runtime v0.0.0-20230906114834-b50190b9f9c2 github.com/iotaledger/iota-core v0.0.0-00010101000000-000000000000 - github.com/iotaledger/iota.go/v4 v4.0.0-20230911155200-b5188e9c8b9e + github.com/iotaledger/iota.go/v4 v4.0.0-20230911163626-ec5833cb0094 github.com/mr-tron/base58 v1.2.0 github.com/spf13/pflag v1.0.5 golang.org/x/crypto v0.13.0 diff --git a/tools/genesis-snapshot/go.sum b/tools/genesis-snapshot/go.sum index d1464e91c..2ca8cb673 100644 --- a/tools/genesis-snapshot/go.sum +++ b/tools/genesis-snapshot/go.sum @@ -50,8 +50,8 @@ github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20230906114834-b50190b github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20230906114834-b50190b9f9c2/go.mod h1:IJgaaxbgKCsNat18jlJJEAxCY2oVYR3F30B+M4vJ89I= github.com/iotaledger/hive.go/stringify v0.0.0-20230906114834-b50190b9f9c2 h1:exATYMLT/d8fgMuVNO6kMDsFn9DUJEcyCuoBv9sP13g= github.com/iotaledger/hive.go/stringify v0.0.0-20230906114834-b50190b9f9c2/go.mod h1:FTo/UWzNYgnQ082GI9QVM9HFDERqf9rw9RivNpqrnTs= -github.com/iotaledger/iota.go/v4 v4.0.0-20230911155200-b5188e9c8b9e h1:5Z8Li4OVm+a8xLaFds6v0e8UXbOxVoyJLifQY/x9riE= -github.com/iotaledger/iota.go/v4 v4.0.0-20230911155200-b5188e9c8b9e/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= +github.com/iotaledger/iota.go/v4 v4.0.0-20230911163626-ec5833cb0094 h1:zd3ar9F3j/LJF1fedOdSUc4X876gJCFy1zMFa9yg1X8= +github.com/iotaledger/iota.go/v4 v4.0.0-20230911163626-ec5833cb0094/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= From 6d303f346636e232fb0cb2d960cb8c5415ac374a Mon Sep 17 00:00:00 2001 From: Andrew Date: Tue, 12 Sep 2023 08:23:13 +0100 Subject: [PATCH 21/24] add burning of Mana proportional to stake for over issuance --- .../engine/accounts/accountsledger/manager.go | 34 +++++++++++-------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/pkg/protocol/engine/accounts/accountsledger/manager.go b/pkg/protocol/engine/accounts/accountsledger/manager.go index 11d26003a..7422807d0 100644 --- a/pkg/protocol/engine/accounts/accountsledger/manager.go +++ b/pkg/protocol/engine/accounts/accountsledger/manager.go @@ -412,6 +412,8 @@ func (m *Manager) preserveDestroyedAccountData(accountID iotago.AccountID) (acco func (m *Manager) computeBlockBurnsForSlot(slotIndex iotago.SlotIndex, rmc iotago.Mana) (burns map[iotago.AccountID]iotago.Mana, err error) { burns = make(map[iotago.AccountID]iotago.Mana) + validationBlockCount := make(map[iotago.AccountID]int) + apiForSlot := m.apiProvider.APIForSlot(slotIndex) if set, exists := m.blockBurns.Get(slotIndex); exists { for it := set.Iterator(); it.HasNext(); { blockID := it.Next() @@ -421,22 +423,26 @@ func (m *Manager) computeBlockBurnsForSlot(slotIndex iotago.SlotIndex, rmc iotag } if _, isBasicBlock := block.BasicBlock(); isBasicBlock { burns[block.ProtocolBlock().IssuerID] += iotago.Mana(block.WorkScore()) * rmc + } else if _, isValidationBlock := block.ValidationBlock(); isValidationBlock { + validationBlockCount[block.ProtocolBlock().IssuerID]++ + } + } + validationBlocksPerSlot := int(apiForSlot.ProtocolParameters().ValidationBlocksPerSlot()) + for accountID, count := range validationBlockCount { + if count > validationBlocksPerSlot { + // penalize over-issuance + accountData, exists, err := m.Account(accountID, slotIndex) + if !exists { + return nil, ierrors.Wrapf(err, "cannot compute penalty for over-issuing validator, account %s could not be retrieved", accountID) + } + punishmentEpochs := apiForSlot.ProtocolParameters().PunishmentEpochs() + manaPunishment, err := apiForSlot.ManaDecayProvider().ManaGenerationWithDecay(accountData.ValidatorStake, slotIndex, slotIndex+apiForSlot.TimeProvider().EpochDurationSlots()*iotago.SlotIndex(punishmentEpochs)) + if err != nil { + return nil, ierrors.Wrapf(err, "cannot compute penalty for over-issuing validator with account ID %s due to problem with mana generation", accountID) + } + burns[accountID] += iotago.Mana(count-validationBlocksPerSlot) * manaPunishment } } - - // TODO: issue #338 enable this block of code and fix the tests to issue correct rate of validation blocks. - // validationBlockCount := make(map[iotago.AccountID]int) - // else if _, isValidationBlock := block.ValidationBlock(); isValidationBlock { - // validationBlockCount[block.ProtocolBlock().IssuerID]++ - // } - // } - // validationBlocksPerSlot := int(m.apiProvider.APIForSlot(slotIndex).ProtocolParameters().ValidationBlocksPerSlot()) - // for accountID, count := range validationBlockCount { - // if count > validationBlocksPerSlot { - // // penalize over-issuance by charging for a maximum work score block for each validation block over the quota - // burns[accountID] += iotago.Mana(count-validationBlocksPerSlot) * iotago.Mana(m.apiProvider.CurrentAPI().MaxBlockWork()) * rmc - // } - // } } return burns, nil From fee1bc85aa7af9e43986a853e3275daba6177262 Mon Sep 17 00:00:00 2001 From: Andrew Date: Tue, 12 Sep 2023 12:20:45 +0100 Subject: [PATCH 22/24] increase validation blocks per slot in tests --- pkg/testsuite/testsuite.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/testsuite/testsuite.go b/pkg/testsuite/testsuite.go index cd4f56535..9b5095114 100644 --- a/pkg/testsuite/testsuite.go +++ b/pkg/testsuite/testsuite.go @@ -138,7 +138,7 @@ func NewTestSuite(testingT *testing.T, opts ...options.Option[TestSuite]) *TestS t.optsMaxBufferSize, t.optsMaxBufferSize, ), - iotago.WithStakingOptions(1, 1, 1), + iotago.WithStakingOptions(1, 100, 1), ), ) From 2e539d5fc0dc4b59f884cdd23cd2ea335d5444f0 Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 13 Sep 2023 15:32:46 +0100 Subject: [PATCH 23/24] fix bugs in account manager --- components/validator/issuer.go | 2 +- pkg/protocol/engine/accounts/accountsledger/manager.go | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/components/validator/issuer.go b/components/validator/issuer.go index 9b913159c..83555a134 100644 --- a/components/validator/issuer.go +++ b/components/validator/issuer.go @@ -62,6 +62,6 @@ func issueValidatorBlock(ctx context.Context) { return } - Component.LogDebug("Issued validator block: %s - commitment %s %d - latest finalized slot %d", modelBlock.ID(), modelBlock.ProtocolBlock().SlotCommitmentID, modelBlock.ProtocolBlock().SlotCommitmentID.Index(), modelBlock.ProtocolBlock().LatestFinalizedSlot) + Component.LogDebugf("Issued validator block: %s - commitment %s %d - latest finalized slot %d", modelBlock.ID(), modelBlock.ProtocolBlock().SlotCommitmentID, modelBlock.ProtocolBlock().SlotCommitmentID.Index(), modelBlock.ProtocolBlock().LatestFinalizedSlot) } diff --git a/pkg/protocol/engine/accounts/accountsledger/manager.go b/pkg/protocol/engine/accounts/accountsledger/manager.go index 7422807d0..5bf108a72 100644 --- a/pkg/protocol/engine/accounts/accountsledger/manager.go +++ b/pkg/protocol/engine/accounts/accountsledger/manager.go @@ -200,6 +200,11 @@ func (m *Manager) Account(accountID iotago.AccountID, targetIndex iotago.SlotInd m.mutex.RLock() defer m.mutex.RUnlock() + return m.account(accountID, targetIndex) + +} + +func (m *Manager) account(accountID iotago.AccountID, targetIndex iotago.SlotIndex) (accountData *accounts.AccountData, exists bool, err error) { // if m.latestCommittedSlot < maxCommittableAge we should have all history maxCommittableAge := m.apiProvider.APIForSlot(targetIndex).ProtocolParameters().MaxCommittableAge() if m.latestCommittedSlot >= maxCommittableAge && targetIndex+maxCommittableAge < m.latestCommittedSlot { @@ -431,7 +436,7 @@ func (m *Manager) computeBlockBurnsForSlot(slotIndex iotago.SlotIndex, rmc iotag for accountID, count := range validationBlockCount { if count > validationBlocksPerSlot { // penalize over-issuance - accountData, exists, err := m.Account(accountID, slotIndex) + accountData, exists, err := m.account(accountID, m.latestCommittedSlot) if !exists { return nil, ierrors.Wrapf(err, "cannot compute penalty for over-issuing validator, account %s could not be retrieved", accountID) } From 81dade85c9b59dcb4a7de512e22eaf8f8bef1d99 Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 13 Sep 2023 15:38:01 +0100 Subject: [PATCH 24/24] go mod tidy --- go.mod | 4 ++-- go.sum | 8 ++++---- tools/evil-spammer/go.mod | 4 ++-- tools/evil-spammer/go.sum | 8 ++++---- tools/gendoc/go.mod | 4 ++-- tools/gendoc/go.sum | 8 ++++---- tools/genesis-snapshot/go.mod | 4 ++-- tools/genesis-snapshot/go.sum | 8 ++++---- 8 files changed, 24 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index 1879c0213..b462156bb 100644 --- a/go.mod +++ b/go.mod @@ -20,11 +20,11 @@ require ( github.com/iotaledger/hive.go/lo v0.0.0-20230906114834-b50190b9f9c2 github.com/iotaledger/hive.go/logger v0.0.0-20230906114834-b50190b9f9c2 github.com/iotaledger/hive.go/runtime v0.0.0-20230906114834-b50190b9f9c2 - github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20230906114834-b50190b9f9c2 + github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20230912111751-d84fba02bb7c github.com/iotaledger/hive.go/stringify v0.0.0-20230906114834-b50190b9f9c2 github.com/iotaledger/inx-app v1.0.0-rc.3.0.20230829161228-3f4eb50a4d14 github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230829160617-69b96c7c9f9b - github.com/iotaledger/iota.go/v4 v4.0.0-20230911163626-ec5833cb0094 + github.com/iotaledger/iota.go/v4 v4.0.0-20230913143616-917572c7752d github.com/labstack/echo/v4 v4.11.1 github.com/labstack/gommon v0.4.0 github.com/libp2p/go-libp2p v0.30.0 diff --git a/go.sum b/go.sum index 5ecfd51d0..f10a8c760 100644 --- a/go.sum +++ b/go.sum @@ -299,16 +299,16 @@ github.com/iotaledger/hive.go/logger v0.0.0-20230906114834-b50190b9f9c2 h1:vVs4Z github.com/iotaledger/hive.go/logger v0.0.0-20230906114834-b50190b9f9c2/go.mod h1:sxqWRdZ1OOxwkxVczuGcW034Mpt2vFh5ebJHO++ZYeI= github.com/iotaledger/hive.go/runtime v0.0.0-20230906114834-b50190b9f9c2 h1:jpX2K+d9+FaCngP3dTjSIabm+OIxThc/AQPKvp2d23c= github.com/iotaledger/hive.go/runtime v0.0.0-20230906114834-b50190b9f9c2/go.mod h1:fXVyQ1MAwxe/EmjAnG8WcQqbzGk9EW/FsJ/n16H/f/w= -github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20230906114834-b50190b9f9c2 h1:kEakiZRk/LUb2CxQvSOsdTgUXK1Ee4BNec9ps40+iXQ= -github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20230906114834-b50190b9f9c2/go.mod h1:IJgaaxbgKCsNat18jlJJEAxCY2oVYR3F30B+M4vJ89I= +github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20230912111751-d84fba02bb7c h1:dijcFDsCtbUIGxeC5XiWIl1azYHb3x/zGJZ3P+9mABY= +github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20230912111751-d84fba02bb7c/go.mod h1:IJgaaxbgKCsNat18jlJJEAxCY2oVYR3F30B+M4vJ89I= github.com/iotaledger/hive.go/stringify v0.0.0-20230906114834-b50190b9f9c2 h1:exATYMLT/d8fgMuVNO6kMDsFn9DUJEcyCuoBv9sP13g= github.com/iotaledger/hive.go/stringify v0.0.0-20230906114834-b50190b9f9c2/go.mod h1:FTo/UWzNYgnQ082GI9QVM9HFDERqf9rw9RivNpqrnTs= github.com/iotaledger/inx-app v1.0.0-rc.3.0.20230829161228-3f4eb50a4d14 h1:BkDuQxUYo9aZ4XYuh8EbXWtZBdh7WvL7oh2unWNUFMo= github.com/iotaledger/inx-app v1.0.0-rc.3.0.20230829161228-3f4eb50a4d14/go.mod h1:ADBXzdHXTldP0NB2Vf+KbhDxkYciGRjzQVXT6Rdne1g= github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230829160617-69b96c7c9f9b h1:EPB/+iWeSx/WgJlzaXl8yjinxuD8CCOdi2ZPMLeeMVY= github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230829160617-69b96c7c9f9b/go.mod h1:B7gyJP6GshCSlEmY3CxEk5TZdsMs3UNz5U92hkFDdMs= -github.com/iotaledger/iota.go/v4 v4.0.0-20230911163626-ec5833cb0094 h1:zd3ar9F3j/LJF1fedOdSUc4X876gJCFy1zMFa9yg1X8= -github.com/iotaledger/iota.go/v4 v4.0.0-20230911163626-ec5833cb0094/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= +github.com/iotaledger/iota.go/v4 v4.0.0-20230913143616-917572c7752d h1:p9IchKq6kft758XDlnN/tAEXJMXGlmQPmbdxolba1gs= +github.com/iotaledger/iota.go/v4 v4.0.0-20230913143616-917572c7752d/go.mod h1:DWCa+mXRTGWBV0EHVuvToUxAEcICe2Pab9hBlxBamKo= github.com/ipfs/boxo v0.10.0 h1:tdDAxq8jrsbRkYoF+5Rcqyeb91hgWe2hp7iLu7ORZLY= github.com/ipfs/boxo v0.10.0/go.mod h1:Fg+BnfxZ0RPzR0nOodzdIq3A7KgoWAOWsEIImrIQdBM= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= diff --git a/tools/evil-spammer/go.mod b/tools/evil-spammer/go.mod index 8cbf3a4c3..0d1f5f2f8 100644 --- a/tools/evil-spammer/go.mod +++ b/tools/evil-spammer/go.mod @@ -17,7 +17,7 @@ require ( github.com/iotaledger/hive.go/runtime v0.0.0-20230906114834-b50190b9f9c2 github.com/iotaledger/iota-core v0.0.0-00010101000000-000000000000 github.com/iotaledger/iota-core/tools/genesis-snapshot v0.0.0-00010101000000-000000000000 - github.com/iotaledger/iota.go/v4 v4.0.0-20230911163626-ec5833cb0094 + github.com/iotaledger/iota.go/v4 v4.0.0-20230913143616-917572c7752d github.com/mr-tron/base58 v1.2.0 go.uber.org/atomic v1.11.0 ) @@ -41,7 +41,7 @@ require ( github.com/iotaledger/hive.go/constraints v0.0.0-20230906114834-b50190b9f9c2 // indirect github.com/iotaledger/hive.go/core v1.0.0-rc.3.0.20230906114834-b50190b9f9c2 // indirect github.com/iotaledger/hive.go/kvstore v0.0.0-20230906114834-b50190b9f9c2 // indirect - github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20230906114834-b50190b9f9c2 // indirect + github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20230912111751-d84fba02bb7c // indirect github.com/iotaledger/hive.go/stringify v0.0.0-20230906114834-b50190b9f9c2 // indirect github.com/ipfs/go-cid v0.4.1 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect diff --git a/tools/evil-spammer/go.sum b/tools/evil-spammer/go.sum index f5d58c49d..11ff5b5f3 100644 --- a/tools/evil-spammer/go.sum +++ b/tools/evil-spammer/go.sum @@ -193,12 +193,12 @@ github.com/iotaledger/hive.go/logger v0.0.0-20230906114834-b50190b9f9c2 h1:vVs4Z github.com/iotaledger/hive.go/logger v0.0.0-20230906114834-b50190b9f9c2/go.mod h1:sxqWRdZ1OOxwkxVczuGcW034Mpt2vFh5ebJHO++ZYeI= github.com/iotaledger/hive.go/runtime v0.0.0-20230906114834-b50190b9f9c2 h1:jpX2K+d9+FaCngP3dTjSIabm+OIxThc/AQPKvp2d23c= github.com/iotaledger/hive.go/runtime v0.0.0-20230906114834-b50190b9f9c2/go.mod h1:fXVyQ1MAwxe/EmjAnG8WcQqbzGk9EW/FsJ/n16H/f/w= -github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20230906114834-b50190b9f9c2 h1:kEakiZRk/LUb2CxQvSOsdTgUXK1Ee4BNec9ps40+iXQ= -github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20230906114834-b50190b9f9c2/go.mod h1:IJgaaxbgKCsNat18jlJJEAxCY2oVYR3F30B+M4vJ89I= +github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20230912111751-d84fba02bb7c h1:dijcFDsCtbUIGxeC5XiWIl1azYHb3x/zGJZ3P+9mABY= +github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20230912111751-d84fba02bb7c/go.mod h1:IJgaaxbgKCsNat18jlJJEAxCY2oVYR3F30B+M4vJ89I= github.com/iotaledger/hive.go/stringify v0.0.0-20230906114834-b50190b9f9c2 h1:exATYMLT/d8fgMuVNO6kMDsFn9DUJEcyCuoBv9sP13g= github.com/iotaledger/hive.go/stringify v0.0.0-20230906114834-b50190b9f9c2/go.mod h1:FTo/UWzNYgnQ082GI9QVM9HFDERqf9rw9RivNpqrnTs= -github.com/iotaledger/iota.go/v4 v4.0.0-20230911163626-ec5833cb0094 h1:zd3ar9F3j/LJF1fedOdSUc4X876gJCFy1zMFa9yg1X8= -github.com/iotaledger/iota.go/v4 v4.0.0-20230911163626-ec5833cb0094/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= +github.com/iotaledger/iota.go/v4 v4.0.0-20230913143616-917572c7752d h1:p9IchKq6kft758XDlnN/tAEXJMXGlmQPmbdxolba1gs= +github.com/iotaledger/iota.go/v4 v4.0.0-20230913143616-917572c7752d/go.mod h1:DWCa+mXRTGWBV0EHVuvToUxAEcICe2Pab9hBlxBamKo= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= diff --git a/tools/gendoc/go.mod b/tools/gendoc/go.mod index 9b40c1a82..0321b8553 100644 --- a/tools/gendoc/go.mod +++ b/tools/gendoc/go.mod @@ -68,11 +68,11 @@ require ( github.com/iotaledger/hive.go/lo v0.0.0-20230906114834-b50190b9f9c2 // indirect github.com/iotaledger/hive.go/logger v0.0.0-20230906114834-b50190b9f9c2 // indirect github.com/iotaledger/hive.go/runtime v0.0.0-20230906114834-b50190b9f9c2 // indirect - github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20230906114834-b50190b9f9c2 // indirect + github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20230912111751-d84fba02bb7c // indirect github.com/iotaledger/hive.go/stringify v0.0.0-20230906114834-b50190b9f9c2 // indirect github.com/iotaledger/inx-app v1.0.0-rc.3.0.20230829161228-3f4eb50a4d14 // indirect github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230829160617-69b96c7c9f9b // indirect - github.com/iotaledger/iota.go/v4 v4.0.0-20230911163626-ec5833cb0094 // indirect + github.com/iotaledger/iota.go/v4 v4.0.0-20230913143616-917572c7752d // indirect github.com/ipfs/boxo v0.10.0 // indirect github.com/ipfs/go-cid v0.4.1 // indirect github.com/ipfs/go-datastore v0.6.0 // indirect diff --git a/tools/gendoc/go.sum b/tools/gendoc/go.sum index fcfcc3a77..43a124e49 100644 --- a/tools/gendoc/go.sum +++ b/tools/gendoc/go.sum @@ -303,16 +303,16 @@ github.com/iotaledger/hive.go/logger v0.0.0-20230906114834-b50190b9f9c2 h1:vVs4Z github.com/iotaledger/hive.go/logger v0.0.0-20230906114834-b50190b9f9c2/go.mod h1:sxqWRdZ1OOxwkxVczuGcW034Mpt2vFh5ebJHO++ZYeI= github.com/iotaledger/hive.go/runtime v0.0.0-20230906114834-b50190b9f9c2 h1:jpX2K+d9+FaCngP3dTjSIabm+OIxThc/AQPKvp2d23c= github.com/iotaledger/hive.go/runtime v0.0.0-20230906114834-b50190b9f9c2/go.mod h1:fXVyQ1MAwxe/EmjAnG8WcQqbzGk9EW/FsJ/n16H/f/w= -github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20230906114834-b50190b9f9c2 h1:kEakiZRk/LUb2CxQvSOsdTgUXK1Ee4BNec9ps40+iXQ= -github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20230906114834-b50190b9f9c2/go.mod h1:IJgaaxbgKCsNat18jlJJEAxCY2oVYR3F30B+M4vJ89I= +github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20230912111751-d84fba02bb7c h1:dijcFDsCtbUIGxeC5XiWIl1azYHb3x/zGJZ3P+9mABY= +github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20230912111751-d84fba02bb7c/go.mod h1:IJgaaxbgKCsNat18jlJJEAxCY2oVYR3F30B+M4vJ89I= github.com/iotaledger/hive.go/stringify v0.0.0-20230906114834-b50190b9f9c2 h1:exATYMLT/d8fgMuVNO6kMDsFn9DUJEcyCuoBv9sP13g= github.com/iotaledger/hive.go/stringify v0.0.0-20230906114834-b50190b9f9c2/go.mod h1:FTo/UWzNYgnQ082GI9QVM9HFDERqf9rw9RivNpqrnTs= github.com/iotaledger/inx-app v1.0.0-rc.3.0.20230829161228-3f4eb50a4d14 h1:BkDuQxUYo9aZ4XYuh8EbXWtZBdh7WvL7oh2unWNUFMo= github.com/iotaledger/inx-app v1.0.0-rc.3.0.20230829161228-3f4eb50a4d14/go.mod h1:ADBXzdHXTldP0NB2Vf+KbhDxkYciGRjzQVXT6Rdne1g= github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230829160617-69b96c7c9f9b h1:EPB/+iWeSx/WgJlzaXl8yjinxuD8CCOdi2ZPMLeeMVY= github.com/iotaledger/inx/go v1.0.0-rc.2.0.20230829160617-69b96c7c9f9b/go.mod h1:B7gyJP6GshCSlEmY3CxEk5TZdsMs3UNz5U92hkFDdMs= -github.com/iotaledger/iota.go/v4 v4.0.0-20230911163626-ec5833cb0094 h1:zd3ar9F3j/LJF1fedOdSUc4X876gJCFy1zMFa9yg1X8= -github.com/iotaledger/iota.go/v4 v4.0.0-20230911163626-ec5833cb0094/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= +github.com/iotaledger/iota.go/v4 v4.0.0-20230913143616-917572c7752d h1:p9IchKq6kft758XDlnN/tAEXJMXGlmQPmbdxolba1gs= +github.com/iotaledger/iota.go/v4 v4.0.0-20230913143616-917572c7752d/go.mod h1:DWCa+mXRTGWBV0EHVuvToUxAEcICe2Pab9hBlxBamKo= github.com/ipfs/boxo v0.10.0 h1:tdDAxq8jrsbRkYoF+5Rcqyeb91hgWe2hp7iLu7ORZLY= github.com/ipfs/boxo v0.10.0/go.mod h1:Fg+BnfxZ0RPzR0nOodzdIq3A7KgoWAOWsEIImrIQdBM= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= diff --git a/tools/genesis-snapshot/go.mod b/tools/genesis-snapshot/go.mod index e72788fcf..700a39470 100644 --- a/tools/genesis-snapshot/go.mod +++ b/tools/genesis-snapshot/go.mod @@ -10,7 +10,7 @@ require ( github.com/iotaledger/hive.go/lo v0.0.0-20230906114834-b50190b9f9c2 github.com/iotaledger/hive.go/runtime v0.0.0-20230906114834-b50190b9f9c2 github.com/iotaledger/iota-core v0.0.0-00010101000000-000000000000 - github.com/iotaledger/iota.go/v4 v4.0.0-20230911163626-ec5833cb0094 + github.com/iotaledger/iota.go/v4 v4.0.0-20230913143616-917572c7752d github.com/mr-tron/base58 v1.2.0 github.com/spf13/pflag v1.0.5 golang.org/x/crypto v0.13.0 @@ -32,7 +32,7 @@ require ( github.com/iotaledger/hive.go/core v1.0.0-rc.3.0.20230906114834-b50190b9f9c2 // indirect github.com/iotaledger/hive.go/ds v0.0.0-20230906114834-b50190b9f9c2 // indirect github.com/iotaledger/hive.go/kvstore v0.0.0-20230906114834-b50190b9f9c2 // indirect - github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20230906114834-b50190b9f9c2 // indirect + github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20230912111751-d84fba02bb7c // indirect github.com/iotaledger/hive.go/stringify v0.0.0-20230906114834-b50190b9f9c2 // indirect github.com/ipfs/go-cid v0.4.1 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect diff --git a/tools/genesis-snapshot/go.sum b/tools/genesis-snapshot/go.sum index 2ca8cb673..4248f134b 100644 --- a/tools/genesis-snapshot/go.sum +++ b/tools/genesis-snapshot/go.sum @@ -46,12 +46,12 @@ github.com/iotaledger/hive.go/lo v0.0.0-20230906114834-b50190b9f9c2 h1:wLZPoEOVv github.com/iotaledger/hive.go/lo v0.0.0-20230906114834-b50190b9f9c2/go.mod h1:/LERu5vqcessCqr40Wxmbx4x0bbymsK7GuL+TK/ckKo= github.com/iotaledger/hive.go/runtime v0.0.0-20230906114834-b50190b9f9c2 h1:jpX2K+d9+FaCngP3dTjSIabm+OIxThc/AQPKvp2d23c= github.com/iotaledger/hive.go/runtime v0.0.0-20230906114834-b50190b9f9c2/go.mod h1:fXVyQ1MAwxe/EmjAnG8WcQqbzGk9EW/FsJ/n16H/f/w= -github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20230906114834-b50190b9f9c2 h1:kEakiZRk/LUb2CxQvSOsdTgUXK1Ee4BNec9ps40+iXQ= -github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20230906114834-b50190b9f9c2/go.mod h1:IJgaaxbgKCsNat18jlJJEAxCY2oVYR3F30B+M4vJ89I= +github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20230912111751-d84fba02bb7c h1:dijcFDsCtbUIGxeC5XiWIl1azYHb3x/zGJZ3P+9mABY= +github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20230912111751-d84fba02bb7c/go.mod h1:IJgaaxbgKCsNat18jlJJEAxCY2oVYR3F30B+M4vJ89I= github.com/iotaledger/hive.go/stringify v0.0.0-20230906114834-b50190b9f9c2 h1:exATYMLT/d8fgMuVNO6kMDsFn9DUJEcyCuoBv9sP13g= github.com/iotaledger/hive.go/stringify v0.0.0-20230906114834-b50190b9f9c2/go.mod h1:FTo/UWzNYgnQ082GI9QVM9HFDERqf9rw9RivNpqrnTs= -github.com/iotaledger/iota.go/v4 v4.0.0-20230911163626-ec5833cb0094 h1:zd3ar9F3j/LJF1fedOdSUc4X876gJCFy1zMFa9yg1X8= -github.com/iotaledger/iota.go/v4 v4.0.0-20230911163626-ec5833cb0094/go.mod h1:MM3RLtTEsfT6Wh0EhpgmzVO/HM0/NOw+E7+mnGTnyA0= +github.com/iotaledger/iota.go/v4 v4.0.0-20230913143616-917572c7752d h1:p9IchKq6kft758XDlnN/tAEXJMXGlmQPmbdxolba1gs= +github.com/iotaledger/iota.go/v4 v4.0.0-20230913143616-917572c7752d/go.mod h1:DWCa+mXRTGWBV0EHVuvToUxAEcICe2Pab9hBlxBamKo= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=