From 10afdba7bd16808d8622afe735b637b8a8e2777e Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Thu, 12 Sep 2024 12:47:20 +0300 Subject: [PATCH 01/37] initial commit. --- process/interceptors/baseDataInterceptor.go | 24 +++++++++++++++++++ process/interceptors/multiDataInterceptor.go | 15 +++++++++++- .../interceptors/multiDataInterceptor_test.go | 8 +++++-- process/interceptors/singleDataInterceptor.go | 16 ++++++++++++- .../singleDataInterceptor_test.go | 2 ++ 5 files changed, 61 insertions(+), 4 deletions(-) diff --git a/process/interceptors/baseDataInterceptor.go b/process/interceptors/baseDataInterceptor.go index 64efb852238..52b23a14618 100644 --- a/process/interceptors/baseDataInterceptor.go +++ b/process/interceptors/baseDataInterceptor.go @@ -2,12 +2,15 @@ package interceptors import ( "bytes" + "fmt" "sync" "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/multiversx/mx-chain-go/p2p" "github.com/multiversx/mx-chain-go/process" + "github.com/multiversx/mx-chain-go/storage/cache" ) type baseDataInterceptor struct { @@ -19,6 +22,8 @@ type baseDataInterceptor struct { mutDebugHandler sync.RWMutex debugHandler process.InterceptedDebugger preferredPeersHolder process.PreferredPeersHolderHandler + + timeCache *cache.TimeCache } func (bdi *baseDataInterceptor) preProcessMesage(message p2p.MessageP2P, fromConnectedPeer core.PeerID) error { @@ -119,6 +124,25 @@ func (bdi *baseDataInterceptor) receivedDebugInterceptedData(interceptedData pro bdi.mutDebugHandler.RUnlock() } +func (bdi *baseDataInterceptor) checkIfMessageHasBeenProcessed(interceptedData process.InterceptedData) error { + hash := string(interceptedData.Hash()) + + if hash == "" { + return nil + } + + if bdi.timeCache.Has(hash) { + return fmt.Errorf("intercepted data has already been processed") + } + + err := bdi.timeCache.Add(hash) + if err != nil { + return fmt.Errorf("intercepted data could not be added to the cache") + } + + return nil +} + // SetInterceptedDebugHandler will set a new intercepted debug handler func (bdi *baseDataInterceptor) SetInterceptedDebugHandler(handler process.InterceptedDebugger) error { if check.IfNil(handler) { diff --git a/process/interceptors/multiDataInterceptor.go b/process/interceptors/multiDataInterceptor.go index 9e0197ea741..37df640fadd 100644 --- a/process/interceptors/multiDataInterceptor.go +++ b/process/interceptors/multiDataInterceptor.go @@ -2,17 +2,20 @@ package interceptors import ( "sync" + "time" "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data/batch" "github.com/multiversx/mx-chain-core-go/marshal" + logger "github.com/multiversx/mx-chain-logger-go" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/debug/handler" "github.com/multiversx/mx-chain-go/p2p" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/interceptors/disabled" - logger "github.com/multiversx/mx-chain-logger-go" + "github.com/multiversx/mx-chain-go/storage/cache" ) var log = logger.GetOrCreate("process/interceptors") @@ -79,6 +82,7 @@ func NewMultiDataInterceptor(arg ArgMultiDataInterceptor) (*MultiDataInterceptor processor: arg.Processor, preferredPeersHolder: arg.PreferredPeersHolder, debugHandler: handler.NewDisabledInterceptorDebugHandler(), + timeCache: cache.NewTimeCache(30 * time.Second), }, marshalizer: arg.Marshalizer, factory: arg.DataFactory, @@ -152,6 +156,11 @@ func (mdi *MultiDataInterceptor) ProcessReceivedMessage(message p2p.MessageP2P, for index, dataBuff := range multiDataBuff { var interceptedData process.InterceptedData interceptedData, err = mdi.interceptedData(dataBuff, message.Peer(), fromConnectedPeer) + + errCache := mdi.checkIfMessageHasBeenProcessed(interceptedData) + if errCache != nil { + continue + } listInterceptedData[index] = interceptedData if err != nil { mdi.throttler.EndProcessing() @@ -206,6 +215,10 @@ func (mdi *MultiDataInterceptor) interceptedData(dataBuff []byte, originator cor } mdi.receivedDebugInterceptedData(interceptedData) + //shouldProcess := mdi.checkIfMessageHasBeenProcessed(interceptedData) + //if !shouldProcess { + // return nil, nil + //} err = interceptedData.CheckValidity() if err != nil { diff --git a/process/interceptors/multiDataInterceptor_test.go b/process/interceptors/multiDataInterceptor_test.go index 6ca244409b7..45c8b307b17 100644 --- a/process/interceptors/multiDataInterceptor_test.go +++ b/process/interceptors/multiDataInterceptor_test.go @@ -10,13 +10,14 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data/batch" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/interceptors" "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/testscommon" "github.com/multiversx/mx-chain-go/testscommon/p2pmocks" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) var fromConnectedPeerId = core.PeerID("from connected peer Id") @@ -570,6 +571,9 @@ func processReceivedMessageMultiDataInvalidVersion(t *testing.T, expectedErr err checkCalledNum := int32(0) processCalledNum := int32(0) interceptedData := &testscommon.InterceptedDataStub{ + HashCalled: func() []byte { + return []byte("hash") + }, CheckValidityCalled: func() error { return expectedErr }, diff --git a/process/interceptors/singleDataInterceptor.go b/process/interceptors/singleDataInterceptor.go index 84f3296acd7..14eb0ec6d16 100644 --- a/process/interceptors/singleDataInterceptor.go +++ b/process/interceptors/singleDataInterceptor.go @@ -1,12 +1,17 @@ package interceptors import ( + "errors" + "time" + "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/debug/handler" "github.com/multiversx/mx-chain-go/p2p" "github.com/multiversx/mx-chain-go/process" + "github.com/multiversx/mx-chain-go/storage/cache" ) // ArgSingleDataInterceptor is the argument for the single-data interceptor @@ -64,6 +69,7 @@ func NewSingleDataInterceptor(arg ArgSingleDataInterceptor) (*SingleDataIntercep processor: arg.Processor, preferredPeersHolder: arg.PreferredPeersHolder, debugHandler: handler.NewDisabledInterceptorDebugHandler(), + timeCache: cache.NewTimeCache(30 * time.Second), }, factory: arg.DataFactory, whiteListRequest: arg.WhiteListRequest, @@ -75,6 +81,9 @@ func NewSingleDataInterceptor(arg ArgSingleDataInterceptor) (*SingleDataIntercep // ProcessReceivedMessage is the callback func from the p2p.Messenger and will be called each time a new message was received // (for the topic this validator was registered to) func (sdi *SingleDataInterceptor) ProcessReceivedMessage(message p2p.MessageP2P, fromConnectedPeer core.PeerID, _ p2p.MessageHandler) error { + // Sweep the time cache before processing the message + sdi.timeCache.Sweep() + err := sdi.preProcessMesage(message, fromConnectedPeer) if err != nil { return err @@ -93,13 +102,18 @@ func (sdi *SingleDataInterceptor) ProcessReceivedMessage(message p2p.MessageP2P, } sdi.receivedDebugInterceptedData(interceptedData) + err = sdi.checkIfMessageHasBeenProcessed(interceptedData) + if err != nil { + sdi.throttler.EndProcessing() + return err + } err = interceptedData.CheckValidity() if err != nil { sdi.throttler.EndProcessing() sdi.processDebugInterceptedData(interceptedData, err) - isWrongVersion := err == process.ErrInvalidTransactionVersion || err == process.ErrInvalidChainID + isWrongVersion := errors.Is(err, process.ErrInvalidTransactionVersion) || errors.Is(err, process.ErrInvalidChainID) if isWrongVersion { // this situation is so severe that we need to black list de peers reason := "wrong version of received intercepted data, topic " + sdi.topic + ", error " + err.Error() diff --git a/process/interceptors/singleDataInterceptor_test.go b/process/interceptors/singleDataInterceptor_test.go index 515c2a8724c..759d4541e97 100644 --- a/process/interceptors/singleDataInterceptor_test.go +++ b/process/interceptors/singleDataInterceptor_test.go @@ -2,6 +2,8 @@ package interceptors_test import ( "errors" + "fmt" + "sync" "sync/atomic" "testing" "time" From aaa9d4bebacf1d338e4482d98083714b55861ce4 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Fri, 13 Sep 2024 13:41:07 +0300 Subject: [PATCH 02/37] added test for processing message. --- go.mod | 2 +- process/interceptors/baseDataInterceptor.go | 4 +- .../singleDataInterceptor_test.go | 68 ++++++++++++++++++- 3 files changed, 68 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 3c5c1af9488..ac1e8c26364 100644 --- a/go.mod +++ b/go.mod @@ -33,6 +33,7 @@ require ( github.com/stretchr/testify v1.8.4 github.com/urfave/cli v1.22.10 golang.org/x/crypto v0.10.0 + golang.org/x/exp v0.0.0-20230321023759-10a507213a29 gopkg.in/go-playground/validator.v8 v8.18.2 ) @@ -173,7 +174,6 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.24.0 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect golang.org/x/mod v0.10.0 // indirect golang.org/x/net v0.11.0 // indirect golang.org/x/sync v0.2.0 // indirect diff --git a/process/interceptors/baseDataInterceptor.go b/process/interceptors/baseDataInterceptor.go index 52b23a14618..8361a724a96 100644 --- a/process/interceptors/baseDataInterceptor.go +++ b/process/interceptors/baseDataInterceptor.go @@ -132,12 +132,12 @@ func (bdi *baseDataInterceptor) checkIfMessageHasBeenProcessed(interceptedData p } if bdi.timeCache.Has(hash) { - return fmt.Errorf("intercepted data has already been processed") + return fmt.Errorf("processed intercepted data with hash: %s", hash) } err := bdi.timeCache.Add(hash) if err != nil { - return fmt.Errorf("intercepted data could not be added to the cache") + return fmt.Errorf("failed to add to time cache intercepted data with hash: %s", hash) } return nil diff --git a/process/interceptors/singleDataInterceptor_test.go b/process/interceptors/singleDataInterceptor_test.go index 759d4541e97..df49ecf1e9a 100644 --- a/process/interceptors/singleDataInterceptor_test.go +++ b/process/interceptors/singleDataInterceptor_test.go @@ -2,7 +2,7 @@ package interceptors_test import ( "errors" - "fmt" + "strings" "sync" "sync/atomic" "testing" @@ -10,13 +10,14 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/interceptors" "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/testscommon" "github.com/multiversx/mx-chain-go/testscommon/p2pmocks" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func createMockArgSingleDataInterceptor() interceptors.ArgSingleDataInterceptor { @@ -470,6 +471,67 @@ func TestSingleDataInterceptor_Close(t *testing.T) { assert.Nil(t, err) } +func TestSingleDataInterceptor_ProcessSameMessage(t *testing.T) { + t.Parallel() + + checkCalledNum := int32(0) + processCalledNum := int32(0) + throttler := createMockThrottler() + interceptedData := &testscommon.InterceptedDataStub{ + HashCalled: func() []byte { + return []byte("hash") + }, + CheckValidityCalled: func() error { + return nil + }, + IsForCurrentShardCalled: func() bool { + return false + }, + } + + whiteListHandler := &testscommon.WhiteListHandlerStub{ + IsWhiteListedCalled: func(interceptedData process.InterceptedData) bool { + return true + }, + } + arg := createMockArgSingleDataInterceptor() + arg.DataFactory = &mock.InterceptedDataFactoryStub{ + CreateCalled: func(buff []byte) (data process.InterceptedData, e error) { + return interceptedData, nil + }, + } + arg.Processor = createMockInterceptorStub(&checkCalledNum, &processCalledNum) + arg.Throttler = throttler + arg.AntifloodHandler = &mock.P2PAntifloodHandlerStub{ + IsOriginatorEligibleForTopicCalled: func(pid core.PeerID, topic string) error { + return process.ErrOnlyValidatorsCanUseThisTopic + }, + } + arg.WhiteListRequest = whiteListHandler + sdi, _ := interceptors.NewSingleDataInterceptor(arg) + + msg := &p2pmocks.P2PMessageMock{ + DataField: []byte("data to be processed"), + } + + wg := sync.WaitGroup{} + errCount := atomic.Uint32{} + for i := 0; i < 3; i++ { + wg.Add(1) + go func() { + defer wg.Done() + + err := sdi.ProcessReceivedMessage(msg, fromConnectedPeerId, &p2pmocks.MessengerStub{}) + if err != nil && strings.Contains(err.Error(), "has already been processed") { + errCount.Add(1) + } + }() + } + + wg.Wait() + require.Equal(t, uint32(2), errCount.Load()) +} + //------- IsInterfaceNil func TestSingleDataInterceptor_IsInterfaceNil(t *testing.T) { From 603bd691a75b3124dedd5fa9cd3630c081aefdea Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Mon, 16 Sep 2024 15:18:57 +0300 Subject: [PATCH 03/37] added message validation cache. --- factory/processing/processComponents.go | 8 + process/errors.go | 3 + process/factory/interceptorscontainer/args.go | 4 + .../baseInterceptorsContainerFactory.go | 274 ++++++++++++------ .../metaInterceptorsContainerFactory.go | 19 +- .../shardInterceptorsContainerFactory.go | 2 + process/interceptors/baseDataInterceptor.go | 40 +-- process/interceptors/multiDataInterceptor.go | 58 ++-- .../interceptors/multiDataInterceptor_test.go | 21 +- process/interceptors/singleDataInterceptor.go | 42 +-- .../singleDataInterceptor_test.go | 33 ++- sharding/nodesSetup.go | 3 + 12 files changed, 318 insertions(+), 189 deletions(-) diff --git a/factory/processing/processComponents.go b/factory/processing/processComponents.go index 0376a7235ce..1c8d2e00de5 100644 --- a/factory/processing/processComponents.go +++ b/factory/processing/processComponents.go @@ -1501,6 +1501,8 @@ func (pcf *processComponentsFactory) newInterceptorContainerFactory( nodeOperationMode = common.FullArchiveMode } + processedMessagesCacheMap := make(map[string]storage.Cacher) + shardCoordinator := pcf.bootstrapComponents.ShardCoordinator() if shardCoordinator.SelfId() < shardCoordinator.NumberOfShards() { return pcf.newShardInterceptorContainerFactory( @@ -1513,6 +1515,7 @@ func (pcf *processComponentsFactory) newInterceptorContainerFactory( fullArchivePeerShardMapper, hardforkTrigger, nodeOperationMode, + processedMessagesCacheMap, ) } if shardCoordinator.SelfId() == core.MetachainShardId { @@ -1526,6 +1529,7 @@ func (pcf *processComponentsFactory) newInterceptorContainerFactory( fullArchivePeerShardMapper, hardforkTrigger, nodeOperationMode, + processedMessagesCacheMap, ) } @@ -1665,6 +1669,7 @@ func (pcf *processComponentsFactory) newShardInterceptorContainerFactory( fullArchivePeerShardMapper *networksharding.PeerShardMapper, hardforkTrigger factory.HardforkTrigger, nodeOperationMode common.NodeOperation, + processedMessagesCacheMap map[string]storage.Cacher, ) (process.InterceptorsContainerFactory, process.TimeCacher, error) { headerBlackList := cache.NewTimeCache(timeSpanForBadHeaders) shardInterceptorsContainerFactoryArgs := interceptorscontainer.CommonInterceptorsContainerFactoryArgs{ @@ -1698,6 +1703,7 @@ func (pcf *processComponentsFactory) newShardInterceptorContainerFactory( FullArchivePeerShardMapper: fullArchivePeerShardMapper, HardforkTrigger: hardforkTrigger, NodeOperationMode: nodeOperationMode, + ProcessedMessagesCacheMap: processedMessagesCacheMap, } interceptorContainerFactory, err := interceptorscontainer.NewShardInterceptorsContainerFactory(shardInterceptorsContainerFactoryArgs) @@ -1718,6 +1724,7 @@ func (pcf *processComponentsFactory) newMetaInterceptorContainerFactory( fullArchivePeerShardMapper *networksharding.PeerShardMapper, hardforkTrigger factory.HardforkTrigger, nodeOperationMode common.NodeOperation, + processedMessageCacheMap map[string]storage.Cacher, ) (process.InterceptorsContainerFactory, process.TimeCacher, error) { headerBlackList := cache.NewTimeCache(timeSpanForBadHeaders) metaInterceptorsContainerFactoryArgs := interceptorscontainer.CommonInterceptorsContainerFactoryArgs{ @@ -1751,6 +1758,7 @@ func (pcf *processComponentsFactory) newMetaInterceptorContainerFactory( FullArchivePeerShardMapper: fullArchivePeerShardMapper, HardforkTrigger: hardforkTrigger, NodeOperationMode: nodeOperationMode, + ProcessedMessagesCacheMap: processedMessageCacheMap, } interceptorContainerFactory, err := interceptorscontainer.NewMetaInterceptorsContainerFactory(metaInterceptorsContainerFactoryArgs) diff --git a/process/errors.go b/process/errors.go index f1e17a78f55..6268f38fc59 100644 --- a/process/errors.go +++ b/process/errors.go @@ -942,6 +942,9 @@ var ErrNilEpochStartSystemSCProcessor = errors.New("nil epoch start system sc pr // ErrEmptyPeerID signals that an empty peer ID has been provided var ErrEmptyPeerID = errors.New("empty peer ID") +// ErrNilProcessedMessagesCacheMap signals that an empty cache map has been provided +var ErrNilProcessedMessagesCacheMap = errors.New("empty processed messages cache map") + // ErrNilFallbackHeaderValidator signals that a nil fallback header validator has been provided var ErrNilFallbackHeaderValidator = errors.New("nil fallback header validator") diff --git a/process/factory/interceptorscontainer/args.go b/process/factory/interceptorscontainer/args.go index 294e66290b3..f490a13a54c 100644 --- a/process/factory/interceptorscontainer/args.go +++ b/process/factory/interceptorscontainer/args.go @@ -2,6 +2,7 @@ package interceptorscontainer import ( crypto "github.com/multiversx/mx-chain-crypto-go" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/heartbeat" @@ -9,6 +10,7 @@ import ( "github.com/multiversx/mx-chain-go/sharding" "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator" "github.com/multiversx/mx-chain-go/state" + "github.com/multiversx/mx-chain-go/storage" ) // CommonInterceptorsContainerFactoryArgs holds the arguments needed for the metachain/shard interceptors factories @@ -43,4 +45,6 @@ type CommonInterceptorsContainerFactoryArgs struct { FullArchivePeerShardMapper process.PeerShardMapper HardforkTrigger heartbeat.HardforkTrigger NodeOperationMode common.NodeOperation + ProcessedMessagesCache storage.Cacher + ProcessedMessagesCacheMap map[string]storage.Cacher } diff --git a/process/factory/interceptorscontainer/baseInterceptorsContainerFactory.go b/process/factory/interceptorscontainer/baseInterceptorsContainerFactory.go index cfed22b39c9..999e9e51487 100644 --- a/process/factory/interceptorscontainer/baseInterceptorsContainerFactory.go +++ b/process/factory/interceptorscontainer/baseInterceptorsContainerFactory.go @@ -7,6 +7,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/hashing" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/heartbeat" @@ -20,6 +21,7 @@ import ( "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator" "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/storage" + "github.com/multiversx/mx-chain-go/storage/cache" ) const ( @@ -28,6 +30,8 @@ const ( minTimespanDurationInSec = int64(1) errorOnMainNetworkString = "on main network" errorOnFullArchiveNetworkString = "on full archive network" + cacheDefaultSpan = 30 * time.Second + cacheDefaultExpiry = 30 * time.Second ) type baseInterceptorsContainerFactory struct { @@ -54,6 +58,7 @@ type baseInterceptorsContainerFactory struct { fullArchivePeerShardMapper process.PeerShardMapper hardforkTrigger heartbeat.HardforkTrigger nodeOperationMode common.NodeOperation + processedMessagesCacheMap map[string]storage.Cacher } func checkBaseParams( @@ -285,18 +290,24 @@ func (bicf *baseInterceptorsContainerFactory) createOneTxInterceptor(topic strin return nil, err } + err = bicf.createCacheForInterceptor(topic) + if err != nil { + return nil, err + } + internalMarshaller := bicf.argInterceptorFactory.CoreComponents.InternalMarshalizer() interceptor, err := interceptors.NewMultiDataInterceptor( interceptors.ArgMultiDataInterceptor{ - Topic: topic, - Marshalizer: internalMarshaller, - DataFactory: txFactory, - Processor: txProcessor, - Throttler: bicf.globalThrottler, - AntifloodHandler: bicf.antifloodHandler, - WhiteListRequest: bicf.whiteListHandler, - CurrentPeerId: bicf.mainMessenger.ID(), - PreferredPeersHolder: bicf.preferredPeersHolder, + Topic: topic, + Marshalizer: internalMarshaller, + DataFactory: txFactory, + Processor: txProcessor, + Throttler: bicf.globalThrottler, + AntifloodHandler: bicf.antifloodHandler, + WhiteListRequest: bicf.whiteListHandler, + CurrentPeerId: bicf.mainMessenger.ID(), + PreferredPeersHolder: bicf.preferredPeersHolder, + ProcessedMessagesCacheMap: bicf.processedMessagesCacheMap, }, ) if err != nil { @@ -328,18 +339,24 @@ func (bicf *baseInterceptorsContainerFactory) createOneUnsignedTxInterceptor(top return nil, err } + err = bicf.createCacheForInterceptor(topic) + if err != nil { + return nil, err + } + internalMarshaller := bicf.argInterceptorFactory.CoreComponents.InternalMarshalizer() interceptor, err := interceptors.NewMultiDataInterceptor( interceptors.ArgMultiDataInterceptor{ - Topic: topic, - Marshalizer: internalMarshaller, - DataFactory: txFactory, - Processor: txProcessor, - Throttler: bicf.globalThrottler, - AntifloodHandler: bicf.antifloodHandler, - WhiteListRequest: bicf.whiteListHandler, - CurrentPeerId: bicf.mainMessenger.ID(), - PreferredPeersHolder: bicf.preferredPeersHolder, + Topic: topic, + Marshalizer: internalMarshaller, + DataFactory: txFactory, + Processor: txProcessor, + Throttler: bicf.globalThrottler, + AntifloodHandler: bicf.antifloodHandler, + WhiteListRequest: bicf.whiteListHandler, + CurrentPeerId: bicf.mainMessenger.ID(), + PreferredPeersHolder: bicf.preferredPeersHolder, + ProcessedMessagesCacheMap: bicf.processedMessagesCacheMap, }, ) if err != nil { @@ -371,18 +388,24 @@ func (bicf *baseInterceptorsContainerFactory) createOneRewardTxInterceptor(topic return nil, err } + err = bicf.createCacheForInterceptor(topic) + if err != nil { + return nil, err + } + internalMarshaller := bicf.argInterceptorFactory.CoreComponents.InternalMarshalizer() interceptor, err := interceptors.NewMultiDataInterceptor( interceptors.ArgMultiDataInterceptor{ - Topic: topic, - Marshalizer: internalMarshaller, - DataFactory: txFactory, - Processor: txProcessor, - Throttler: bicf.globalThrottler, - AntifloodHandler: bicf.antifloodHandler, - WhiteListRequest: bicf.whiteListHandler, - CurrentPeerId: bicf.mainMessenger.ID(), - PreferredPeersHolder: bicf.preferredPeersHolder, + Topic: topic, + Marshalizer: internalMarshaller, + DataFactory: txFactory, + Processor: txProcessor, + Throttler: bicf.globalThrottler, + AntifloodHandler: bicf.antifloodHandler, + WhiteListRequest: bicf.whiteListHandler, + CurrentPeerId: bicf.mainMessenger.ID(), + PreferredPeersHolder: bicf.preferredPeersHolder, + ProcessedMessagesCacheMap: bicf.processedMessagesCacheMap, }, ) if err != nil { @@ -414,17 +437,23 @@ func (bicf *baseInterceptorsContainerFactory) generateHeaderInterceptors() error // compose header shard topic, for example: shardBlocks_0_META identifierHdr := factory.ShardBlocksTopic + shardC.CommunicationIdentifier(core.MetachainShardId) + err = bicf.createCacheForInterceptor(identifierHdr) + if err != nil { + return err + } + // only one intrashard header topic interceptor, err := interceptors.NewSingleDataInterceptor( interceptors.ArgSingleDataInterceptor{ - Topic: identifierHdr, - DataFactory: hdrFactory, - Processor: hdrProcessor, - Throttler: bicf.globalThrottler, - AntifloodHandler: bicf.antifloodHandler, - WhiteListRequest: bicf.whiteListHandler, - CurrentPeerId: bicf.mainMessenger.ID(), - PreferredPeersHolder: bicf.preferredPeersHolder, + Topic: identifierHdr, + DataFactory: hdrFactory, + Processor: hdrProcessor, + Throttler: bicf.globalThrottler, + AntifloodHandler: bicf.antifloodHandler, + WhiteListRequest: bicf.whiteListHandler, + CurrentPeerId: bicf.mainMessenger.ID(), + PreferredPeersHolder: bicf.preferredPeersHolder, + ProcessedMessagesCacheMap: bicf.processedMessagesCacheMap, }, ) if err != nil { @@ -502,17 +531,23 @@ func (bicf *baseInterceptorsContainerFactory) createOneMiniBlocksInterceptor(top return nil, err } + err = bicf.createCacheForInterceptor(topic) + if err != nil { + return nil, err + } + interceptor, err := interceptors.NewMultiDataInterceptor( interceptors.ArgMultiDataInterceptor{ - Topic: topic, - Marshalizer: internalMarshaller, - DataFactory: miniblockFactory, - Processor: miniblockProcessor, - Throttler: bicf.globalThrottler, - AntifloodHandler: bicf.antifloodHandler, - WhiteListRequest: bicf.whiteListHandler, - CurrentPeerId: bicf.mainMessenger.ID(), - PreferredPeersHolder: bicf.preferredPeersHolder, + Topic: topic, + Marshalizer: internalMarshaller, + DataFactory: miniblockFactory, + Processor: miniblockProcessor, + Throttler: bicf.globalThrottler, + AntifloodHandler: bicf.antifloodHandler, + WhiteListRequest: bicf.whiteListHandler, + CurrentPeerId: bicf.mainMessenger.ID(), + PreferredPeersHolder: bicf.preferredPeersHolder, + ProcessedMessagesCacheMap: bicf.processedMessagesCacheMap, }, ) if err != nil { @@ -541,17 +576,23 @@ func (bicf *baseInterceptorsContainerFactory) generateMetachainHeaderInterceptor return err } + err = bicf.createCacheForInterceptor(identifierHdr) + if err != nil { + return err + } + // only one metachain header topic interceptor, err := interceptors.NewSingleDataInterceptor( interceptors.ArgSingleDataInterceptor{ - Topic: identifierHdr, - DataFactory: hdrFactory, - Processor: hdrProcessor, - Throttler: bicf.globalThrottler, - AntifloodHandler: bicf.antifloodHandler, - WhiteListRequest: bicf.whiteListHandler, - CurrentPeerId: bicf.mainMessenger.ID(), - PreferredPeersHolder: bicf.preferredPeersHolder, + Topic: identifierHdr, + DataFactory: hdrFactory, + Processor: hdrProcessor, + Throttler: bicf.globalThrottler, + AntifloodHandler: bicf.antifloodHandler, + WhiteListRequest: bicf.whiteListHandler, + CurrentPeerId: bicf.mainMessenger.ID(), + PreferredPeersHolder: bicf.preferredPeersHolder, + ProcessedMessagesCacheMap: bicf.processedMessagesCacheMap, }, ) if err != nil { @@ -577,18 +618,24 @@ func (bicf *baseInterceptorsContainerFactory) createOneTrieNodesInterceptor(topi return nil, err } + err = bicf.createCacheForInterceptor(topic) + if err != nil { + return nil, err + } + internalMarshaller := bicf.argInterceptorFactory.CoreComponents.InternalMarshalizer() interceptor, err := interceptors.NewMultiDataInterceptor( interceptors.ArgMultiDataInterceptor{ - Topic: topic, - Marshalizer: internalMarshaller, - DataFactory: trieNodesFactory, - Processor: trieNodesProcessor, - Throttler: bicf.globalThrottler, - AntifloodHandler: bicf.antifloodHandler, - WhiteListRequest: bicf.whiteListHandler, - CurrentPeerId: bicf.mainMessenger.ID(), - PreferredPeersHolder: bicf.preferredPeersHolder, + Topic: topic, + Marshalizer: internalMarshaller, + DataFactory: trieNodesFactory, + Processor: trieNodesProcessor, + Throttler: bicf.globalThrottler, + AntifloodHandler: bicf.antifloodHandler, + WhiteListRequest: bicf.whiteListHandler, + CurrentPeerId: bicf.mainMessenger.ID(), + PreferredPeersHolder: bicf.preferredPeersHolder, + ProcessedMessagesCacheMap: bicf.processedMessagesCacheMap, }, ) if err != nil { @@ -669,17 +716,23 @@ func (bicf *baseInterceptorsContainerFactory) generatePeerAuthenticationIntercep return err } + err = bicf.createCacheForInterceptor(identifierPeerAuthentication) + if err != nil { + return err + } + mdInterceptor, err := interceptors.NewMultiDataInterceptor( interceptors.ArgMultiDataInterceptor{ - Topic: identifierPeerAuthentication, - Marshalizer: internalMarshaller, - DataFactory: peerAuthenticationFactory, - Processor: peerAuthenticationProcessor, - Throttler: bicf.globalThrottler, - AntifloodHandler: bicf.antifloodHandler, - WhiteListRequest: bicf.whiteListHandler, - PreferredPeersHolder: bicf.preferredPeersHolder, - CurrentPeerId: bicf.mainMessenger.ID(), + Topic: identifierPeerAuthentication, + Marshalizer: internalMarshaller, + DataFactory: peerAuthenticationFactory, + Processor: peerAuthenticationProcessor, + Throttler: bicf.globalThrottler, + AntifloodHandler: bicf.antifloodHandler, + WhiteListRequest: bicf.whiteListHandler, + PreferredPeersHolder: bicf.preferredPeersHolder, + CurrentPeerId: bicf.mainMessenger.ID(), + ProcessedMessagesCacheMap: bicf.processedMessagesCacheMap, }, ) if err != nil { @@ -728,16 +781,22 @@ func (bicf *baseInterceptorsContainerFactory) createHeartbeatV2Interceptor( return nil, err } + err = bicf.createCacheForInterceptor(identifier) + if err != nil { + return nil, err + } + interceptor, err := interceptors.NewSingleDataInterceptor( interceptors.ArgSingleDataInterceptor{ - Topic: identifier, - DataFactory: heartbeatFactory, - Processor: heartbeatProcessor, - Throttler: bicf.globalThrottler, - AntifloodHandler: bicf.antifloodHandler, - WhiteListRequest: bicf.whiteListHandler, - PreferredPeersHolder: bicf.preferredPeersHolder, - CurrentPeerId: bicf.mainMessenger.ID(), + Topic: identifier, + DataFactory: heartbeatFactory, + Processor: heartbeatProcessor, + Throttler: bicf.globalThrottler, + AntifloodHandler: bicf.antifloodHandler, + WhiteListRequest: bicf.whiteListHandler, + PreferredPeersHolder: bicf.preferredPeersHolder, + CurrentPeerId: bicf.mainMessenger.ID(), + ProcessedMessagesCacheMap: bicf.processedMessagesCacheMap, }, ) if err != nil { @@ -777,16 +836,22 @@ func (bicf *baseInterceptorsContainerFactory) createPeerShardInterceptor( return nil, err } + err = bicf.createCacheForInterceptor(identifier) + if err != nil { + return nil, err + } + interceptor, err := interceptors.NewSingleDataInterceptor( interceptors.ArgSingleDataInterceptor{ - Topic: identifier, - DataFactory: interceptedPeerShardFactory, - Processor: psiProcessor, - Throttler: bicf.globalThrottler, - AntifloodHandler: bicf.antifloodHandler, - WhiteListRequest: bicf.whiteListHandler, - CurrentPeerId: bicf.mainMessenger.ID(), - PreferredPeersHolder: bicf.preferredPeersHolder, + Topic: identifier, + DataFactory: interceptedPeerShardFactory, + Processor: psiProcessor, + Throttler: bicf.globalThrottler, + AntifloodHandler: bicf.antifloodHandler, + WhiteListRequest: bicf.whiteListHandler, + CurrentPeerId: bicf.mainMessenger.ID(), + PreferredPeersHolder: bicf.preferredPeersHolder, + ProcessedMessagesCacheMap: bicf.processedMessagesCacheMap, }, ) if err != nil { @@ -814,17 +879,23 @@ func (bicf *baseInterceptorsContainerFactory) generateValidatorInfoInterceptor() return err } + err = bicf.createCacheForInterceptor(identifier) + if err != nil { + return err + } + mdInterceptor, err := interceptors.NewMultiDataInterceptor( interceptors.ArgMultiDataInterceptor{ - Topic: identifier, - Marshalizer: internalMarshaller, - DataFactory: interceptedValidatorInfoFactory, - Processor: validatorInfoProcessor, - Throttler: bicf.globalThrottler, - AntifloodHandler: bicf.antifloodHandler, - WhiteListRequest: bicf.whiteListHandler, - PreferredPeersHolder: bicf.preferredPeersHolder, - CurrentPeerId: bicf.mainMessenger.ID(), + Topic: identifier, + Marshalizer: internalMarshaller, + DataFactory: interceptedValidatorInfoFactory, + Processor: validatorInfoProcessor, + Throttler: bicf.globalThrottler, + AntifloodHandler: bicf.antifloodHandler, + WhiteListRequest: bicf.whiteListHandler, + PreferredPeersHolder: bicf.preferredPeersHolder, + CurrentPeerId: bicf.mainMessenger.ID(), + ProcessedMessagesCacheMap: bicf.processedMessagesCacheMap, }, ) if err != nil { @@ -851,3 +922,16 @@ func (bicf *baseInterceptorsContainerFactory) addInterceptorsToContainers(keys [ return bicf.fullArchiveContainer.AddMultiple(keys, interceptors) } + +func (bicf *baseInterceptorsContainerFactory) createCacheForInterceptor(topic string) error { + internalCache, err := cache.NewTimeCacher(cache.ArgTimeCacher{ + DefaultSpan: cacheDefaultSpan, + CacheExpiry: cacheDefaultExpiry, + }) + if err != nil { + return err + } + + bicf.processedMessagesCacheMap[topic] = internalCache + return nil +} diff --git a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go index 38d3e460bce..5f1abbaa4b9 100644 --- a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go +++ b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go @@ -5,6 +5,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/core/throttler" "github.com/multiversx/mx-chain-core-go/marshal" + "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/factory" "github.com/multiversx/mx-chain-go/process/factory/containers" @@ -124,6 +125,7 @@ func NewMetaInterceptorsContainerFactory( fullArchivePeerShardMapper: args.FullArchivePeerShardMapper, hardforkTrigger: args.HardforkTrigger, nodeOperationMode: args.NodeOperationMode, + processedMessagesCacheMap: args.ProcessedMessagesCacheMap, } icf := &metaInterceptorsContainerFactory{ @@ -263,14 +265,15 @@ func (micf *metaInterceptorsContainerFactory) createOneShardHeaderInterceptor(to interceptor, err := processInterceptors.NewSingleDataInterceptor( processInterceptors.ArgSingleDataInterceptor{ - Topic: topic, - DataFactory: hdrFactory, - Processor: hdrProcessor, - Throttler: micf.globalThrottler, - AntifloodHandler: micf.antifloodHandler, - WhiteListRequest: micf.whiteListHandler, - CurrentPeerId: micf.mainMessenger.ID(), - PreferredPeersHolder: micf.preferredPeersHolder, + Topic: topic, + DataFactory: hdrFactory, + Processor: hdrProcessor, + Throttler: micf.globalThrottler, + AntifloodHandler: micf.antifloodHandler, + WhiteListRequest: micf.whiteListHandler, + CurrentPeerId: micf.mainMessenger.ID(), + PreferredPeersHolder: micf.preferredPeersHolder, + ProcessedMessagesCacheMap: micf.processedMessagesCacheMap, }, ) if err != nil { diff --git a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go index beef288c54c..77e3469c917 100644 --- a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go +++ b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go @@ -5,6 +5,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/core/throttler" "github.com/multiversx/mx-chain-core-go/marshal" + "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/factory" "github.com/multiversx/mx-chain-go/process/factory/containers" @@ -123,6 +124,7 @@ func NewShardInterceptorsContainerFactory( fullArchivePeerShardMapper: args.FullArchivePeerShardMapper, hardforkTrigger: args.HardforkTrigger, nodeOperationMode: args.NodeOperationMode, + processedMessagesCacheMap: args.ProcessedMessagesCacheMap, } icf := &shardInterceptorsContainerFactory{ diff --git a/process/interceptors/baseDataInterceptor.go b/process/interceptors/baseDataInterceptor.go index 8361a724a96..de679963988 100644 --- a/process/interceptors/baseDataInterceptor.go +++ b/process/interceptors/baseDataInterceptor.go @@ -10,20 +10,19 @@ import ( "github.com/multiversx/mx-chain-go/p2p" "github.com/multiversx/mx-chain-go/process" - "github.com/multiversx/mx-chain-go/storage/cache" + "github.com/multiversx/mx-chain-go/storage" ) type baseDataInterceptor struct { - throttler process.InterceptorThrottler - antifloodHandler process.P2PAntifloodHandler - topic string - currentPeerId core.PeerID - processor process.InterceptorProcessor - mutDebugHandler sync.RWMutex - debugHandler process.InterceptedDebugger - preferredPeersHolder process.PreferredPeersHolderHandler - - timeCache *cache.TimeCache + throttler process.InterceptorThrottler + antifloodHandler process.P2PAntifloodHandler + topic string + currentPeerId core.PeerID + processor process.InterceptorProcessor + mutDebugHandler sync.RWMutex + debugHandler process.InterceptedDebugger + preferredPeersHolder process.PreferredPeersHolderHandler + processedMessagesCacheMap map[string]storage.Cacher } func (bdi *baseDataInterceptor) preProcessMesage(message p2p.MessageP2P, fromConnectedPeer core.PeerID) error { @@ -125,19 +124,22 @@ func (bdi *baseDataInterceptor) receivedDebugInterceptedData(interceptedData pro } func (bdi *baseDataInterceptor) checkIfMessageHasBeenProcessed(interceptedData process.InterceptedData) error { - hash := string(interceptedData.Hash()) - - if hash == "" { + if len(interceptedData.Hash()) == 0 { return nil } - if bdi.timeCache.Has(hash) { - return fmt.Errorf("processed intercepted data with hash: %s", hash) + c, ok := bdi.processedMessagesCacheMap[bdi.topic] + if !ok { + return fmt.Errorf("cache for topic %q does not exist", bdi.topic) } - err := bdi.timeCache.Add(hash) - if err != nil { - return fmt.Errorf("failed to add to time cache intercepted data with hash: %s", hash) + cache, ok := c.(storage.Cacher) + if !ok { + return fmt.Errorf("failed to cast cacher") + } + + if has, _ := cache.HasOrAdd(interceptedData.Hash(), nil, 0); has { + return fmt.Errorf("processed intercepted data with hash: %s", interceptedData.Hash()) } return nil diff --git a/process/interceptors/multiDataInterceptor.go b/process/interceptors/multiDataInterceptor.go index 37df640fadd..db80bdd7be8 100644 --- a/process/interceptors/multiDataInterceptor.go +++ b/process/interceptors/multiDataInterceptor.go @@ -2,35 +2,36 @@ package interceptors import ( "sync" - "time" "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data/batch" "github.com/multiversx/mx-chain-core-go/marshal" logger "github.com/multiversx/mx-chain-logger-go" + "github.com/pkg/errors" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/debug/handler" "github.com/multiversx/mx-chain-go/p2p" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/interceptors/disabled" - "github.com/multiversx/mx-chain-go/storage/cache" + "github.com/multiversx/mx-chain-go/storage" ) var log = logger.GetOrCreate("process/interceptors") // ArgMultiDataInterceptor is the argument for the multi-data interceptor type ArgMultiDataInterceptor struct { - Topic string - Marshalizer marshal.Marshalizer - DataFactory process.InterceptedDataFactory - Processor process.InterceptorProcessor - Throttler process.InterceptorThrottler - AntifloodHandler process.P2PAntifloodHandler - WhiteListRequest process.WhiteListHandler - PreferredPeersHolder process.PreferredPeersHolderHandler - CurrentPeerId core.PeerID + Topic string + Marshalizer marshal.Marshalizer + DataFactory process.InterceptedDataFactory + Processor process.InterceptorProcessor + Throttler process.InterceptorThrottler + AntifloodHandler process.P2PAntifloodHandler + WhiteListRequest process.WhiteListHandler + PreferredPeersHolder process.PreferredPeersHolderHandler + CurrentPeerId core.PeerID + ProcessedMessagesCacheMap map[string]storage.Cacher } // MultiDataInterceptor is used for intercepting packed multi data @@ -72,17 +73,20 @@ func NewMultiDataInterceptor(arg ArgMultiDataInterceptor) (*MultiDataInterceptor if len(arg.CurrentPeerId) == 0 { return nil, process.ErrEmptyPeerID } + if arg.ProcessedMessagesCacheMap == nil { + return nil, process.ErrNilProcessedMessagesCacheMap + } multiDataIntercept := &MultiDataInterceptor{ baseDataInterceptor: &baseDataInterceptor{ - throttler: arg.Throttler, - antifloodHandler: arg.AntifloodHandler, - topic: arg.Topic, - currentPeerId: arg.CurrentPeerId, - processor: arg.Processor, - preferredPeersHolder: arg.PreferredPeersHolder, - debugHandler: handler.NewDisabledInterceptorDebugHandler(), - timeCache: cache.NewTimeCache(30 * time.Second), + throttler: arg.Throttler, + antifloodHandler: arg.AntifloodHandler, + topic: arg.Topic, + currentPeerId: arg.CurrentPeerId, + processor: arg.Processor, + preferredPeersHolder: arg.PreferredPeersHolder, + debugHandler: handler.NewDisabledInterceptorDebugHandler(), + processedMessagesCacheMap: arg.ProcessedMessagesCacheMap, }, marshalizer: arg.Marshalizer, factory: arg.DataFactory, @@ -157,16 +161,18 @@ func (mdi *MultiDataInterceptor) ProcessReceivedMessage(message p2p.MessageP2P, var interceptedData process.InterceptedData interceptedData, err = mdi.interceptedData(dataBuff, message.Peer(), fromConnectedPeer) - errCache := mdi.checkIfMessageHasBeenProcessed(interceptedData) - if errCache != nil { - continue - } listInterceptedData[index] = interceptedData if err != nil { mdi.throttler.EndProcessing() return err } + errCache := mdi.checkIfMessageHasBeenProcessed(interceptedData) + if errCache != nil { + mdi.throttler.EndProcessing() + continue + } + isWhiteListed := mdi.whiteListRequest.IsWhiteListed(interceptedData) if !isWhiteListed && errOriginator != nil { mdi.throttler.EndProcessing() @@ -215,16 +221,12 @@ func (mdi *MultiDataInterceptor) interceptedData(dataBuff []byte, originator cor } mdi.receivedDebugInterceptedData(interceptedData) - //shouldProcess := mdi.checkIfMessageHasBeenProcessed(interceptedData) - //if !shouldProcess { - // return nil, nil - //} err = interceptedData.CheckValidity() if err != nil { mdi.processDebugInterceptedData(interceptedData, err) - isWrongVersion := err == process.ErrInvalidTransactionVersion || err == process.ErrInvalidChainID + isWrongVersion := errors.Is(err, process.ErrInvalidTransactionVersion) || errors.Is(err, process.ErrInvalidChainID) if isWrongVersion { // this situation is so severe that we need to black list de peers reason := "wrong version of received intercepted data, topic " + mdi.topic + ", error " + err.Error() diff --git a/process/interceptors/multiDataInterceptor_test.go b/process/interceptors/multiDataInterceptor_test.go index 45c8b307b17..bc3599c4f0b 100644 --- a/process/interceptors/multiDataInterceptor_test.go +++ b/process/interceptors/multiDataInterceptor_test.go @@ -16,6 +16,7 @@ import ( "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/interceptors" "github.com/multiversx/mx-chain-go/process/mock" + "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/testscommon" "github.com/multiversx/mx-chain-go/testscommon/p2pmocks" ) @@ -24,15 +25,16 @@ var fromConnectedPeerId = core.PeerID("from connected peer Id") func createMockArgMultiDataInterceptor() interceptors.ArgMultiDataInterceptor { return interceptors.ArgMultiDataInterceptor{ - Topic: "test topic", - Marshalizer: &mock.MarshalizerMock{}, - DataFactory: &mock.InterceptedDataFactoryStub{}, - Processor: &mock.InterceptorProcessorStub{}, - Throttler: createMockThrottler(), - AntifloodHandler: &mock.P2PAntifloodHandlerStub{}, - WhiteListRequest: &testscommon.WhiteListHandlerStub{}, - PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, - CurrentPeerId: "pid", + Topic: "test topic", + Marshalizer: &mock.MarshalizerMock{}, + DataFactory: &mock.InterceptedDataFactoryStub{}, + Processor: &mock.InterceptorProcessorStub{}, + Throttler: createMockThrottler(), + AntifloodHandler: &mock.P2PAntifloodHandlerStub{}, + WhiteListRequest: &testscommon.WhiteListHandlerStub{}, + PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, + CurrentPeerId: "pid", + ProcessedMessagesCacheMap: make(map[string]storage.Cacher), } } @@ -283,6 +285,7 @@ func TestMultiDataInterceptor_ProcessReceivedPartiallyCorrectDataShouldErr(t *te IsForCurrentShardCalled: func() bool { return true }, + HashCalled: func() []byte { return []byte("hash") }, } arg := createMockArgMultiDataInterceptor() arg.DataFactory = &mock.InterceptedDataFactoryStub{ diff --git a/process/interceptors/singleDataInterceptor.go b/process/interceptors/singleDataInterceptor.go index 14eb0ec6d16..b3cab35b741 100644 --- a/process/interceptors/singleDataInterceptor.go +++ b/process/interceptors/singleDataInterceptor.go @@ -2,7 +2,6 @@ package interceptors import ( "errors" - "time" "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" @@ -11,19 +10,20 @@ import ( "github.com/multiversx/mx-chain-go/debug/handler" "github.com/multiversx/mx-chain-go/p2p" "github.com/multiversx/mx-chain-go/process" - "github.com/multiversx/mx-chain-go/storage/cache" + "github.com/multiversx/mx-chain-go/storage" ) // ArgSingleDataInterceptor is the argument for the single-data interceptor type ArgSingleDataInterceptor struct { - Topic string - DataFactory process.InterceptedDataFactory - Processor process.InterceptorProcessor - Throttler process.InterceptorThrottler - AntifloodHandler process.P2PAntifloodHandler - WhiteListRequest process.WhiteListHandler - PreferredPeersHolder process.PreferredPeersHolderHandler - CurrentPeerId core.PeerID + Topic string + DataFactory process.InterceptedDataFactory + Processor process.InterceptorProcessor + Throttler process.InterceptorThrottler + AntifloodHandler process.P2PAntifloodHandler + WhiteListRequest process.WhiteListHandler + PreferredPeersHolder process.PreferredPeersHolderHandler + CurrentPeerId core.PeerID + ProcessedMessagesCacheMap map[string]storage.Cacher } // SingleDataInterceptor is used for intercepting packed multi data @@ -59,17 +59,20 @@ func NewSingleDataInterceptor(arg ArgSingleDataInterceptor) (*SingleDataIntercep if len(arg.CurrentPeerId) == 0 { return nil, process.ErrEmptyPeerID } + if arg.ProcessedMessagesCacheMap == nil { + return nil, process.ErrNilProcessedMessagesCacheMap + } singleDataIntercept := &SingleDataInterceptor{ baseDataInterceptor: &baseDataInterceptor{ - throttler: arg.Throttler, - antifloodHandler: arg.AntifloodHandler, - topic: arg.Topic, - currentPeerId: arg.CurrentPeerId, - processor: arg.Processor, - preferredPeersHolder: arg.PreferredPeersHolder, - debugHandler: handler.NewDisabledInterceptorDebugHandler(), - timeCache: cache.NewTimeCache(30 * time.Second), + throttler: arg.Throttler, + antifloodHandler: arg.AntifloodHandler, + topic: arg.Topic, + currentPeerId: arg.CurrentPeerId, + processor: arg.Processor, + preferredPeersHolder: arg.PreferredPeersHolder, + debugHandler: handler.NewDisabledInterceptorDebugHandler(), + processedMessagesCacheMap: arg.ProcessedMessagesCacheMap, }, factory: arg.DataFactory, whiteListRequest: arg.WhiteListRequest, @@ -81,9 +84,6 @@ func NewSingleDataInterceptor(arg ArgSingleDataInterceptor) (*SingleDataIntercep // ProcessReceivedMessage is the callback func from the p2p.Messenger and will be called each time a new message was received // (for the topic this validator was registered to) func (sdi *SingleDataInterceptor) ProcessReceivedMessage(message p2p.MessageP2P, fromConnectedPeer core.PeerID, _ p2p.MessageHandler) error { - // Sweep the time cache before processing the message - sdi.timeCache.Sweep() - err := sdi.preProcessMesage(message, fromConnectedPeer) if err != nil { return err diff --git a/process/interceptors/singleDataInterceptor_test.go b/process/interceptors/singleDataInterceptor_test.go index df49ecf1e9a..f8d391d3471 100644 --- a/process/interceptors/singleDataInterceptor_test.go +++ b/process/interceptors/singleDataInterceptor_test.go @@ -16,20 +16,23 @@ import ( "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/interceptors" "github.com/multiversx/mx-chain-go/process/mock" + "github.com/multiversx/mx-chain-go/storage" + "github.com/multiversx/mx-chain-go/storage/cache" "github.com/multiversx/mx-chain-go/testscommon" "github.com/multiversx/mx-chain-go/testscommon/p2pmocks" ) func createMockArgSingleDataInterceptor() interceptors.ArgSingleDataInterceptor { return interceptors.ArgSingleDataInterceptor{ - Topic: "test topic", - DataFactory: &mock.InterceptedDataFactoryStub{}, - Processor: &mock.InterceptorProcessorStub{}, - Throttler: createMockThrottler(), - AntifloodHandler: &mock.P2PAntifloodHandlerStub{}, - WhiteListRequest: &testscommon.WhiteListHandlerStub{}, - PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, - CurrentPeerId: "pid", + Topic: "test topic", + DataFactory: &mock.InterceptedDataFactoryStub{}, + Processor: &mock.InterceptorProcessorStub{}, + Throttler: createMockThrottler(), + AntifloodHandler: &mock.P2PAntifloodHandlerStub{}, + WhiteListRequest: &testscommon.WhiteListHandlerStub{}, + PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, + CurrentPeerId: "pid", + ProcessedMessagesCacheMap: make(map[string]storage.Cacher), } } @@ -508,6 +511,14 @@ func TestSingleDataInterceptor_ProcessSameMessage(t *testing.T) { }, } arg.WhiteListRequest = whiteListHandler + + span := 1 * time.Second + c, _ := cache.NewTimeCacher(cache.ArgTimeCacher{ + DefaultSpan: span, + CacheExpiry: time.Second, + }) + arg.ProcessedMessagesCacheMap[arg.Topic] = c + sdi, _ := interceptors.NewSingleDataInterceptor(arg) msg := &p2pmocks.P2PMessageMock{ @@ -522,7 +533,7 @@ func TestSingleDataInterceptor_ProcessSameMessage(t *testing.T) { defer wg.Done() err := sdi.ProcessReceivedMessage(msg, fromConnectedPeerId, &p2pmocks.MessengerStub{}) - if err != nil && strings.Contains(err.Error(), "has already been processed") { + if err != nil && strings.Contains(err.Error(), "processed intercepted data with hash") { errCount.Add(1) } }() @@ -530,6 +541,10 @@ func TestSingleDataInterceptor_ProcessSameMessage(t *testing.T) { wg.Wait() require.Equal(t, uint32(2), errCount.Load()) + + <-time.After(span + time.Millisecond) + err := sdi.ProcessReceivedMessage(msg, fromConnectedPeerId, &p2pmocks.MessengerStub{}) + require.Nil(t, err) } //------- IsInterfaceNil diff --git a/sharding/nodesSetup.go b/sharding/nodesSetup.go index 26e8bee3351..e85a684faf1 100644 --- a/sharding/nodesSetup.go +++ b/sharding/nodesSetup.go @@ -3,9 +3,11 @@ package sharding import ( "bytes" "fmt" + "os" "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/multiversx/mx-chain-go/config" "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator" ) @@ -182,6 +184,7 @@ func (ns *NodesSetup) processConfig() error { return ErrMinNodesPerShardSmallerThanConsensusSize } if ns.nrOfNodes < ns.genesisChainParameters.ShardMinNumNodes { + fmt.Fprintf(os.Stdout, "number of nodes: %s", ns.nrOfNodes) return ErrNodesSizeSmallerThanMinNoOfNodes } if ns.genesisChainParameters.MetachainMinNumNodes < 1 { From 57f009e905e3523d14b0c1a955f5609d0894c221 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Mon, 16 Sep 2024 16:11:31 +0300 Subject: [PATCH 04/37] fix some tests. --- .../epochStartInterceptorsContainerFactory.go | 4 +++ epochStart/bootstrap/process.go | 32 ++++++++++--------- process/factory/interceptorscontainer/args.go | 1 - 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/epochStart/bootstrap/factory/epochStartInterceptorsContainerFactory.go b/epochStart/bootstrap/factory/epochStartInterceptorsContainerFactory.go index d659989896b..f739be83921 100644 --- a/epochStart/bootstrap/factory/epochStartInterceptorsContainerFactory.go +++ b/epochStart/bootstrap/factory/epochStartInterceptorsContainerFactory.go @@ -6,6 +6,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data/typeConverters" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/config" "github.com/multiversx/mx-chain-go/dataRetriever" @@ -16,6 +17,7 @@ import ( "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/factory/interceptorscontainer" "github.com/multiversx/mx-chain-go/sharding" + "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/storage/cache" "github.com/multiversx/mx-chain-go/update" ) @@ -42,6 +44,7 @@ type ArgsEpochStartInterceptorContainer struct { RequestHandler process.RequestHandler SignaturesHandler process.SignaturesHandler NodeOperationMode common.NodeOperation + ProcessedMessagesCache map[string]storage.Cacher } // NewEpochStartInterceptorsContainer will return a real interceptors container factory, but with many disabled components @@ -108,6 +111,7 @@ func NewEpochStartInterceptorsContainer(args ArgsEpochStartInterceptorContainer) FullArchivePeerShardMapper: fullArchivePeerShardMapper, HardforkTrigger: hardforkTrigger, NodeOperationMode: args.NodeOperationMode, + ProcessedMessagesCacheMap: args.ProcessedMessagesCache, } interceptorsContainerFactory, err := interceptorscontainer.NewMetaInterceptorsContainerFactory(containerFactoryArgs) diff --git a/epochStart/bootstrap/process.go b/epochStart/bootstrap/process.go index 27fc5011cb5..1213d512fe8 100644 --- a/epochStart/bootstrap/process.go +++ b/epochStart/bootstrap/process.go @@ -14,6 +14,8 @@ import ( "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/typeConverters/uint64ByteSlice" + logger "github.com/multiversx/mx-chain-logger-go" + "github.com/multiversx/mx-chain-go/common" disabledCommon "github.com/multiversx/mx-chain-go/common/disabled" "github.com/multiversx/mx-chain-go/common/ordering" @@ -52,7 +54,6 @@ import ( "github.com/multiversx/mx-chain-go/trie/storageMarker" "github.com/multiversx/mx-chain-go/update" updateSync "github.com/multiversx/mx-chain-go/update/sync" - logger "github.com/multiversx/mx-chain-logger-go" ) var log = logger.GetOrCreate("epochStart/bootstrap") @@ -589,6 +590,7 @@ func (e *epochStartBootstrap) createSyncers() error { RequestHandler: e.requestHandler, SignaturesHandler: e.mainMessenger, NodeOperationMode: e.nodeOperationMode, + ProcessedMessagesCache: make(map[string]storage.Cacher), } e.mainInterceptorContainer, e.fullArchiveInterceptorContainer, err = factoryInterceptors.NewEpochStartInterceptorsContainer(args) @@ -759,20 +761,20 @@ func (e *epochStartBootstrap) processNodesConfig(pubKey []byte) ([]*block.MiniBl shardId = e.genesisShardCoordinator.SelfId() } argsNewValidatorStatusSyncers := ArgsNewSyncValidatorStatus{ - DataPool: e.dataPool, - Marshalizer: e.coreComponentsHolder.InternalMarshalizer(), - RequestHandler: e.requestHandler, - ChanceComputer: e.rater, - GenesisNodesConfig: e.genesisNodesConfig, - ChainParametersHandler: e.coreComponentsHolder.ChainParametersHandler(), - NodeShuffler: e.nodeShuffler, - Hasher: e.coreComponentsHolder.Hasher(), - PubKey: pubKey, - ShardIdAsObserver: shardId, - ChanNodeStop: e.coreComponentsHolder.ChanStopNodeProcess(), - NodeTypeProvider: e.coreComponentsHolder.NodeTypeProvider(), - IsFullArchive: e.prefsConfig.FullArchive, - EnableEpochsHandler: e.coreComponentsHolder.EnableEpochsHandler(), + DataPool: e.dataPool, + Marshalizer: e.coreComponentsHolder.InternalMarshalizer(), + RequestHandler: e.requestHandler, + ChanceComputer: e.rater, + GenesisNodesConfig: e.genesisNodesConfig, + ChainParametersHandler: e.coreComponentsHolder.ChainParametersHandler(), + NodeShuffler: e.nodeShuffler, + Hasher: e.coreComponentsHolder.Hasher(), + PubKey: pubKey, + ShardIdAsObserver: shardId, + ChanNodeStop: e.coreComponentsHolder.ChanStopNodeProcess(), + NodeTypeProvider: e.coreComponentsHolder.NodeTypeProvider(), + IsFullArchive: e.prefsConfig.FullArchive, + EnableEpochsHandler: e.coreComponentsHolder.EnableEpochsHandler(), NodesCoordinatorRegistryFactory: e.nodesCoordinatorRegistryFactory, } diff --git a/process/factory/interceptorscontainer/args.go b/process/factory/interceptorscontainer/args.go index f490a13a54c..dd08954a3ff 100644 --- a/process/factory/interceptorscontainer/args.go +++ b/process/factory/interceptorscontainer/args.go @@ -45,6 +45,5 @@ type CommonInterceptorsContainerFactoryArgs struct { FullArchivePeerShardMapper process.PeerShardMapper HardforkTrigger heartbeat.HardforkTrigger NodeOperationMode common.NodeOperation - ProcessedMessagesCache storage.Cacher ProcessedMessagesCacheMap map[string]storage.Cacher } From f5a6f2e46e97eda3cd2becf587026259664466fb Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Mon, 16 Sep 2024 17:32:50 +0300 Subject: [PATCH 05/37] fixed missing topic. --- .../factory/epochStartInterceptorsContainerFactory.go | 3 --- epochStart/bootstrap/process.go | 1 - .../baseInterceptorsContainerFactory.go | 1 + .../metaInterceptorsContainerFactory.go | 5 +++++ process/interceptors/baseDataInterceptor.go | 7 +------ sharding/nodesSetup.go | 2 -- 6 files changed, 7 insertions(+), 12 deletions(-) diff --git a/epochStart/bootstrap/factory/epochStartInterceptorsContainerFactory.go b/epochStart/bootstrap/factory/epochStartInterceptorsContainerFactory.go index f739be83921..542a4f8bf78 100644 --- a/epochStart/bootstrap/factory/epochStartInterceptorsContainerFactory.go +++ b/epochStart/bootstrap/factory/epochStartInterceptorsContainerFactory.go @@ -17,7 +17,6 @@ import ( "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/factory/interceptorscontainer" "github.com/multiversx/mx-chain-go/sharding" - "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/storage/cache" "github.com/multiversx/mx-chain-go/update" ) @@ -44,7 +43,6 @@ type ArgsEpochStartInterceptorContainer struct { RequestHandler process.RequestHandler SignaturesHandler process.SignaturesHandler NodeOperationMode common.NodeOperation - ProcessedMessagesCache map[string]storage.Cacher } // NewEpochStartInterceptorsContainer will return a real interceptors container factory, but with many disabled components @@ -111,7 +109,6 @@ func NewEpochStartInterceptorsContainer(args ArgsEpochStartInterceptorContainer) FullArchivePeerShardMapper: fullArchivePeerShardMapper, HardforkTrigger: hardforkTrigger, NodeOperationMode: args.NodeOperationMode, - ProcessedMessagesCacheMap: args.ProcessedMessagesCache, } interceptorsContainerFactory, err := interceptorscontainer.NewMetaInterceptorsContainerFactory(containerFactoryArgs) diff --git a/epochStart/bootstrap/process.go b/epochStart/bootstrap/process.go index 1213d512fe8..31a73bc5680 100644 --- a/epochStart/bootstrap/process.go +++ b/epochStart/bootstrap/process.go @@ -590,7 +590,6 @@ func (e *epochStartBootstrap) createSyncers() error { RequestHandler: e.requestHandler, SignaturesHandler: e.mainMessenger, NodeOperationMode: e.nodeOperationMode, - ProcessedMessagesCache: make(map[string]storage.Cacher), } e.mainInterceptorContainer, e.fullArchiveInterceptorContainer, err = factoryInterceptors.NewEpochStartInterceptorsContainer(args) diff --git a/process/factory/interceptorscontainer/baseInterceptorsContainerFactory.go b/process/factory/interceptorscontainer/baseInterceptorsContainerFactory.go index 999e9e51487..7008a5e3a5e 100644 --- a/process/factory/interceptorscontainer/baseInterceptorsContainerFactory.go +++ b/process/factory/interceptorscontainer/baseInterceptorsContainerFactory.go @@ -341,6 +341,7 @@ func (bicf *baseInterceptorsContainerFactory) createOneUnsignedTxInterceptor(top err = bicf.createCacheForInterceptor(topic) if err != nil { + fmt.Println("something is wrong here") return nil, err } diff --git a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go index 5f1abbaa4b9..96c0365df91 100644 --- a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go +++ b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go @@ -263,6 +263,11 @@ func (micf *metaInterceptorsContainerFactory) createOneShardHeaderInterceptor(to return nil, err } + err = micf.createCacheForInterceptor(topic) + if err != nil { + return nil, err + } + interceptor, err := processInterceptors.NewSingleDataInterceptor( processInterceptors.ArgSingleDataInterceptor{ Topic: topic, diff --git a/process/interceptors/baseDataInterceptor.go b/process/interceptors/baseDataInterceptor.go index de679963988..0b89553acdb 100644 --- a/process/interceptors/baseDataInterceptor.go +++ b/process/interceptors/baseDataInterceptor.go @@ -128,16 +128,11 @@ func (bdi *baseDataInterceptor) checkIfMessageHasBeenProcessed(interceptedData p return nil } - c, ok := bdi.processedMessagesCacheMap[bdi.topic] + cache, ok := bdi.processedMessagesCacheMap[bdi.topic] if !ok { return fmt.Errorf("cache for topic %q does not exist", bdi.topic) } - cache, ok := c.(storage.Cacher) - if !ok { - return fmt.Errorf("failed to cast cacher") - } - if has, _ := cache.HasOrAdd(interceptedData.Hash(), nil, 0); has { return fmt.Errorf("processed intercepted data with hash: %s", interceptedData.Hash()) } diff --git a/sharding/nodesSetup.go b/sharding/nodesSetup.go index e85a684faf1..32f9b1dbc92 100644 --- a/sharding/nodesSetup.go +++ b/sharding/nodesSetup.go @@ -3,7 +3,6 @@ package sharding import ( "bytes" "fmt" - "os" "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" @@ -184,7 +183,6 @@ func (ns *NodesSetup) processConfig() error { return ErrMinNodesPerShardSmallerThanConsensusSize } if ns.nrOfNodes < ns.genesisChainParameters.ShardMinNumNodes { - fmt.Fprintf(os.Stdout, "number of nodes: %s", ns.nrOfNodes) return ErrNodesSizeSmallerThanMinNoOfNodes } if ns.genesisChainParameters.MetachainMinNumNodes < 1 { From 8eec33073d667bfb586b6b83a822b5b29c8df599 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Tue, 17 Sep 2024 10:17:12 +0300 Subject: [PATCH 06/37] fix some more tests. --- .../factory/epochStartInterceptorsContainerFactory.go | 2 ++ .../shardInterceptorsContainerFactory_test.go | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/epochStart/bootstrap/factory/epochStartInterceptorsContainerFactory.go b/epochStart/bootstrap/factory/epochStartInterceptorsContainerFactory.go index 542a4f8bf78..fb6ca753834 100644 --- a/epochStart/bootstrap/factory/epochStartInterceptorsContainerFactory.go +++ b/epochStart/bootstrap/factory/epochStartInterceptorsContainerFactory.go @@ -17,6 +17,7 @@ import ( "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/factory/interceptorscontainer" "github.com/multiversx/mx-chain-go/sharding" + "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/storage/cache" "github.com/multiversx/mx-chain-go/update" ) @@ -109,6 +110,7 @@ func NewEpochStartInterceptorsContainer(args ArgsEpochStartInterceptorContainer) FullArchivePeerShardMapper: fullArchivePeerShardMapper, HardforkTrigger: hardforkTrigger, NodeOperationMode: args.NodeOperationMode, + ProcessedMessagesCacheMap: make(map[string]storage.Cacher), } interceptorsContainerFactory, err := interceptorscontainer.NewMetaInterceptorsContainerFactory(containerFactoryArgs) diff --git a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go index 24564ec1cf1..c50bbf30ba6 100644 --- a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go +++ b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go @@ -6,6 +6,8 @@ import ( "testing" "github.com/multiversx/mx-chain-core-go/core/versioning" + "github.com/stretchr/testify/assert" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/p2p" @@ -26,7 +28,6 @@ import ( "github.com/multiversx/mx-chain-go/testscommon/shardingMocks" stateMock "github.com/multiversx/mx-chain-go/testscommon/state" storageStubs "github.com/multiversx/mx-chain-go/testscommon/storage" - "github.com/stretchr/testify/assert" ) var providedHardforkPubKey = []byte("provided hardfork pub key") @@ -559,6 +560,7 @@ func TestShardInterceptorsContainerFactory_CreateShouldWork(t *testing.T) { }, } args.WhiteListerVerifiedTxs = &testscommon.WhiteListHandlerStub{} + args.ProcessedMessagesCacheMap = make(map[string]storage.Cacher) icf, _ := interceptorscontainer.NewShardInterceptorsContainerFactory(args) From a733097da3068b7fa6f56042db8f3fd70d3a4f15 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Tue, 17 Sep 2024 11:37:14 +0300 Subject: [PATCH 07/37] fix nil pointer dereferences in tests. --- epochStart/bootstrap/storageProcess_test.go | 3 ++- epochStart/bootstrap/syncEpochStartMeta.go | 19 +++++++++++-------- .../metaInterceptorsContainerFactory_test.go | 8 ++++++-- process/interceptors/multiDataInterceptor.go | 6 +++--- 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/epochStart/bootstrap/storageProcess_test.go b/epochStart/bootstrap/storageProcess_test.go index a59b0d125f2..c61ef9f279f 100644 --- a/epochStart/bootstrap/storageProcess_test.go +++ b/epochStart/bootstrap/storageProcess_test.go @@ -11,6 +11,8 @@ import ( "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/endProcess" + "github.com/stretchr/testify/assert" + "github.com/multiversx/mx-chain-go/config" "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/epochStart" @@ -23,7 +25,6 @@ import ( dataRetrieverMock "github.com/multiversx/mx-chain-go/testscommon/dataRetriever" "github.com/multiversx/mx-chain-go/testscommon/economicsmocks" "github.com/multiversx/mx-chain-go/testscommon/genesisMocks" - "github.com/stretchr/testify/assert" ) func createMockStorageEpochStartBootstrapArgs( diff --git a/epochStart/bootstrap/syncEpochStartMeta.go b/epochStart/bootstrap/syncEpochStartMeta.go index fa764a04c4a..dc8164ec269 100644 --- a/epochStart/bootstrap/syncEpochStartMeta.go +++ b/epochStart/bootstrap/syncEpochStartMeta.go @@ -8,6 +8,7 @@ import ( "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/config" "github.com/multiversx/mx-chain-go/epochStart" @@ -17,6 +18,7 @@ import ( "github.com/multiversx/mx-chain-go/process/interceptors" interceptorsFactory "github.com/multiversx/mx-chain-go/process/interceptors/factory" "github.com/multiversx/mx-chain-go/sharding" + "github.com/multiversx/mx-chain-go/storage" ) var _ epochStart.StartOfEpochMetaSyncer = (*epochStartMetaSyncer)(nil) @@ -91,14 +93,15 @@ func NewEpochStartMetaSyncer(args ArgsNewEpochStartMetaSyncer) (*epochStartMetaS e.singleDataInterceptor, err = interceptors.NewSingleDataInterceptor( interceptors.ArgSingleDataInterceptor{ - Topic: factory.MetachainBlocksTopic, - DataFactory: interceptedMetaHdrDataFactory, - Processor: args.MetaBlockProcessor, - Throttler: disabled.NewThrottler(), - AntifloodHandler: disabled.NewAntiFloodHandler(), - WhiteListRequest: args.WhitelistHandler, - CurrentPeerId: args.Messenger.ID(), - PreferredPeersHolder: disabled.NewPreferredPeersHolder(), + Topic: factory.MetachainBlocksTopic, + DataFactory: interceptedMetaHdrDataFactory, + Processor: args.MetaBlockProcessor, + Throttler: disabled.NewThrottler(), + AntifloodHandler: disabled.NewAntiFloodHandler(), + WhiteListRequest: args.WhitelistHandler, + CurrentPeerId: args.Messenger.ID(), + PreferredPeersHolder: disabled.NewPreferredPeersHolder(), + ProcessedMessagesCacheMap: make(map[string]storage.Cacher), }, ) if err != nil { diff --git a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go index 28e93408a5f..8e61bb18e75 100644 --- a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go +++ b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go @@ -5,6 +5,9 @@ import ( "strings" "testing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/p2p" @@ -22,8 +25,6 @@ import ( "github.com/multiversx/mx-chain-go/testscommon/shardingMocks" stateMock "github.com/multiversx/mx-chain-go/testscommon/state" storageStubs "github.com/multiversx/mx-chain-go/testscommon/storage" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) const maxTxNonceDeltaAllowed = 100 @@ -589,6 +590,8 @@ func TestMetaInterceptorsContainerFactory_With4ShardsShouldWork(t *testing.T) { args := getArgumentsMeta(coreComp, cryptoComp) args.ShardCoordinator = shardCoordinator args.NodesCoordinator = nodesCoordinator + args.ProcessedMessagesCacheMap = make(map[string]storage.Cacher) + icf, err := interceptorscontainer.NewMetaInterceptorsContainerFactory(args) require.Nil(t, err) @@ -638,6 +641,7 @@ func TestMetaInterceptorsContainerFactory_With4ShardsShouldWork(t *testing.T) { args.NodeOperationMode = common.FullArchiveMode args.ShardCoordinator = shardCoordinator args.NodesCoordinator = nodesCoordinator + args.ProcessedMessagesCacheMap = make(map[string]storage.Cacher) icf, err := interceptorscontainer.NewMetaInterceptorsContainerFactory(args) require.Nil(t, err) diff --git a/process/interceptors/multiDataInterceptor.go b/process/interceptors/multiDataInterceptor.go index db80bdd7be8..bbe4e71541e 100644 --- a/process/interceptors/multiDataInterceptor.go +++ b/process/interceptors/multiDataInterceptor.go @@ -73,9 +73,9 @@ func NewMultiDataInterceptor(arg ArgMultiDataInterceptor) (*MultiDataInterceptor if len(arg.CurrentPeerId) == 0 { return nil, process.ErrEmptyPeerID } - if arg.ProcessedMessagesCacheMap == nil { - return nil, process.ErrNilProcessedMessagesCacheMap - } + //if arg.ProcessedMessagesCacheMap == nil { + // return nil, process.ErrNilProcessedMessagesCacheMap + //} multiDataIntercept := &MultiDataInterceptor{ baseDataInterceptor: &baseDataInterceptor{ From bab3b05c6a59865267237c9ec37384f7fd5dc9dd Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Tue, 17 Sep 2024 11:47:48 +0300 Subject: [PATCH 08/37] more tests fixed. --- .../metaInterceptorsContainerFactory_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go index 8e61bb18e75..fcbf0bbba55 100644 --- a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go +++ b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go @@ -558,6 +558,7 @@ func TestMetaInterceptorsContainerFactory_CreateShouldWork(t *testing.T) { coreComp, cryptoComp := createMockComponentHolders() args := getArgumentsMeta(coreComp, cryptoComp) + args.ProcessedMessagesCacheMap = make(map[string]storage.Cacher) icf, _ := interceptorscontainer.NewMetaInterceptorsContainerFactory(args) mainContainer, fullArchiveContainer, err := icf.Create() From c3f7120ea7726ef7c6745d69887d6aaf788a8d48 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Tue, 17 Sep 2024 12:11:41 +0300 Subject: [PATCH 09/37] add map for cacher in more unit tests. --- .../shardInterceptorsContainerFactory_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go index c50bbf30ba6..a19e86a1cfa 100644 --- a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go +++ b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go @@ -596,6 +596,7 @@ func TestShardInterceptorsContainerFactory_With4ShardsShouldWork(t *testing.T) { args.ShardCoordinator = shardCoordinator args.NodesCoordinator = nodesCoordinator args.PreferredPeersHolder = &p2pmocks.PeersHolderStub{} + args.ProcessedMessagesCacheMap = make(map[string]storage.Cacher) icf, _ := interceptorscontainer.NewShardInterceptorsContainerFactory(args) @@ -644,6 +645,7 @@ func TestShardInterceptorsContainerFactory_With4ShardsShouldWork(t *testing.T) { args.ShardCoordinator = shardCoordinator args.NodesCoordinator = nodesCoordinator args.PreferredPeersHolder = &p2pmocks.PeersHolderStub{} + args.ProcessedMessagesCacheMap = make(map[string]storage.Cacher) icf, _ := interceptorscontainer.NewShardInterceptorsContainerFactory(args) From b62e37bf562405bb014aedda3f16b9bc0316e862 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Tue, 17 Sep 2024 12:56:32 +0300 Subject: [PATCH 10/37] fix nil map for more tests. --- .../shardInterceptorsContainerFactory_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go index a19e86a1cfa..12f1269516a 100644 --- a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go +++ b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go @@ -494,6 +494,7 @@ func testCreateShardTopicShouldFail(matchStrToErrOnCreate string, matchStrToErrO coreComp, cryptoComp := createMockComponentHolders() args := getArgumentsShard(coreComp, cryptoComp) + args.ProcessedMessagesCacheMap = make(map[string]storage.Cacher) if strings.Contains(t.Name(), "full_archive") { args.NodeOperationMode = common.FullArchiveMode args.FullArchiveMessenger = createShardStubTopicHandler(matchStrToErrOnCreate, matchStrToErrOnRegister) From c00f8ee84f9011c806425ad802e686f36c2ecb7f Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Tue, 17 Sep 2024 13:28:45 +0300 Subject: [PATCH 11/37] fix process tests. --- .../metaInterceptorsContainerFactory_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go index fcbf0bbba55..a464ea9be1b 100644 --- a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go +++ b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go @@ -543,6 +543,7 @@ func testCreateMetaTopicShouldFail(matchStrToErrOnCreate string, matchStrToErrOn } else { args.MainMessenger = createMetaStubTopicHandler(matchStrToErrOnCreate, matchStrToErrOnRegister) } + args.ProcessedMessagesCacheMap = make(map[string]storage.Cacher) icf, _ := interceptorscontainer.NewMetaInterceptorsContainerFactory(args) mainContainer, fullArchiveConatiner, err := icf.Create() From d59c5bd0bca1620d6b95f24354741e2165b752de Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Wed, 18 Sep 2024 10:22:10 +0300 Subject: [PATCH 12/37] fix conflicts with target branch. --- process/errors.go | 3 --- .../interceptorscontainer/shardInterceptorsContainerFactory.go | 1 + process/interceptors/multiDataInterceptor.go | 3 --- process/interceptors/singleDataInterceptor.go | 3 --- 4 files changed, 1 insertion(+), 9 deletions(-) diff --git a/process/errors.go b/process/errors.go index 09065d03144..8edf7342ada 100644 --- a/process/errors.go +++ b/process/errors.go @@ -942,9 +942,6 @@ var ErrNilEpochStartSystemSCProcessor = errors.New("nil epoch start system sc pr // ErrEmptyPeerID signals that an empty peer ID has been provided var ErrEmptyPeerID = errors.New("empty peer ID") -// ErrNilProcessedMessagesCacheMap signals that an empty cache map has been provided -var ErrNilProcessedMessagesCacheMap = errors.New("empty processed messages cache map") - // ErrNilFallbackHeaderValidator signals that a nil fallback header validator has been provided var ErrNilFallbackHeaderValidator = errors.New("nil fallback header validator") diff --git a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go index 08eb4e45814..e1d0b3fbef6 100644 --- a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go +++ b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go @@ -5,6 +5,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/core/throttler" "github.com/multiversx/mx-chain-core-go/marshal" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/process" diff --git a/process/interceptors/multiDataInterceptor.go b/process/interceptors/multiDataInterceptor.go index bbe4e71541e..2b1a9335ead 100644 --- a/process/interceptors/multiDataInterceptor.go +++ b/process/interceptors/multiDataInterceptor.go @@ -73,9 +73,6 @@ func NewMultiDataInterceptor(arg ArgMultiDataInterceptor) (*MultiDataInterceptor if len(arg.CurrentPeerId) == 0 { return nil, process.ErrEmptyPeerID } - //if arg.ProcessedMessagesCacheMap == nil { - // return nil, process.ErrNilProcessedMessagesCacheMap - //} multiDataIntercept := &MultiDataInterceptor{ baseDataInterceptor: &baseDataInterceptor{ diff --git a/process/interceptors/singleDataInterceptor.go b/process/interceptors/singleDataInterceptor.go index b3cab35b741..75038588a84 100644 --- a/process/interceptors/singleDataInterceptor.go +++ b/process/interceptors/singleDataInterceptor.go @@ -59,9 +59,6 @@ func NewSingleDataInterceptor(arg ArgSingleDataInterceptor) (*SingleDataIntercep if len(arg.CurrentPeerId) == 0 { return nil, process.ErrEmptyPeerID } - if arg.ProcessedMessagesCacheMap == nil { - return nil, process.ErrNilProcessedMessagesCacheMap - } singleDataIntercept := &SingleDataInterceptor{ baseDataInterceptor: &baseDataInterceptor{ From 396cd8ebdece6a9ca6a78f3db6ecfc6b15b1d912 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Thu, 19 Sep 2024 18:22:38 +0300 Subject: [PATCH 13/37] refactored message validation to a different component. --- epochStart/bootstrap/syncEpochStartMeta.go | 26 +- .../interceptedBlockHeader.go | 3 +- .../interceptedBlockHeader_test.go | 7 +- .../interceptedMetaBlockHeader.go | 5 +- .../interceptedMetaBlockHeader_test.go | 7 +- .../interceptedMiniblock_test.go | 5 +- process/errors.go | 3 + .../baseInterceptorsContainerFactory.go | 263 +++++++++--------- .../metaInterceptorsContainerFactory.go | 20 +- process/interceptors/baseDataInterceptor.go | 37 +-- .../interceptedDataCacherVerifier.go | 54 ++++ .../interceptedDataVerifier_test.go | 47 ++++ process/interceptors/multiDataInterceptor.go | 54 ++-- .../interceptors/multiDataInterceptor_test.go | 3 + process/interceptors/singleDataInterceptor.go | 49 ++-- process/interface.go | 5 + .../interceptedTransaction_test.go | 2 +- .../interceptedUnsignedTransaction_test.go | 7 +- update/factory/fullSyncInterceptors.go | 177 ++++++++---- 19 files changed, 472 insertions(+), 302 deletions(-) create mode 100644 process/interceptors/interceptedDataCacherVerifier.go create mode 100644 process/interceptors/interceptedDataVerifier_test.go diff --git a/epochStart/bootstrap/syncEpochStartMeta.go b/epochStart/bootstrap/syncEpochStartMeta.go index dc8164ec269..450def2882e 100644 --- a/epochStart/bootstrap/syncEpochStartMeta.go +++ b/epochStart/bootstrap/syncEpochStartMeta.go @@ -18,7 +18,7 @@ import ( "github.com/multiversx/mx-chain-go/process/interceptors" interceptorsFactory "github.com/multiversx/mx-chain-go/process/interceptors/factory" "github.com/multiversx/mx-chain-go/sharding" - "github.com/multiversx/mx-chain-go/storage" + "github.com/multiversx/mx-chain-go/storage/cache" ) var _ epochStart.StartOfEpochMetaSyncer = (*epochStartMetaSyncer)(nil) @@ -91,17 +91,23 @@ func NewEpochStartMetaSyncer(args ArgsNewEpochStartMetaSyncer) (*epochStartMetaS return nil, err } + internalCache, err := cache.NewTimeCacher(cache.ArgTimeCacher{ + DefaultSpan: 30 * time.Second, + CacheExpiry: 30 * time.Second, + }) + interceptedDataVerifier := interceptors.NewInterceptedDataVerifier(internalCache) + e.singleDataInterceptor, err = interceptors.NewSingleDataInterceptor( interceptors.ArgSingleDataInterceptor{ - Topic: factory.MetachainBlocksTopic, - DataFactory: interceptedMetaHdrDataFactory, - Processor: args.MetaBlockProcessor, - Throttler: disabled.NewThrottler(), - AntifloodHandler: disabled.NewAntiFloodHandler(), - WhiteListRequest: args.WhitelistHandler, - CurrentPeerId: args.Messenger.ID(), - PreferredPeersHolder: disabled.NewPreferredPeersHolder(), - ProcessedMessagesCacheMap: make(map[string]storage.Cacher), + Topic: factory.MetachainBlocksTopic, + DataFactory: interceptedMetaHdrDataFactory, + Processor: args.MetaBlockProcessor, + Throttler: disabled.NewThrottler(), + AntifloodHandler: disabled.NewAntiFloodHandler(), + WhiteListRequest: args.WhitelistHandler, + CurrentPeerId: args.Messenger.ID(), + PreferredPeersHolder: disabled.NewPreferredPeersHolder(), + InterceptedDataVerifier: interceptedDataVerifier, }, ) if err != nil { diff --git a/process/block/interceptedBlocks/interceptedBlockHeader.go b/process/block/interceptedBlocks/interceptedBlockHeader.go index 0cdb2cec703..9c009b8bb3f 100644 --- a/process/block/interceptedBlocks/interceptedBlockHeader.go +++ b/process/block/interceptedBlocks/interceptedBlockHeader.go @@ -6,10 +6,11 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/hashing" + logger "github.com/multiversx/mx-chain-logger-go" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/sharding" - logger "github.com/multiversx/mx-chain-logger-go" ) var _ process.HdrValidatorHandler = (*InterceptedHeader)(nil) diff --git a/process/block/interceptedBlocks/interceptedBlockHeader_test.go b/process/block/interceptedBlocks/interceptedBlockHeader_test.go index 8536f799997..5cf017b73dc 100644 --- a/process/block/interceptedBlocks/interceptedBlockHeader_test.go +++ b/process/block/interceptedBlocks/interceptedBlockHeader_test.go @@ -10,6 +10,9 @@ import ( "github.com/multiversx/mx-chain-core-go/data" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/marshal" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/block/interceptedBlocks" @@ -17,8 +20,6 @@ import ( "github.com/multiversx/mx-chain-go/testscommon/consensus" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" "github.com/multiversx/mx-chain-go/testscommon/hashingMocks" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) var testMarshalizer = &mock.MarshalizerMock{} @@ -172,7 +173,7 @@ func TestNewInterceptedHeader_MetachainForThisShardShouldWork(t *testing.T) { assert.True(t, inHdr.IsForCurrentShard()) } -//------- CheckValidity +//------- Verify func TestInterceptedHeader_CheckValidityNilPubKeyBitmapShouldErr(t *testing.T) { t.Parallel() diff --git a/process/block/interceptedBlocks/interceptedMetaBlockHeader.go b/process/block/interceptedBlocks/interceptedMetaBlockHeader.go index 050c457598c..29e8dae4228 100644 --- a/process/block/interceptedBlocks/interceptedMetaBlockHeader.go +++ b/process/block/interceptedBlocks/interceptedMetaBlockHeader.go @@ -8,10 +8,11 @@ import ( "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" + logger "github.com/multiversx/mx-chain-logger-go" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/sharding" - logger "github.com/multiversx/mx-chain-logger-go" ) var _ process.HdrValidatorHandler = (*InterceptedMetaHeader)(nil) @@ -99,7 +100,7 @@ func (imh *InterceptedMetaHeader) CheckValidity() error { } if imh.isMetaHeaderEpochOutOfRange() { - log.Trace("InterceptedMetaHeader.CheckValidity", + log.Trace("InterceptedMetaHeader.Verify", "trigger epoch", imh.epochStartTrigger.Epoch(), "metaBlock epoch", imh.hdr.GetEpoch(), "error", process.ErrMetaHeaderEpochOutOfRange) diff --git a/process/block/interceptedBlocks/interceptedMetaBlockHeader_test.go b/process/block/interceptedBlocks/interceptedMetaBlockHeader_test.go index e952e9fc476..a3776269e21 100644 --- a/process/block/interceptedBlocks/interceptedMetaBlockHeader_test.go +++ b/process/block/interceptedBlocks/interceptedMetaBlockHeader_test.go @@ -8,13 +8,14 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/block/interceptedBlocks" "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/testscommon/consensus" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func createDefaultMetaArgument() *interceptedBlocks.ArgInterceptedBlockHeader { @@ -98,7 +99,7 @@ func TestNewInterceptedMetaHeader_ShouldWork(t *testing.T) { assert.Nil(t, err) } -//------- CheckValidity +//------- Verify func TestInterceptedMetaHeader_CheckValidityNilPubKeyBitmapShouldErr(t *testing.T) { t.Parallel() diff --git a/process/block/interceptedBlocks/interceptedMiniblock_test.go b/process/block/interceptedBlocks/interceptedMiniblock_test.go index 57d53ec251d..46b489b259d 100644 --- a/process/block/interceptedBlocks/interceptedMiniblock_test.go +++ b/process/block/interceptedBlocks/interceptedMiniblock_test.go @@ -5,10 +5,11 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data/block" + "github.com/stretchr/testify/assert" + "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/block/interceptedBlocks" "github.com/multiversx/mx-chain-go/process/mock" - "github.com/stretchr/testify/assert" ) func createDefaultMiniblockArgument() *interceptedBlocks.ArgInterceptedMiniblock { @@ -69,7 +70,7 @@ func TestNewInterceptedMiniblock_ShouldWork(t *testing.T) { assert.Nil(t, err) } -//------- CheckValidity +//------- Verify func TestInterceptedMiniblock_InvalidReceiverShardIdShouldErr(t *testing.T) { t.Parallel() diff --git a/process/errors.go b/process/errors.go index 8edf7342ada..a126b0f7513 100644 --- a/process/errors.go +++ b/process/errors.go @@ -696,6 +696,9 @@ var ErrNilWhiteListHandler = errors.New("nil whitelist handler") // ErrNilPreferredPeersHolder signals that preferred peers holder is nil var ErrNilPreferredPeersHolder = errors.New("nil preferred peers holder") +// ErrNilInterceptedDataVerifier signals that intercepted data verifier is nil +var ErrNilInterceptedDataVerifier = errors.New("nil intercepted data verifier") + // ErrMiniBlocksInWrongOrder signals the miniblocks are in wrong order var ErrMiniBlocksInWrongOrder = errors.New("miniblocks in wrong order, should have been only from me") diff --git a/process/factory/interceptorscontainer/baseInterceptorsContainerFactory.go b/process/factory/interceptorscontainer/baseInterceptorsContainerFactory.go index d69375beb90..bafc3de6d7c 100644 --- a/process/factory/interceptorscontainer/baseInterceptorsContainerFactory.go +++ b/process/factory/interceptorscontainer/baseInterceptorsContainerFactory.go @@ -291,7 +291,7 @@ func (bicf *baseInterceptorsContainerFactory) createOneTxInterceptor(topic strin return nil, err } - err = bicf.createCacheForInterceptor(topic) + interceptedDataVerifier, err := bicf.createCacheForInterceptor(topic) if err != nil { return nil, err } @@ -299,16 +299,16 @@ func (bicf *baseInterceptorsContainerFactory) createOneTxInterceptor(topic strin internalMarshaller := bicf.argInterceptorFactory.CoreComponents.InternalMarshalizer() interceptor, err := interceptors.NewMultiDataInterceptor( interceptors.ArgMultiDataInterceptor{ - Topic: topic, - Marshalizer: internalMarshaller, - DataFactory: txFactory, - Processor: txProcessor, - Throttler: bicf.globalThrottler, - AntifloodHandler: bicf.antifloodHandler, - WhiteListRequest: bicf.whiteListHandler, - CurrentPeerId: bicf.mainMessenger.ID(), - PreferredPeersHolder: bicf.preferredPeersHolder, - ProcessedMessagesCacheMap: bicf.processedMessagesCacheMap, + Topic: topic, + Marshalizer: internalMarshaller, + DataFactory: txFactory, + Processor: txProcessor, + Throttler: bicf.globalThrottler, + AntifloodHandler: bicf.antifloodHandler, + WhiteListRequest: bicf.whiteListHandler, + CurrentPeerId: bicf.mainMessenger.ID(), + PreferredPeersHolder: bicf.preferredPeersHolder, + InterceptedDataVerifier: interceptedDataVerifier, }, ) if err != nil { @@ -340,7 +340,7 @@ func (bicf *baseInterceptorsContainerFactory) createOneUnsignedTxInterceptor(top return nil, err } - err = bicf.createCacheForInterceptor(topic) + interceptedDataVerifier, err := bicf.createCacheForInterceptor(topic) if err != nil { return nil, err } @@ -348,16 +348,16 @@ func (bicf *baseInterceptorsContainerFactory) createOneUnsignedTxInterceptor(top internalMarshaller := bicf.argInterceptorFactory.CoreComponents.InternalMarshalizer() interceptor, err := interceptors.NewMultiDataInterceptor( interceptors.ArgMultiDataInterceptor{ - Topic: topic, - Marshalizer: internalMarshaller, - DataFactory: txFactory, - Processor: txProcessor, - Throttler: bicf.globalThrottler, - AntifloodHandler: bicf.antifloodHandler, - WhiteListRequest: bicf.whiteListHandler, - CurrentPeerId: bicf.mainMessenger.ID(), - PreferredPeersHolder: bicf.preferredPeersHolder, - ProcessedMessagesCacheMap: bicf.processedMessagesCacheMap, + Topic: topic, + Marshalizer: internalMarshaller, + DataFactory: txFactory, + Processor: txProcessor, + Throttler: bicf.globalThrottler, + AntifloodHandler: bicf.antifloodHandler, + WhiteListRequest: bicf.whiteListHandler, + CurrentPeerId: bicf.mainMessenger.ID(), + PreferredPeersHolder: bicf.preferredPeersHolder, + InterceptedDataVerifier: interceptedDataVerifier, }, ) if err != nil { @@ -389,7 +389,7 @@ func (bicf *baseInterceptorsContainerFactory) createOneRewardTxInterceptor(topic return nil, err } - err = bicf.createCacheForInterceptor(topic) + interceptedDataVerifier, err := bicf.createCacheForInterceptor(topic) if err != nil { return nil, err } @@ -397,16 +397,16 @@ func (bicf *baseInterceptorsContainerFactory) createOneRewardTxInterceptor(topic internalMarshaller := bicf.argInterceptorFactory.CoreComponents.InternalMarshalizer() interceptor, err := interceptors.NewMultiDataInterceptor( interceptors.ArgMultiDataInterceptor{ - Topic: topic, - Marshalizer: internalMarshaller, - DataFactory: txFactory, - Processor: txProcessor, - Throttler: bicf.globalThrottler, - AntifloodHandler: bicf.antifloodHandler, - WhiteListRequest: bicf.whiteListHandler, - CurrentPeerId: bicf.mainMessenger.ID(), - PreferredPeersHolder: bicf.preferredPeersHolder, - ProcessedMessagesCacheMap: bicf.processedMessagesCacheMap, + Topic: topic, + Marshalizer: internalMarshaller, + DataFactory: txFactory, + Processor: txProcessor, + Throttler: bicf.globalThrottler, + AntifloodHandler: bicf.antifloodHandler, + WhiteListRequest: bicf.whiteListHandler, + CurrentPeerId: bicf.mainMessenger.ID(), + PreferredPeersHolder: bicf.preferredPeersHolder, + InterceptedDataVerifier: interceptedDataVerifier, }, ) if err != nil { @@ -438,7 +438,7 @@ func (bicf *baseInterceptorsContainerFactory) generateHeaderInterceptors() error // compose header shard topic, for example: shardBlocks_0_META identifierHdr := factory.ShardBlocksTopic + shardC.CommunicationIdentifier(core.MetachainShardId) - err = bicf.createCacheForInterceptor(identifierHdr) + interceptedDataVerifier, err := bicf.createCacheForInterceptor(identifierHdr) if err != nil { return err } @@ -446,15 +446,15 @@ func (bicf *baseInterceptorsContainerFactory) generateHeaderInterceptors() error // only one intrashard header topic interceptor, err := interceptors.NewSingleDataInterceptor( interceptors.ArgSingleDataInterceptor{ - Topic: identifierHdr, - DataFactory: hdrFactory, - Processor: hdrProcessor, - Throttler: bicf.globalThrottler, - AntifloodHandler: bicf.antifloodHandler, - WhiteListRequest: bicf.whiteListHandler, - CurrentPeerId: bicf.mainMessenger.ID(), - PreferredPeersHolder: bicf.preferredPeersHolder, - ProcessedMessagesCacheMap: bicf.processedMessagesCacheMap, + Topic: identifierHdr, + DataFactory: hdrFactory, + Processor: hdrProcessor, + Throttler: bicf.globalThrottler, + AntifloodHandler: bicf.antifloodHandler, + WhiteListRequest: bicf.whiteListHandler, + CurrentPeerId: bicf.mainMessenger.ID(), + PreferredPeersHolder: bicf.preferredPeersHolder, + InterceptedDataVerifier: interceptedDataVerifier, }, ) if err != nil { @@ -532,23 +532,23 @@ func (bicf *baseInterceptorsContainerFactory) createOneMiniBlocksInterceptor(top return nil, err } - err = bicf.createCacheForInterceptor(topic) + interceptedDataVerifier, err := bicf.createCacheForInterceptor(topic) if err != nil { return nil, err } interceptor, err := interceptors.NewMultiDataInterceptor( interceptors.ArgMultiDataInterceptor{ - Topic: topic, - Marshalizer: internalMarshaller, - DataFactory: miniblockFactory, - Processor: miniblockProcessor, - Throttler: bicf.globalThrottler, - AntifloodHandler: bicf.antifloodHandler, - WhiteListRequest: bicf.whiteListHandler, - CurrentPeerId: bicf.mainMessenger.ID(), - PreferredPeersHolder: bicf.preferredPeersHolder, - ProcessedMessagesCacheMap: bicf.processedMessagesCacheMap, + Topic: topic, + Marshalizer: internalMarshaller, + DataFactory: miniblockFactory, + Processor: miniblockProcessor, + Throttler: bicf.globalThrottler, + AntifloodHandler: bicf.antifloodHandler, + WhiteListRequest: bicf.whiteListHandler, + CurrentPeerId: bicf.mainMessenger.ID(), + PreferredPeersHolder: bicf.preferredPeersHolder, + InterceptedDataVerifier: interceptedDataVerifier, }, ) if err != nil { @@ -577,7 +577,7 @@ func (bicf *baseInterceptorsContainerFactory) generateMetachainHeaderInterceptor return err } - err = bicf.createCacheForInterceptor(identifierHdr) + interceptedDataVerifier, err := bicf.createCacheForInterceptor(identifierHdr) if err != nil { return err } @@ -585,15 +585,15 @@ func (bicf *baseInterceptorsContainerFactory) generateMetachainHeaderInterceptor // only one metachain header topic interceptor, err := interceptors.NewSingleDataInterceptor( interceptors.ArgSingleDataInterceptor{ - Topic: identifierHdr, - DataFactory: hdrFactory, - Processor: hdrProcessor, - Throttler: bicf.globalThrottler, - AntifloodHandler: bicf.antifloodHandler, - WhiteListRequest: bicf.whiteListHandler, - CurrentPeerId: bicf.mainMessenger.ID(), - PreferredPeersHolder: bicf.preferredPeersHolder, - ProcessedMessagesCacheMap: bicf.processedMessagesCacheMap, + Topic: identifierHdr, + DataFactory: hdrFactory, + Processor: hdrProcessor, + Throttler: bicf.globalThrottler, + AntifloodHandler: bicf.antifloodHandler, + WhiteListRequest: bicf.whiteListHandler, + CurrentPeerId: bicf.mainMessenger.ID(), + PreferredPeersHolder: bicf.preferredPeersHolder, + InterceptedDataVerifier: interceptedDataVerifier, }, ) if err != nil { @@ -619,7 +619,7 @@ func (bicf *baseInterceptorsContainerFactory) createOneTrieNodesInterceptor(topi return nil, err } - err = bicf.createCacheForInterceptor(topic) + interceptedDataVerifier, err := bicf.createCacheForInterceptor(topic) if err != nil { return nil, err } @@ -627,16 +627,16 @@ func (bicf *baseInterceptorsContainerFactory) createOneTrieNodesInterceptor(topi internalMarshaller := bicf.argInterceptorFactory.CoreComponents.InternalMarshalizer() interceptor, err := interceptors.NewMultiDataInterceptor( interceptors.ArgMultiDataInterceptor{ - Topic: topic, - Marshalizer: internalMarshaller, - DataFactory: trieNodesFactory, - Processor: trieNodesProcessor, - Throttler: bicf.globalThrottler, - AntifloodHandler: bicf.antifloodHandler, - WhiteListRequest: bicf.whiteListHandler, - CurrentPeerId: bicf.mainMessenger.ID(), - PreferredPeersHolder: bicf.preferredPeersHolder, - ProcessedMessagesCacheMap: bicf.processedMessagesCacheMap, + Topic: topic, + Marshalizer: internalMarshaller, + DataFactory: trieNodesFactory, + Processor: trieNodesProcessor, + Throttler: bicf.globalThrottler, + AntifloodHandler: bicf.antifloodHandler, + WhiteListRequest: bicf.whiteListHandler, + CurrentPeerId: bicf.mainMessenger.ID(), + PreferredPeersHolder: bicf.preferredPeersHolder, + InterceptedDataVerifier: interceptedDataVerifier, }, ) if err != nil { @@ -717,23 +717,23 @@ func (bicf *baseInterceptorsContainerFactory) generatePeerAuthenticationIntercep return err } - err = bicf.createCacheForInterceptor(identifierPeerAuthentication) + interceptedDataVerifier, err := bicf.createCacheForInterceptor(identifierPeerAuthentication) if err != nil { return err } mdInterceptor, err := interceptors.NewMultiDataInterceptor( interceptors.ArgMultiDataInterceptor{ - Topic: identifierPeerAuthentication, - Marshalizer: internalMarshaller, - DataFactory: peerAuthenticationFactory, - Processor: peerAuthenticationProcessor, - Throttler: bicf.globalThrottler, - AntifloodHandler: bicf.antifloodHandler, - WhiteListRequest: bicf.whiteListHandler, - PreferredPeersHolder: bicf.preferredPeersHolder, - CurrentPeerId: bicf.mainMessenger.ID(), - ProcessedMessagesCacheMap: bicf.processedMessagesCacheMap, + Topic: identifierPeerAuthentication, + Marshalizer: internalMarshaller, + DataFactory: peerAuthenticationFactory, + Processor: peerAuthenticationProcessor, + Throttler: bicf.globalThrottler, + AntifloodHandler: bicf.antifloodHandler, + WhiteListRequest: bicf.whiteListHandler, + PreferredPeersHolder: bicf.preferredPeersHolder, + CurrentPeerId: bicf.mainMessenger.ID(), + InterceptedDataVerifier: interceptedDataVerifier, }, ) if err != nil { @@ -782,22 +782,22 @@ func (bicf *baseInterceptorsContainerFactory) createHeartbeatV2Interceptor( return nil, err } - err = bicf.createCacheForInterceptor(identifier) + interceptedDataVerifier, err := bicf.createCacheForInterceptor(identifier) if err != nil { return nil, err } interceptor, err := interceptors.NewSingleDataInterceptor( interceptors.ArgSingleDataInterceptor{ - Topic: identifier, - DataFactory: heartbeatFactory, - Processor: heartbeatProcessor, - Throttler: bicf.globalThrottler, - AntifloodHandler: bicf.antifloodHandler, - WhiteListRequest: bicf.whiteListHandler, - PreferredPeersHolder: bicf.preferredPeersHolder, - CurrentPeerId: bicf.mainMessenger.ID(), - ProcessedMessagesCacheMap: bicf.processedMessagesCacheMap, + Topic: identifier, + DataFactory: heartbeatFactory, + Processor: heartbeatProcessor, + Throttler: bicf.globalThrottler, + AntifloodHandler: bicf.antifloodHandler, + WhiteListRequest: bicf.whiteListHandler, + PreferredPeersHolder: bicf.preferredPeersHolder, + CurrentPeerId: bicf.mainMessenger.ID(), + InterceptedDataVerifier: interceptedDataVerifier, }, ) if err != nil { @@ -837,22 +837,22 @@ func (bicf *baseInterceptorsContainerFactory) createPeerShardInterceptor( return nil, err } - err = bicf.createCacheForInterceptor(identifier) + interceptedDataVerifier, err := bicf.createCacheForInterceptor(identifier) if err != nil { return nil, err } interceptor, err := interceptors.NewSingleDataInterceptor( interceptors.ArgSingleDataInterceptor{ - Topic: identifier, - DataFactory: interceptedPeerShardFactory, - Processor: psiProcessor, - Throttler: bicf.globalThrottler, - AntifloodHandler: bicf.antifloodHandler, - WhiteListRequest: bicf.whiteListHandler, - CurrentPeerId: bicf.mainMessenger.ID(), - PreferredPeersHolder: bicf.preferredPeersHolder, - ProcessedMessagesCacheMap: bicf.processedMessagesCacheMap, + Topic: identifier, + DataFactory: interceptedPeerShardFactory, + Processor: psiProcessor, + Throttler: bicf.globalThrottler, + AntifloodHandler: bicf.antifloodHandler, + WhiteListRequest: bicf.whiteListHandler, + CurrentPeerId: bicf.mainMessenger.ID(), + PreferredPeersHolder: bicf.preferredPeersHolder, + InterceptedDataVerifier: interceptedDataVerifier, }, ) if err != nil { @@ -880,23 +880,23 @@ func (bicf *baseInterceptorsContainerFactory) generateValidatorInfoInterceptor() return err } - err = bicf.createCacheForInterceptor(identifier) + interceptedDataVerifier, err := bicf.createCacheForInterceptor(identifier) if err != nil { return err } mdInterceptor, err := interceptors.NewMultiDataInterceptor( interceptors.ArgMultiDataInterceptor{ - Topic: identifier, - Marshalizer: internalMarshaller, - DataFactory: interceptedValidatorInfoFactory, - Processor: validatorInfoProcessor, - Throttler: bicf.globalThrottler, - AntifloodHandler: bicf.antifloodHandler, - WhiteListRequest: bicf.whiteListHandler, - PreferredPeersHolder: bicf.preferredPeersHolder, - CurrentPeerId: bicf.mainMessenger.ID(), - ProcessedMessagesCacheMap: bicf.processedMessagesCacheMap, + Topic: identifier, + Marshalizer: internalMarshaller, + DataFactory: interceptedValidatorInfoFactory, + Processor: validatorInfoProcessor, + Throttler: bicf.globalThrottler, + AntifloodHandler: bicf.antifloodHandler, + WhiteListRequest: bicf.whiteListHandler, + PreferredPeersHolder: bicf.preferredPeersHolder, + CurrentPeerId: bicf.mainMessenger.ID(), + InterceptedDataVerifier: interceptedDataVerifier, }, ) if err != nil { @@ -924,16 +924,22 @@ func (bicf *baseInterceptorsContainerFactory) createOneShardEquivalentProofsInte return nil, err } + interceptedDataVerifier, err := bicf.createCacheForInterceptor(topic) + if err != nil { + return nil, err + } + interceptor, err := interceptors.NewSingleDataInterceptor( interceptors.ArgSingleDataInterceptor{ - Topic: topic, - DataFactory: equivalentProofsFactory, - Processor: equivalentProofsProcessor, - Throttler: bicf.globalThrottler, - AntifloodHandler: bicf.antifloodHandler, - WhiteListRequest: bicf.whiteListHandler, - CurrentPeerId: bicf.mainMessenger.ID(), - PreferredPeersHolder: bicf.preferredPeersHolder, + Topic: topic, + DataFactory: equivalentProofsFactory, + Processor: equivalentProofsProcessor, + Throttler: bicf.globalThrottler, + AntifloodHandler: bicf.antifloodHandler, + WhiteListRequest: bicf.whiteListHandler, + CurrentPeerId: bicf.mainMessenger.ID(), + PreferredPeersHolder: bicf.preferredPeersHolder, + InterceptedDataVerifier: interceptedDataVerifier, }, ) if err != nil { @@ -956,15 +962,16 @@ func (bicf *baseInterceptorsContainerFactory) addInterceptorsToContainers(keys [ return bicf.fullArchiveContainer.AddMultiple(keys, interceptors) } -func (bicf *baseInterceptorsContainerFactory) createCacheForInterceptor(topic string) error { +func (bicf *baseInterceptorsContainerFactory) createCacheForInterceptor(topic string) (process.InterceptedDataVerifier, error) { internalCache, err := cache.NewTimeCacher(cache.ArgTimeCacher{ DefaultSpan: cacheDefaultSpan, CacheExpiry: cacheDefaultExpiry, }) if err != nil { - return err + return nil, err } bicf.processedMessagesCacheMap[topic] = internalCache - return nil + verifier := interceptors.NewInterceptedDataVerifier(internalCache) + return verifier, nil } diff --git a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go index 36490461001..42c85e85084 100644 --- a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go +++ b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go @@ -269,22 +269,22 @@ func (micf *metaInterceptorsContainerFactory) createOneShardHeaderInterceptor(to return nil, err } - err = micf.createCacheForInterceptor(topic) + interceptedDataVerifier, err := micf.createCacheForInterceptor(topic) if err != nil { return nil, err } interceptor, err := processInterceptors.NewSingleDataInterceptor( processInterceptors.ArgSingleDataInterceptor{ - Topic: topic, - DataFactory: hdrFactory, - Processor: hdrProcessor, - Throttler: micf.globalThrottler, - AntifloodHandler: micf.antifloodHandler, - WhiteListRequest: micf.whiteListHandler, - CurrentPeerId: micf.mainMessenger.ID(), - PreferredPeersHolder: micf.preferredPeersHolder, - ProcessedMessagesCacheMap: micf.processedMessagesCacheMap, + Topic: topic, + DataFactory: hdrFactory, + Processor: hdrProcessor, + Throttler: micf.globalThrottler, + AntifloodHandler: micf.antifloodHandler, + WhiteListRequest: micf.whiteListHandler, + CurrentPeerId: micf.mainMessenger.ID(), + PreferredPeersHolder: micf.preferredPeersHolder, + InterceptedDataVerifier: interceptedDataVerifier, }, ) if err != nil { diff --git a/process/interceptors/baseDataInterceptor.go b/process/interceptors/baseDataInterceptor.go index 0b89553acdb..cec00abd756 100644 --- a/process/interceptors/baseDataInterceptor.go +++ b/process/interceptors/baseDataInterceptor.go @@ -2,7 +2,6 @@ package interceptors import ( "bytes" - "fmt" "sync" "github.com/multiversx/mx-chain-core-go/core" @@ -10,19 +9,18 @@ import ( "github.com/multiversx/mx-chain-go/p2p" "github.com/multiversx/mx-chain-go/process" - "github.com/multiversx/mx-chain-go/storage" ) type baseDataInterceptor struct { - throttler process.InterceptorThrottler - antifloodHandler process.P2PAntifloodHandler - topic string - currentPeerId core.PeerID - processor process.InterceptorProcessor - mutDebugHandler sync.RWMutex - debugHandler process.InterceptedDebugger - preferredPeersHolder process.PreferredPeersHolderHandler - processedMessagesCacheMap map[string]storage.Cacher + throttler process.InterceptorThrottler + antifloodHandler process.P2PAntifloodHandler + topic string + currentPeerId core.PeerID + processor process.InterceptorProcessor + mutDebugHandler sync.RWMutex + debugHandler process.InterceptedDebugger + preferredPeersHolder process.PreferredPeersHolderHandler + interceptedDataVerifier process.InterceptedDataVerifier } func (bdi *baseDataInterceptor) preProcessMesage(message p2p.MessageP2P, fromConnectedPeer core.PeerID) error { @@ -123,23 +121,6 @@ func (bdi *baseDataInterceptor) receivedDebugInterceptedData(interceptedData pro bdi.mutDebugHandler.RUnlock() } -func (bdi *baseDataInterceptor) checkIfMessageHasBeenProcessed(interceptedData process.InterceptedData) error { - if len(interceptedData.Hash()) == 0 { - return nil - } - - cache, ok := bdi.processedMessagesCacheMap[bdi.topic] - if !ok { - return fmt.Errorf("cache for topic %q does not exist", bdi.topic) - } - - if has, _ := cache.HasOrAdd(interceptedData.Hash(), nil, 0); has { - return fmt.Errorf("processed intercepted data with hash: %s", interceptedData.Hash()) - } - - return nil -} - // SetInterceptedDebugHandler will set a new intercepted debug handler func (bdi *baseDataInterceptor) SetInterceptedDebugHandler(handler process.InterceptedDebugger) error { if check.IfNil(handler) { diff --git a/process/interceptors/interceptedDataCacherVerifier.go b/process/interceptors/interceptedDataCacherVerifier.go new file mode 100644 index 00000000000..fd71b28e3c0 --- /dev/null +++ b/process/interceptors/interceptedDataCacherVerifier.go @@ -0,0 +1,54 @@ +package interceptors + +import ( + "errors" + + "github.com/multiversx/mx-chain-go/process" + "github.com/multiversx/mx-chain-go/storage" +) + +type interceptedDataStatus int + +const ( + ValidInterceptedData interceptedDataStatus = iota + InvalidInterceptedData +) + +var ( + ErrInvalidInterceptedData = errors.New("invalid intercepted data") +) + +type interceptedDataVerifier struct { + cache storage.Cacher +} + +func NewInterceptedDataVerifier(cache storage.Cacher) *interceptedDataVerifier { + return &interceptedDataVerifier{cache: cache} +} + +func (idv *interceptedDataVerifier) Verify(interceptedData process.InterceptedData) error { + if len(interceptedData.Hash()) == 0 { + return nil + } + + if val, ok := idv.cache.Get(interceptedData.Hash()); ok { + if val == ValidInterceptedData { + return nil + } + + return ErrInvalidInterceptedData + } + + err := interceptedData.CheckValidity() + if err != nil { + idv.cache.Put(interceptedData.Hash(), InvalidInterceptedData, 8) + return ErrInvalidInterceptedData + } + + idv.cache.Put(interceptedData.Hash(), ValidInterceptedData, 100) + return nil +} + +func (idv *interceptedDataVerifier) IsInterfaceNil() bool { + return idv == nil +} diff --git a/process/interceptors/interceptedDataVerifier_test.go b/process/interceptors/interceptedDataVerifier_test.go new file mode 100644 index 00000000000..6d5d3e268bb --- /dev/null +++ b/process/interceptors/interceptedDataVerifier_test.go @@ -0,0 +1,47 @@ +package interceptors + +import ( + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/multiversx/mx-chain-go/storage/cache" + "github.com/multiversx/mx-chain-go/testscommon" +) + +func TestInterceptedDataVerifier_CheckValidity(t *testing.T) { + interceptedData := &testscommon.InterceptedDataStub{ + CheckValidityCalled: func() error { + return nil + }, + IsForCurrentShardCalled: func() bool { + return false + }, + HashCalled: func() []byte { + return []byte("hash") + }, + } + + span := 1 * time.Second + c, _ := cache.NewTimeCacher(cache.ArgTimeCacher{ + DefaultSpan: span, + CacheExpiry: time.Second, + }) + + verifier := NewInterceptedDataVerifier(c) + + err := verifier.Verify(interceptedData) + require.Nil(t, err) + + wg := sync.WaitGroup{} + + for i := 0; i < 3; i++ { + wg.Add(1) + err := verifier.Verify(interceptedData) + if err != nil { + return + } + } +} diff --git a/process/interceptors/multiDataInterceptor.go b/process/interceptors/multiDataInterceptor.go index 2b1a9335ead..96e0f17691c 100644 --- a/process/interceptors/multiDataInterceptor.go +++ b/process/interceptors/multiDataInterceptor.go @@ -1,6 +1,7 @@ package interceptors import ( + "fmt" "sync" "github.com/multiversx/mx-chain-core-go/core" @@ -15,23 +16,22 @@ import ( "github.com/multiversx/mx-chain-go/p2p" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/interceptors/disabled" - "github.com/multiversx/mx-chain-go/storage" ) var log = logger.GetOrCreate("process/interceptors") // ArgMultiDataInterceptor is the argument for the multi-data interceptor type ArgMultiDataInterceptor struct { - Topic string - Marshalizer marshal.Marshalizer - DataFactory process.InterceptedDataFactory - Processor process.InterceptorProcessor - Throttler process.InterceptorThrottler - AntifloodHandler process.P2PAntifloodHandler - WhiteListRequest process.WhiteListHandler - PreferredPeersHolder process.PreferredPeersHolderHandler - CurrentPeerId core.PeerID - ProcessedMessagesCacheMap map[string]storage.Cacher + Topic string + Marshalizer marshal.Marshalizer + DataFactory process.InterceptedDataFactory + Processor process.InterceptorProcessor + Throttler process.InterceptorThrottler + AntifloodHandler process.P2PAntifloodHandler + WhiteListRequest process.WhiteListHandler + PreferredPeersHolder process.PreferredPeersHolderHandler + CurrentPeerId core.PeerID + InterceptedDataVerifier process.InterceptedDataVerifier } // MultiDataInterceptor is used for intercepting packed multi data @@ -70,20 +70,23 @@ func NewMultiDataInterceptor(arg ArgMultiDataInterceptor) (*MultiDataInterceptor if check.IfNil(arg.PreferredPeersHolder) { return nil, process.ErrNilPreferredPeersHolder } + if check.IfNil(arg.InterceptedDataVerifier) { + return nil, process.ErrNilInterceptedDataVerifier + } if len(arg.CurrentPeerId) == 0 { return nil, process.ErrEmptyPeerID } multiDataIntercept := &MultiDataInterceptor{ baseDataInterceptor: &baseDataInterceptor{ - throttler: arg.Throttler, - antifloodHandler: arg.AntifloodHandler, - topic: arg.Topic, - currentPeerId: arg.CurrentPeerId, - processor: arg.Processor, - preferredPeersHolder: arg.PreferredPeersHolder, - debugHandler: handler.NewDisabledInterceptorDebugHandler(), - processedMessagesCacheMap: arg.ProcessedMessagesCacheMap, + throttler: arg.Throttler, + antifloodHandler: arg.AntifloodHandler, + topic: arg.Topic, + currentPeerId: arg.CurrentPeerId, + processor: arg.Processor, + preferredPeersHolder: arg.PreferredPeersHolder, + debugHandler: handler.NewDisabledInterceptorDebugHandler(), + interceptedDataVerifier: arg.InterceptedDataVerifier, }, marshalizer: arg.Marshalizer, factory: arg.DataFactory, @@ -157,19 +160,16 @@ func (mdi *MultiDataInterceptor) ProcessReceivedMessage(message p2p.MessageP2P, for index, dataBuff := range multiDataBuff { var interceptedData process.InterceptedData interceptedData, err = mdi.interceptedData(dataBuff, message.Peer(), fromConnectedPeer) + fmt.Println(err) - listInterceptedData[index] = interceptedData + if !errors.Is(err, ErrInvalidInterceptedData) { + listInterceptedData[index] = interceptedData + } if err != nil { mdi.throttler.EndProcessing() return err } - errCache := mdi.checkIfMessageHasBeenProcessed(interceptedData) - if errCache != nil { - mdi.throttler.EndProcessing() - continue - } - isWhiteListed := mdi.whiteListRequest.IsWhiteListed(interceptedData) if !isWhiteListed && errOriginator != nil { mdi.throttler.EndProcessing() @@ -219,7 +219,7 @@ func (mdi *MultiDataInterceptor) interceptedData(dataBuff []byte, originator cor mdi.receivedDebugInterceptedData(interceptedData) - err = interceptedData.CheckValidity() + err = mdi.interceptedDataVerifier.Verify(interceptedData) if err != nil { mdi.processDebugInterceptedData(interceptedData, err) diff --git a/process/interceptors/multiDataInterceptor_test.go b/process/interceptors/multiDataInterceptor_test.go index bc3599c4f0b..50e21498699 100644 --- a/process/interceptors/multiDataInterceptor_test.go +++ b/process/interceptors/multiDataInterceptor_test.go @@ -665,6 +665,9 @@ func TestMultiDataInterceptor_ProcessReceivedMessageIsOriginatorNotOkButWhiteLis IsForCurrentShardCalled: func() bool { return false }, + HashCalled: func() []byte { + return []byte("hash") + }, } whiteListHandler := &testscommon.WhiteListHandlerStub{ diff --git a/process/interceptors/singleDataInterceptor.go b/process/interceptors/singleDataInterceptor.go index 75038588a84..b23db11ff1f 100644 --- a/process/interceptors/singleDataInterceptor.go +++ b/process/interceptors/singleDataInterceptor.go @@ -2,6 +2,7 @@ package interceptors import ( "errors" + "fmt" "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" @@ -10,20 +11,19 @@ import ( "github.com/multiversx/mx-chain-go/debug/handler" "github.com/multiversx/mx-chain-go/p2p" "github.com/multiversx/mx-chain-go/process" - "github.com/multiversx/mx-chain-go/storage" ) // ArgSingleDataInterceptor is the argument for the single-data interceptor type ArgSingleDataInterceptor struct { - Topic string - DataFactory process.InterceptedDataFactory - Processor process.InterceptorProcessor - Throttler process.InterceptorThrottler - AntifloodHandler process.P2PAntifloodHandler - WhiteListRequest process.WhiteListHandler - PreferredPeersHolder process.PreferredPeersHolderHandler - CurrentPeerId core.PeerID - ProcessedMessagesCacheMap map[string]storage.Cacher + Topic string + DataFactory process.InterceptedDataFactory + Processor process.InterceptorProcessor + Throttler process.InterceptorThrottler + AntifloodHandler process.P2PAntifloodHandler + WhiteListRequest process.WhiteListHandler + PreferredPeersHolder process.PreferredPeersHolderHandler + CurrentPeerId core.PeerID + InterceptedDataVerifier process.InterceptedDataVerifier } // SingleDataInterceptor is used for intercepting packed multi data @@ -56,20 +56,24 @@ func NewSingleDataInterceptor(arg ArgSingleDataInterceptor) (*SingleDataIntercep if check.IfNil(arg.PreferredPeersHolder) { return nil, process.ErrNilPreferredPeersHolder } + if check.IfNil(arg.InterceptedDataVerifier) { + fmt.Println(arg.Topic) + return nil, process.ErrNilInterceptedDataVerifier + } if len(arg.CurrentPeerId) == 0 { return nil, process.ErrEmptyPeerID } singleDataIntercept := &SingleDataInterceptor{ baseDataInterceptor: &baseDataInterceptor{ - throttler: arg.Throttler, - antifloodHandler: arg.AntifloodHandler, - topic: arg.Topic, - currentPeerId: arg.CurrentPeerId, - processor: arg.Processor, - preferredPeersHolder: arg.PreferredPeersHolder, - debugHandler: handler.NewDisabledInterceptorDebugHandler(), - processedMessagesCacheMap: arg.ProcessedMessagesCacheMap, + throttler: arg.Throttler, + antifloodHandler: arg.AntifloodHandler, + topic: arg.Topic, + currentPeerId: arg.CurrentPeerId, + processor: arg.Processor, + preferredPeersHolder: arg.PreferredPeersHolder, + debugHandler: handler.NewDisabledInterceptorDebugHandler(), + interceptedDataVerifier: arg.InterceptedDataVerifier, }, factory: arg.DataFactory, whiteListRequest: arg.WhiteListRequest, @@ -99,13 +103,8 @@ func (sdi *SingleDataInterceptor) ProcessReceivedMessage(message p2p.MessageP2P, } sdi.receivedDebugInterceptedData(interceptedData) - err = sdi.checkIfMessageHasBeenProcessed(interceptedData) - if err != nil { - sdi.throttler.EndProcessing() - return err - } - - err = interceptedData.CheckValidity() + err = sdi.interceptedDataVerifier.Verify(interceptedData) + fmt.Println(err) if err != nil { sdi.throttler.EndProcessing() sdi.processDebugInterceptedData(interceptedData, err) diff --git a/process/interface.go b/process/interface.go index 2d1ff18e22a..d39e91f8e8d 100644 --- a/process/interface.go +++ b/process/interface.go @@ -1401,3 +1401,8 @@ type SentSignaturesTracker interface { ResetCountersForManagedBlockSigner(signerPk []byte) IsInterfaceNil() bool } + +type InterceptedDataVerifier interface { + Verify(interceptedData InterceptedData) error + IsInterfaceNil() bool +} diff --git a/process/transaction/interceptedTransaction_test.go b/process/transaction/interceptedTransaction_test.go index 1312f5cba4f..de5536ea886 100644 --- a/process/transaction/interceptedTransaction_test.go +++ b/process/transaction/interceptedTransaction_test.go @@ -612,7 +612,7 @@ func TestNewInterceptedTransaction_ShouldWork(t *testing.T) { assert.Equal(t, tx, txi.Transaction()) } -// ------- CheckValidity +// ------- Verify func TestInterceptedTransaction_CheckValidityNilSignatureShouldErr(t *testing.T) { t.Parallel() diff --git a/process/unsigned/interceptedUnsignedTransaction_test.go b/process/unsigned/interceptedUnsignedTransaction_test.go index b0c00e4982e..102b76c0975 100644 --- a/process/unsigned/interceptedUnsignedTransaction_test.go +++ b/process/unsigned/interceptedUnsignedTransaction_test.go @@ -11,13 +11,14 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data/smartContractResult" + logger "github.com/multiversx/mx-chain-logger-go" + "github.com/stretchr/testify/assert" + "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/process/unsigned" "github.com/multiversx/mx-chain-go/testscommon" "github.com/multiversx/mx-chain-go/testscommon/hashingMocks" - logger "github.com/multiversx/mx-chain-logger-go" - "github.com/stretchr/testify/assert" ) var senderShard = uint32(2) @@ -170,7 +171,7 @@ func TestNewInterceptedUnsignedTransaction_ShouldWork(t *testing.T) { assert.Nil(t, err) } -// ------- CheckValidity +// ------- Verify func TestInterceptedUnsignedTransaction_CheckValidityNilTxHashShouldErr(t *testing.T) { t.Parallel() diff --git a/update/factory/fullSyncInterceptors.go b/update/factory/fullSyncInterceptors.go index 0fe0298c4d6..ae39982c15e 100644 --- a/update/factory/fullSyncInterceptors.go +++ b/update/factory/fullSyncInterceptors.go @@ -2,11 +2,13 @@ package factory import ( "fmt" + "time" "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/core/throttler" "github.com/multiversx/mx-chain-core-go/marshal" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/process" @@ -19,6 +21,7 @@ import ( "github.com/multiversx/mx-chain-go/sharding" "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator" "github.com/multiversx/mx-chain-go/state" + "github.com/multiversx/mx-chain-go/storage/cache" "github.com/multiversx/mx-chain-go/update" "github.com/multiversx/mx-chain-go/update/disabled" ) @@ -349,15 +352,21 @@ func (ficf *fullSyncInterceptorsContainerFactory) createOneShardHeaderIntercepto return nil, err } + interceptedDataVerifier, err := ficf.createCacheForInterceptor(topic) + if err != nil { + return nil, err + } + interceptor, err := interceptors.NewSingleDataInterceptor( interceptors.ArgSingleDataInterceptor{ - Topic: topic, - DataFactory: hdrFactory, - Processor: hdrProcessor, - Throttler: ficf.globalThrottler, - AntifloodHandler: ficf.antifloodHandler, - WhiteListRequest: ficf.whiteListHandler, - CurrentPeerId: ficf.mainMessenger.ID(), + Topic: topic, + DataFactory: hdrFactory, + Processor: hdrProcessor, + Throttler: ficf.globalThrottler, + AntifloodHandler: ficf.antifloodHandler, + WhiteListRequest: ficf.whiteListHandler, + CurrentPeerId: ficf.mainMessenger.ID(), + InterceptedDataVerifier: interceptedDataVerifier, }, ) if err != nil { @@ -551,17 +560,23 @@ func (ficf *fullSyncInterceptorsContainerFactory) createOneTxInterceptor(topic s return nil, err } + interceptedDataVerifier, err := ficf.createCacheForInterceptor(topic) + if err != nil { + return nil, err + } + interceptor, err := interceptors.NewMultiDataInterceptor( interceptors.ArgMultiDataInterceptor{ - Topic: topic, - Marshalizer: ficf.argInterceptorFactory.CoreComponents.InternalMarshalizer(), - DataFactory: txFactory, - Processor: txProcessor, - Throttler: ficf.globalThrottler, - AntifloodHandler: ficf.antifloodHandler, - WhiteListRequest: ficf.whiteListHandler, - CurrentPeerId: ficf.mainMessenger.ID(), - PreferredPeersHolder: ficf.preferredPeersHolder, + Topic: topic, + Marshalizer: ficf.argInterceptorFactory.CoreComponents.InternalMarshalizer(), + DataFactory: txFactory, + Processor: txProcessor, + Throttler: ficf.globalThrottler, + AntifloodHandler: ficf.antifloodHandler, + WhiteListRequest: ficf.whiteListHandler, + CurrentPeerId: ficf.mainMessenger.ID(), + PreferredPeersHolder: ficf.preferredPeersHolder, + InterceptedDataVerifier: interceptedDataVerifier, }, ) if err != nil { @@ -586,17 +601,23 @@ func (ficf *fullSyncInterceptorsContainerFactory) createOneUnsignedTxInterceptor return nil, err } + interceptedDataVerifier, err := ficf.createCacheForInterceptor(topic) + if err != nil { + return nil, err + } + interceptor, err := interceptors.NewMultiDataInterceptor( interceptors.ArgMultiDataInterceptor{ - Topic: topic, - Marshalizer: ficf.argInterceptorFactory.CoreComponents.InternalMarshalizer(), - DataFactory: txFactory, - Processor: txProcessor, - Throttler: ficf.globalThrottler, - AntifloodHandler: ficf.antifloodHandler, - WhiteListRequest: ficf.whiteListHandler, - CurrentPeerId: ficf.mainMessenger.ID(), - PreferredPeersHolder: ficf.preferredPeersHolder, + Topic: topic, + Marshalizer: ficf.argInterceptorFactory.CoreComponents.InternalMarshalizer(), + DataFactory: txFactory, + Processor: txProcessor, + Throttler: ficf.globalThrottler, + AntifloodHandler: ficf.antifloodHandler, + WhiteListRequest: ficf.whiteListHandler, + CurrentPeerId: ficf.mainMessenger.ID(), + PreferredPeersHolder: ficf.preferredPeersHolder, + InterceptedDataVerifier: interceptedDataVerifier, }, ) if err != nil { @@ -621,17 +642,23 @@ func (ficf *fullSyncInterceptorsContainerFactory) createOneRewardTxInterceptor(t return nil, err } + interceptedDataVerifier, err := ficf.createCacheForInterceptor(topic) + if err != nil { + return nil, err + } + interceptor, err := interceptors.NewMultiDataInterceptor( interceptors.ArgMultiDataInterceptor{ - Topic: topic, - Marshalizer: ficf.argInterceptorFactory.CoreComponents.InternalMarshalizer(), - DataFactory: txFactory, - Processor: txProcessor, - Throttler: ficf.globalThrottler, - AntifloodHandler: ficf.antifloodHandler, - WhiteListRequest: ficf.whiteListHandler, - CurrentPeerId: ficf.mainMessenger.ID(), - PreferredPeersHolder: ficf.preferredPeersHolder, + Topic: topic, + Marshalizer: ficf.argInterceptorFactory.CoreComponents.InternalMarshalizer(), + DataFactory: txFactory, + Processor: txProcessor, + Throttler: ficf.globalThrottler, + AntifloodHandler: ficf.antifloodHandler, + WhiteListRequest: ficf.whiteListHandler, + CurrentPeerId: ficf.mainMessenger.ID(), + PreferredPeersHolder: ficf.preferredPeersHolder, + InterceptedDataVerifier: interceptedDataVerifier, }, ) if err != nil { @@ -694,16 +721,22 @@ func (ficf *fullSyncInterceptorsContainerFactory) createOneMiniBlocksInterceptor return nil, err } + interceptedDataVerifier, err := ficf.createCacheForInterceptor(topic) + if err != nil { + return nil, err + } + interceptor, err := interceptors.NewSingleDataInterceptor( interceptors.ArgSingleDataInterceptor{ - Topic: topic, - DataFactory: txFactory, - Processor: txBlockBodyProcessor, - Throttler: ficf.globalThrottler, - AntifloodHandler: ficf.antifloodHandler, - WhiteListRequest: ficf.whiteListHandler, - CurrentPeerId: ficf.mainMessenger.ID(), - PreferredPeersHolder: ficf.preferredPeersHolder, + Topic: topic, + DataFactory: txFactory, + Processor: txBlockBodyProcessor, + Throttler: ficf.globalThrottler, + AntifloodHandler: ficf.antifloodHandler, + WhiteListRequest: ficf.whiteListHandler, + CurrentPeerId: ficf.mainMessenger.ID(), + PreferredPeersHolder: ficf.preferredPeersHolder, + InterceptedDataVerifier: interceptedDataVerifier, }, ) if err != nil { @@ -733,17 +766,23 @@ func (ficf *fullSyncInterceptorsContainerFactory) generateMetachainHeaderInterce return err } + interceptedDataVerifier, err := ficf.createCacheForInterceptor(identifierHdr) + if err != nil { + return err + } + //only one metachain header topic interceptor, err := interceptors.NewSingleDataInterceptor( interceptors.ArgSingleDataInterceptor{ - Topic: identifierHdr, - DataFactory: hdrFactory, - Processor: hdrProcessor, - Throttler: ficf.globalThrottler, - AntifloodHandler: ficf.antifloodHandler, - WhiteListRequest: ficf.whiteListHandler, - CurrentPeerId: ficf.mainMessenger.ID(), - PreferredPeersHolder: ficf.preferredPeersHolder, + Topic: identifierHdr, + DataFactory: hdrFactory, + Processor: hdrProcessor, + Throttler: ficf.globalThrottler, + AntifloodHandler: ficf.antifloodHandler, + WhiteListRequest: ficf.whiteListHandler, + CurrentPeerId: ficf.mainMessenger.ID(), + PreferredPeersHolder: ficf.preferredPeersHolder, + InterceptedDataVerifier: interceptedDataVerifier, }, ) if err != nil { @@ -769,17 +808,23 @@ func (ficf *fullSyncInterceptorsContainerFactory) createOneTrieNodesInterceptor( return nil, err } + interceptedDataVerifier, err := ficf.createCacheForInterceptor(topic) + if err != nil { + return nil, err + } + interceptor, err := interceptors.NewMultiDataInterceptor( interceptors.ArgMultiDataInterceptor{ - Topic: topic, - Marshalizer: ficf.argInterceptorFactory.CoreComponents.InternalMarshalizer(), - DataFactory: trieNodesFactory, - Processor: trieNodesProcessor, - Throttler: ficf.globalThrottler, - AntifloodHandler: ficf.antifloodHandler, - WhiteListRequest: ficf.whiteListHandler, - CurrentPeerId: ficf.mainMessenger.ID(), - PreferredPeersHolder: ficf.preferredPeersHolder, + Topic: topic, + Marshalizer: ficf.argInterceptorFactory.CoreComponents.InternalMarshalizer(), + DataFactory: trieNodesFactory, + Processor: trieNodesProcessor, + Throttler: ficf.globalThrottler, + AntifloodHandler: ficf.antifloodHandler, + WhiteListRequest: ficf.whiteListHandler, + CurrentPeerId: ficf.mainMessenger.ID(), + PreferredPeersHolder: ficf.preferredPeersHolder, + InterceptedDataVerifier: interceptedDataVerifier, }, ) if err != nil { @@ -832,6 +877,20 @@ func (ficf *fullSyncInterceptorsContainerFactory) addInterceptorsToContainers(ke return ficf.fullArchiveContainer.AddMultiple(keys, interceptors) } +func (ficf *fullSyncInterceptorsContainerFactory) createCacheForInterceptor(topic string) (process.InterceptedDataVerifier, error) { + internalCache, err := cache.NewTimeCacher(cache.ArgTimeCacher{ + DefaultSpan: 30 * time.Second, + CacheExpiry: 30 * time.Second, + }) + if err != nil { + return nil, err + } + + //ficf.processedMessagesCacheMap[topic] = internalCache + verifier := interceptors.NewInterceptedDataVerifier(internalCache) + return verifier, nil +} + // IsInterfaceNil returns true if there is no value under the interface func (ficf *fullSyncInterceptorsContainerFactory) IsInterfaceNil() bool { return ficf == nil From a44129f88ea2318a260b8c0ac8ead1c7f8976a70 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Fri, 20 Sep 2024 12:16:54 +0300 Subject: [PATCH 14/37] cosmetic changes. --- ...Verifier.go => interceptedDataVerifier.go} | 4 + .../interceptedDataVerifier_test.go | 79 ++++++++++++++++--- process/interceptors/multiDataInterceptor.go | 2 - .../interceptors/multiDataInterceptor_test.go | 21 +++-- .../singleDataInterceptor_test.go | 38 +++++---- process/mock/interceptedDataVerifierMock.go | 25 +++--- 6 files changed, 118 insertions(+), 51 deletions(-) rename process/interceptors/{interceptedDataCacherVerifier.go => interceptedDataVerifier.go} (75%) diff --git a/process/interceptors/interceptedDataCacherVerifier.go b/process/interceptors/interceptedDataVerifier.go similarity index 75% rename from process/interceptors/interceptedDataCacherVerifier.go rename to process/interceptors/interceptedDataVerifier.go index fd71b28e3c0..d6c1019084f 100644 --- a/process/interceptors/interceptedDataCacherVerifier.go +++ b/process/interceptors/interceptedDataVerifier.go @@ -26,6 +26,9 @@ func NewInterceptedDataVerifier(cache storage.Cacher) *interceptedDataVerifier { return &interceptedDataVerifier{cache: cache} } +// Verify will check if the intercepted data has been validated before and put in the time cache. +// It will retrieve the status in the cache if it exists, otherwise it will validate it and store the status of the +// validation in the cache. Note that the entries are stored for a set period of time func (idv *interceptedDataVerifier) Verify(interceptedData process.InterceptedData) error { if len(interceptedData.Hash()) == 0 { return nil @@ -49,6 +52,7 @@ func (idv *interceptedDataVerifier) Verify(interceptedData process.InterceptedDa return nil } +// IsInterfaceNil returns true if there is no value under the interface func (idv *interceptedDataVerifier) IsInterfaceNil() bool { return idv == nil } diff --git a/process/interceptors/interceptedDataVerifier_test.go b/process/interceptors/interceptedDataVerifier_test.go index 6d5d3e268bb..da899764d00 100644 --- a/process/interceptors/interceptedDataVerifier_test.go +++ b/process/interceptors/interceptedDataVerifier_test.go @@ -1,17 +1,31 @@ package interceptors import ( + "errors" "sync" "testing" "time" + "github.com/multiversx/mx-chain-core-go/core/atomic" "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/storage/cache" "github.com/multiversx/mx-chain-go/testscommon" ) -func TestInterceptedDataVerifier_CheckValidity(t *testing.T) { +const defaultSpan = 1 * time.Second + +func defaultInterceptedDataVerifier(span time.Duration) process.InterceptedDataVerifier { + c, _ := cache.NewTimeCacher(cache.ArgTimeCacher{ + DefaultSpan: span, + CacheExpiry: span, + }) + + return NewInterceptedDataVerifier(c) +} + +func TestInterceptedDataVerifier_CheckValidityShouldWork(t *testing.T) { interceptedData := &testscommon.InterceptedDataStub{ CheckValidityCalled: func() error { return nil @@ -24,24 +38,63 @@ func TestInterceptedDataVerifier_CheckValidity(t *testing.T) { }, } - span := 1 * time.Second - c, _ := cache.NewTimeCacher(cache.ArgTimeCacher{ - DefaultSpan: span, - CacheExpiry: time.Second, - }) - - verifier := NewInterceptedDataVerifier(c) + verifier := defaultInterceptedDataVerifier(defaultSpan) err := verifier.Verify(interceptedData) require.Nil(t, err) + errCount := atomic.Counter{} wg := sync.WaitGroup{} - for i := 0; i < 3; i++ { wg.Add(1) - err := verifier.Verify(interceptedData) - if err != nil { - return - } + go func() { + defer wg.Done() + err := verifier.Verify(interceptedData) + if err != nil { + errCount.Add(1) + } + }() + } + wg.Wait() + + require.Equal(t, int64(0), errCount.Get()) +} + +func TestInterceptedDataVerifier_CheckValidityShouldNotWork(t *testing.T) { + interceptedData := &testscommon.InterceptedDataStub{ + CheckValidityCalled: func() error { + return nil + }, + IsForCurrentShardCalled: func() bool { + return false + }, + HashCalled: func() []byte { + return []byte("hash") + }, } + + interceptedDataWithErr := &testscommon.InterceptedDataStub{ + CheckValidityCalled: func() error { + return errors.New("error") + }, + IsForCurrentShardCalled: func() bool { + return false + }, + HashCalled: func() []byte { + return []byte("hash") + }, + } + + verifier := defaultInterceptedDataVerifier(defaultSpan) + + err := verifier.Verify(interceptedDataWithErr) + require.Equal(t, ErrInvalidInterceptedData, err) + + err = verifier.Verify(interceptedData) + require.NotNil(t, ErrInvalidInterceptedData, err) + + <-time.After(defaultSpan) + + err = verifier.Verify(interceptedData) + require.Nil(t, err) } diff --git a/process/interceptors/multiDataInterceptor.go b/process/interceptors/multiDataInterceptor.go index 96e0f17691c..2bf6f29918b 100644 --- a/process/interceptors/multiDataInterceptor.go +++ b/process/interceptors/multiDataInterceptor.go @@ -1,7 +1,6 @@ package interceptors import ( - "fmt" "sync" "github.com/multiversx/mx-chain-core-go/core" @@ -160,7 +159,6 @@ func (mdi *MultiDataInterceptor) ProcessReceivedMessage(message p2p.MessageP2P, for index, dataBuff := range multiDataBuff { var interceptedData process.InterceptedData interceptedData, err = mdi.interceptedData(dataBuff, message.Peer(), fromConnectedPeer) - fmt.Println(err) if !errors.Is(err, ErrInvalidInterceptedData) { listInterceptedData[index] = interceptedData diff --git a/process/interceptors/multiDataInterceptor_test.go b/process/interceptors/multiDataInterceptor_test.go index 50e21498699..7e69a5bd957 100644 --- a/process/interceptors/multiDataInterceptor_test.go +++ b/process/interceptors/multiDataInterceptor_test.go @@ -16,7 +16,6 @@ import ( "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/interceptors" "github.com/multiversx/mx-chain-go/process/mock" - "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/testscommon" "github.com/multiversx/mx-chain-go/testscommon/p2pmocks" ) @@ -25,16 +24,16 @@ var fromConnectedPeerId = core.PeerID("from connected peer Id") func createMockArgMultiDataInterceptor() interceptors.ArgMultiDataInterceptor { return interceptors.ArgMultiDataInterceptor{ - Topic: "test topic", - Marshalizer: &mock.MarshalizerMock{}, - DataFactory: &mock.InterceptedDataFactoryStub{}, - Processor: &mock.InterceptorProcessorStub{}, - Throttler: createMockThrottler(), - AntifloodHandler: &mock.P2PAntifloodHandlerStub{}, - WhiteListRequest: &testscommon.WhiteListHandlerStub{}, - PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, - CurrentPeerId: "pid", - ProcessedMessagesCacheMap: make(map[string]storage.Cacher), + Topic: "test topic", + Marshalizer: &mock.MarshalizerMock{}, + DataFactory: &mock.InterceptedDataFactoryStub{}, + Processor: &mock.InterceptorProcessorStub{}, + Throttler: createMockThrottler(), + AntifloodHandler: &mock.P2PAntifloodHandlerStub{}, + WhiteListRequest: &testscommon.WhiteListHandlerStub{}, + PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, + CurrentPeerId: "pid", + InterceptedDataVerifier: &mock.InterceptedDataVerifierStub{}, } } diff --git a/process/interceptors/singleDataInterceptor_test.go b/process/interceptors/singleDataInterceptor_test.go index f8d391d3471..4f7189bd6b2 100644 --- a/process/interceptors/singleDataInterceptor_test.go +++ b/process/interceptors/singleDataInterceptor_test.go @@ -16,23 +16,21 @@ import ( "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/interceptors" "github.com/multiversx/mx-chain-go/process/mock" - "github.com/multiversx/mx-chain-go/storage" - "github.com/multiversx/mx-chain-go/storage/cache" "github.com/multiversx/mx-chain-go/testscommon" "github.com/multiversx/mx-chain-go/testscommon/p2pmocks" ) func createMockArgSingleDataInterceptor() interceptors.ArgSingleDataInterceptor { return interceptors.ArgSingleDataInterceptor{ - Topic: "test topic", - DataFactory: &mock.InterceptedDataFactoryStub{}, - Processor: &mock.InterceptorProcessorStub{}, - Throttler: createMockThrottler(), - AntifloodHandler: &mock.P2PAntifloodHandlerStub{}, - WhiteListRequest: &testscommon.WhiteListHandlerStub{}, - PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, - CurrentPeerId: "pid", - ProcessedMessagesCacheMap: make(map[string]storage.Cacher), + Topic: "test topic", + DataFactory: &mock.InterceptedDataFactoryStub{}, + Processor: &mock.InterceptorProcessorStub{}, + Throttler: createMockThrottler(), + AntifloodHandler: &mock.P2PAntifloodHandlerStub{}, + WhiteListRequest: &testscommon.WhiteListHandlerStub{}, + PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, + CurrentPeerId: "pid", + //ProcessedMessagesCacheMap: make(map[string]storage.Cacher), } } @@ -63,6 +61,14 @@ func createMockThrottler() *mock.InterceptorThrottlerStub { } } +func createMockInterceptedDataVerifier() *mock.InterceptedDataVerifierStub { + return &mock.InterceptedDataVerifierStub{ + VerifyCalled: func(interceptedData process.InterceptedData) error { + return nil + }, + } +} + func TestNewSingleDataInterceptor_EmptyTopicShouldErr(t *testing.T) { t.Parallel() @@ -513,11 +519,11 @@ func TestSingleDataInterceptor_ProcessSameMessage(t *testing.T) { arg.WhiteListRequest = whiteListHandler span := 1 * time.Second - c, _ := cache.NewTimeCacher(cache.ArgTimeCacher{ - DefaultSpan: span, - CacheExpiry: time.Second, - }) - arg.ProcessedMessagesCacheMap[arg.Topic] = c + //c, _ := cache.NewTimeCacher(cache.ArgTimeCacher{ + // DefaultSpan: span, + // CacheExpiry: time.Second, + //}) + //arg.ProcessedMessagesCacheMap[arg.Topic] = c sdi, _ := interceptors.NewSingleDataInterceptor(arg) diff --git a/process/mock/interceptedDataVerifierMock.go b/process/mock/interceptedDataVerifierMock.go index c8d4d14392b..f6329cc408c 100644 --- a/process/mock/interceptedDataVerifierMock.go +++ b/process/mock/interceptedDataVerifierMock.go @@ -1,17 +1,24 @@ package mock -import "github.com/multiversx/mx-chain-go/process" +import ( + "github.com/multiversx/mx-chain-go/process" +) -// InterceptedDataVerifierMock - -type InterceptedDataVerifierMock struct { +// InterceptedDataVerifierStub - +type InterceptedDataVerifierStub struct { + VerifyCalled func(interceptedData process.InterceptedData) error } -// IsForCurrentShard - -func (i *InterceptedDataVerifierMock) IsForCurrentShard(_ process.InterceptedData) bool { - return true +// Verify - +func (idv *InterceptedDataVerifierStub) Verify(interceptedData process.InterceptedData) error { + if idv.VerifyCalled != nil { + return idv.VerifyCalled(interceptedData) + } + + return nil } -// IsInterfaceNil returns true if underlying object is -func (i *InterceptedDataVerifierMock) IsInterfaceNil() bool { - return i == nil +// IsInterfaceNil - +func (idv *InterceptedDataVerifierStub) IsInterfaceNil() bool { + return idv == nil } From afdceb13d5a4f3597d28326808f71d7283cf112e Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Fri, 20 Sep 2024 12:28:48 +0300 Subject: [PATCH 15/37] added intercepted data verifier stub in multi data tests. --- process/interceptors/multiDataInterceptor_test.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/process/interceptors/multiDataInterceptor_test.go b/process/interceptors/multiDataInterceptor_test.go index 7e69a5bd957..13b9dadae38 100644 --- a/process/interceptors/multiDataInterceptor_test.go +++ b/process/interceptors/multiDataInterceptor_test.go @@ -357,6 +357,11 @@ func testProcessReceiveMessageMultiData(t *testing.T, isForCurrentShard bool, ex } arg.Processor = createMockInterceptorStub(&checkCalledNum, &processCalledNum) arg.Throttler = throttler + arg.InterceptedDataVerifier = &mock.InterceptedDataVerifierStub{ + VerifyCalled: func(interceptedData process.InterceptedData) error { + return interceptedData.CheckValidity() + }, + } mdi, _ := interceptors.NewMultiDataInterceptor(arg) dataField, _ := marshalizer.Marshal(&batch.Batch{Data: buffData}) @@ -609,6 +614,11 @@ func processReceivedMessageMultiDataInvalidVersion(t *testing.T, expectedErr err return true }, } + arg.InterceptedDataVerifier = &mock.InterceptedDataVerifierStub{ + VerifyCalled: func(interceptedData process.InterceptedData) error { + return interceptedData.CheckValidity() + }, + } mdi, _ := interceptors.NewMultiDataInterceptor(arg) dataField, _ := marshalizer.Marshal(&batch.Batch{Data: buffData}) From d8d8113eabe4c3f9bcc7219db20fb2ee632bdb36 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Fri, 20 Sep 2024 13:05:13 +0300 Subject: [PATCH 16/37] fix single data interceptor tests. --- .../interceptedDataVerifier_test.go | 4 + .../singleDataInterceptor_test.go | 95 ++----------------- 2 files changed, 14 insertions(+), 85 deletions(-) diff --git a/process/interceptors/interceptedDataVerifier_test.go b/process/interceptors/interceptedDataVerifier_test.go index da899764d00..8cb7153b9fc 100644 --- a/process/interceptors/interceptedDataVerifier_test.go +++ b/process/interceptors/interceptedDataVerifier_test.go @@ -26,6 +26,8 @@ func defaultInterceptedDataVerifier(span time.Duration) process.InterceptedDataV } func TestInterceptedDataVerifier_CheckValidityShouldWork(t *testing.T) { + t.Parallel() + interceptedData := &testscommon.InterceptedDataStub{ CheckValidityCalled: func() error { return nil @@ -61,6 +63,8 @@ func TestInterceptedDataVerifier_CheckValidityShouldWork(t *testing.T) { } func TestInterceptedDataVerifier_CheckValidityShouldNotWork(t *testing.T) { + t.Parallel() + interceptedData := &testscommon.InterceptedDataStub{ CheckValidityCalled: func() error { return nil diff --git a/process/interceptors/singleDataInterceptor_test.go b/process/interceptors/singleDataInterceptor_test.go index 4f7189bd6b2..408d6d52078 100644 --- a/process/interceptors/singleDataInterceptor_test.go +++ b/process/interceptors/singleDataInterceptor_test.go @@ -2,8 +2,6 @@ package interceptors_test import ( "errors" - "strings" - "sync" "sync/atomic" "testing" "time" @@ -22,15 +20,15 @@ import ( func createMockArgSingleDataInterceptor() interceptors.ArgSingleDataInterceptor { return interceptors.ArgSingleDataInterceptor{ - Topic: "test topic", - DataFactory: &mock.InterceptedDataFactoryStub{}, - Processor: &mock.InterceptorProcessorStub{}, - Throttler: createMockThrottler(), - AntifloodHandler: &mock.P2PAntifloodHandlerStub{}, - WhiteListRequest: &testscommon.WhiteListHandlerStub{}, - PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, - CurrentPeerId: "pid", - //ProcessedMessagesCacheMap: make(map[string]storage.Cacher), + Topic: "test topic", + DataFactory: &mock.InterceptedDataFactoryStub{}, + Processor: &mock.InterceptorProcessorStub{}, + Throttler: createMockThrottler(), + AntifloodHandler: &mock.P2PAntifloodHandlerStub{}, + WhiteListRequest: &testscommon.WhiteListHandlerStub{}, + PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, + CurrentPeerId: "pid", + InterceptedDataVerifier: createMockInterceptedDataVerifier(), } } @@ -64,7 +62,7 @@ func createMockThrottler() *mock.InterceptorThrottlerStub { func createMockInterceptedDataVerifier() *mock.InterceptedDataVerifierStub { return &mock.InterceptedDataVerifierStub{ VerifyCalled: func(interceptedData process.InterceptedData) error { - return nil + return interceptedData.CheckValidity() }, } } @@ -480,79 +478,6 @@ func TestSingleDataInterceptor_Close(t *testing.T) { assert.Nil(t, err) } -func TestSingleDataInterceptor_ProcessSameMessage(t *testing.T) { - t.Parallel() - - checkCalledNum := int32(0) - processCalledNum := int32(0) - throttler := createMockThrottler() - interceptedData := &testscommon.InterceptedDataStub{ - HashCalled: func() []byte { - return []byte("hash") - }, - CheckValidityCalled: func() error { - return nil - }, - IsForCurrentShardCalled: func() bool { - return false - }, - } - - whiteListHandler := &testscommon.WhiteListHandlerStub{ - IsWhiteListedCalled: func(interceptedData process.InterceptedData) bool { - return true - }, - } - arg := createMockArgSingleDataInterceptor() - arg.DataFactory = &mock.InterceptedDataFactoryStub{ - CreateCalled: func(buff []byte) (data process.InterceptedData, e error) { - return interceptedData, nil - }, - } - arg.Processor = createMockInterceptorStub(&checkCalledNum, &processCalledNum) - arg.Throttler = throttler - arg.AntifloodHandler = &mock.P2PAntifloodHandlerStub{ - IsOriginatorEligibleForTopicCalled: func(pid core.PeerID, topic string) error { - return process.ErrOnlyValidatorsCanUseThisTopic - }, - } - arg.WhiteListRequest = whiteListHandler - - span := 1 * time.Second - //c, _ := cache.NewTimeCacher(cache.ArgTimeCacher{ - // DefaultSpan: span, - // CacheExpiry: time.Second, - //}) - //arg.ProcessedMessagesCacheMap[arg.Topic] = c - - sdi, _ := interceptors.NewSingleDataInterceptor(arg) - - msg := &p2pmocks.P2PMessageMock{ - DataField: []byte("data to be processed"), - } - - wg := sync.WaitGroup{} - errCount := atomic.Uint32{} - for i := 0; i < 3; i++ { - wg.Add(1) - go func() { - defer wg.Done() - - err := sdi.ProcessReceivedMessage(msg, fromConnectedPeerId, &p2pmocks.MessengerStub{}) - if err != nil && strings.Contains(err.Error(), "processed intercepted data with hash") { - errCount.Add(1) - } - }() - } - - wg.Wait() - require.Equal(t, uint32(2), errCount.Load()) - - <-time.After(span + time.Millisecond) - err := sdi.ProcessReceivedMessage(msg, fromConnectedPeerId, &p2pmocks.MessengerStub{}) - require.Nil(t, err) -} - //------- IsInterfaceNil func TestSingleDataInterceptor_IsInterfaceNil(t *testing.T) { From fe2082cda745a98cd352cc335641bbd2bff0e495 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Fri, 20 Sep 2024 13:45:05 +0300 Subject: [PATCH 17/37] commit debug strings for CI. --- process/interceptors/interceptedDataVerifier.go | 5 +++++ process/interceptors/interceptedDataVerifier_test.go | 5 +++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/process/interceptors/interceptedDataVerifier.go b/process/interceptors/interceptedDataVerifier.go index d6c1019084f..743da1d75f9 100644 --- a/process/interceptors/interceptedDataVerifier.go +++ b/process/interceptors/interceptedDataVerifier.go @@ -2,6 +2,7 @@ package interceptors import ( "errors" + "fmt" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/storage" @@ -36,18 +37,22 @@ func (idv *interceptedDataVerifier) Verify(interceptedData process.InterceptedDa if val, ok := idv.cache.Get(interceptedData.Hash()); ok { if val == ValidInterceptedData { + fmt.Println("it is in the cache: valid") return nil } + fmt.Println("it is in the cache: invalid") return ErrInvalidInterceptedData } err := interceptedData.CheckValidity() if err != nil { + fmt.Println("wasnt' in the cache: invalid") idv.cache.Put(interceptedData.Hash(), InvalidInterceptedData, 8) return ErrInvalidInterceptedData } + fmt.Println("wasnt' in the cache: valid") idv.cache.Put(interceptedData.Hash(), ValidInterceptedData, 100) return nil } diff --git a/process/interceptors/interceptedDataVerifier_test.go b/process/interceptors/interceptedDataVerifier_test.go index 8cb7153b9fc..93a15204e51 100644 --- a/process/interceptors/interceptedDataVerifier_test.go +++ b/process/interceptors/interceptedDataVerifier_test.go @@ -95,9 +95,10 @@ func TestInterceptedDataVerifier_CheckValidityShouldNotWork(t *testing.T) { require.Equal(t, ErrInvalidInterceptedData, err) err = verifier.Verify(interceptedData) - require.NotNil(t, ErrInvalidInterceptedData, err) + // It is still invalid because it has the same hash. + require.Equal(t, ErrInvalidInterceptedData, err) - <-time.After(defaultSpan) + <-time.After(defaultSpan + 1*time.Millisecond) err = verifier.Verify(interceptedData) require.Nil(t, err) From c6b7a0f0f5606c8107bedfc40c8c5842594b3760 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Fri, 20 Sep 2024 15:14:46 +0300 Subject: [PATCH 18/37] moved map of cachers in node processor. --- epochStart/bootstrap/process.go | 5 ++++ epochStart/bootstrap/syncEpochStartMeta.go | 7 +++++ factory/bootstrap/bootstrapComponents.go | 24 ++++++++------- factory/processing/processComponents.go | 6 ++++ node/nodeRunner.go | 30 ++++++++++++------- .../interceptors/interceptedDataVerifier.go | 5 ---- process/interceptors/singleDataInterceptor.go | 3 -- update/factory/exportHandlerFactory.go | 7 ++++- update/factory/fullSyncInterceptors.go | 5 +++- 9 files changed, 60 insertions(+), 32 deletions(-) diff --git a/epochStart/bootstrap/process.go b/epochStart/bootstrap/process.go index 31a73bc5680..ac6b7fb371b 100644 --- a/epochStart/bootstrap/process.go +++ b/epochStart/bootstrap/process.go @@ -153,6 +153,8 @@ type epochStartBootstrap struct { nodeType core.NodeType startEpoch uint32 shuffledOut bool + + interceptedDataCache map[string]storage.Cacher } type baseDataInStorage struct { @@ -191,6 +193,7 @@ type ArgsEpochStartBootstrap struct { NodeProcessingMode common.NodeProcessingMode StateStatsHandler common.StateStatisticsHandler NodesCoordinatorRegistryFactory nodesCoordinator.NodesCoordinatorRegistryFactory + InterceptedDataCache map[string]storage.Cacher } type dataToSync struct { @@ -243,6 +246,7 @@ func NewEpochStartBootstrap(args ArgsEpochStartBootstrap) (*epochStartBootstrap, stateStatsHandler: args.StateStatsHandler, startEpoch: args.GeneralConfig.EpochStartConfig.GenesisEpoch, nodesCoordinatorRegistryFactory: args.NodesCoordinatorRegistryFactory, + interceptedDataCache: args.InterceptedDataCache, } if epochStartProvider.prefsConfig.FullArchive { @@ -564,6 +568,7 @@ func (e *epochStartBootstrap) prepareComponentsToSyncFromNetwork() error { StartInEpochConfig: epochStartConfig, HeaderIntegrityVerifier: e.headerIntegrityVerifier, MetaBlockProcessor: metaBlockProcessor, + InterceptedDataCache: e.interceptedDataCache, } e.epochStartMetaBlockSyncer, err = NewEpochStartMetaSyncer(argsEpochStartSyncer) if err != nil { diff --git a/epochStart/bootstrap/syncEpochStartMeta.go b/epochStart/bootstrap/syncEpochStartMeta.go index 450def2882e..bf9672a2655 100644 --- a/epochStart/bootstrap/syncEpochStartMeta.go +++ b/epochStart/bootstrap/syncEpochStartMeta.go @@ -18,6 +18,7 @@ import ( "github.com/multiversx/mx-chain-go/process/interceptors" interceptorsFactory "github.com/multiversx/mx-chain-go/process/interceptors/factory" "github.com/multiversx/mx-chain-go/sharding" + "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/storage/cache" ) @@ -45,6 +46,7 @@ type ArgsNewEpochStartMetaSyncer struct { ArgsParser process.ArgumentsParser HeaderIntegrityVerifier process.HeaderIntegrityVerifier MetaBlockProcessor EpochStartMetaBlockInterceptorProcessor + InterceptedDataCache map[string]storage.Cacher } // NewEpochStartMetaSyncer will return a new instance of epochStartMetaSyncer @@ -91,10 +93,15 @@ func NewEpochStartMetaSyncer(args ArgsNewEpochStartMetaSyncer) (*epochStartMetaS return nil, err } + //TODO: maybe move this into a function internalCache, err := cache.NewTimeCacher(cache.ArgTimeCacher{ DefaultSpan: 30 * time.Second, CacheExpiry: 30 * time.Second, }) + if err != nil { + return nil, err + } + args.InterceptedDataCache[factory.MetachainBlocksTopic] = internalCache interceptedDataVerifier := interceptors.NewInterceptedDataVerifier(internalCache) e.singleDataInterceptor, err = interceptors.NewSingleDataInterceptor( diff --git a/factory/bootstrap/bootstrapComponents.go b/factory/bootstrap/bootstrapComponents.go index a9ef7851ccb..cb325cb2e71 100644 --- a/factory/bootstrap/bootstrapComponents.go +++ b/factory/bootstrap/bootstrapComponents.go @@ -6,6 +6,8 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" + logger "github.com/multiversx/mx-chain-logger-go" + nodeFactory "github.com/multiversx/mx-chain-go/cmd/node/factory" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/config" @@ -24,23 +26,23 @@ import ( storageFactory "github.com/multiversx/mx-chain-go/storage/factory" "github.com/multiversx/mx-chain-go/storage/latestData" "github.com/multiversx/mx-chain-go/storage/storageunit" - logger "github.com/multiversx/mx-chain-logger-go" ) var log = logger.GetOrCreate("factory") // BootstrapComponentsFactoryArgs holds the arguments needed to create a bootstrap components factory type BootstrapComponentsFactoryArgs struct { - Config config.Config - RoundConfig config.RoundConfig - PrefConfig config.Preferences - ImportDbConfig config.ImportDbConfig - FlagsConfig config.ContextFlagsConfig - WorkingDir string - CoreComponents factory.CoreComponentsHolder - CryptoComponents factory.CryptoComponentsHolder - NetworkComponents factory.NetworkComponentsHolder - StatusCoreComponents factory.StatusCoreComponentsHolder + Config config.Config + RoundConfig config.RoundConfig + PrefConfig config.Preferences + ImportDbConfig config.ImportDbConfig + FlagsConfig config.ContextFlagsConfig + WorkingDir string + CoreComponents factory.CoreComponentsHolder + CryptoComponents factory.CryptoComponentsHolder + NetworkComponents factory.NetworkComponentsHolder + StatusCoreComponents factory.StatusCoreComponentsHolder + InterceptedDataCacheMap map[string]storage.Cacher } type bootstrapComponentsFactory struct { diff --git a/factory/processing/processComponents.go b/factory/processing/processComponents.go index 1c8d2e00de5..1df74b3f9dd 100644 --- a/factory/processing/processComponents.go +++ b/factory/processing/processComponents.go @@ -168,6 +168,8 @@ type ProcessComponentsFactoryArgs struct { GenesisNonce uint64 GenesisRound uint64 + + InterceptedDataCacheMap map[string]storage.Cacher } type processComponentsFactory struct { @@ -208,6 +210,8 @@ type processComponentsFactory struct { genesisNonce uint64 genesisRound uint64 + + interceptedDataCacheMap map[string]storage.Cacher } // NewProcessComponentsFactory will return a new instance of processComponentsFactory @@ -248,6 +252,7 @@ func NewProcessComponentsFactory(args ProcessComponentsFactoryArgs) (*processCom genesisNonce: args.GenesisNonce, genesisRound: args.GenesisRound, roundConfig: args.RoundConfig, + interceptedDataCacheMap: args.InterceptedDataCacheMap, }, nil } @@ -1858,6 +1863,7 @@ func (pcf *processComponentsFactory) createExportFactoryHandler( NumConcurrentTrieSyncers: pcf.config.TrieSync.NumConcurrentTrieSyncers, TrieSyncerVersion: pcf.config.TrieSync.TrieSyncerVersion, NodeOperationMode: nodeOperationMode, + InterceptedDataCacheMap: pcf.interceptedDataCacheMap, } return updateFactory.NewExportHandlerFactory(argsExporter) } diff --git a/node/nodeRunner.go b/node/nodeRunner.go index 1378007ad64..23dd7a6fc15 100644 --- a/node/nodeRunner.go +++ b/node/nodeRunner.go @@ -20,6 +20,8 @@ import ( "github.com/multiversx/mx-chain-core-go/core/throttler" "github.com/multiversx/mx-chain-core-go/data/endProcess" outportCore "github.com/multiversx/mx-chain-core-go/data/outport" + logger "github.com/multiversx/mx-chain-logger-go" + "github.com/multiversx/mx-chain-go/api/gin" "github.com/multiversx/mx-chain-go/api/shared" "github.com/multiversx/mx-chain-go/common" @@ -56,12 +58,12 @@ import ( "github.com/multiversx/mx-chain-go/process/interceptors" "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator" "github.com/multiversx/mx-chain-go/state/syncer" + "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/storage/cache" storageFactory "github.com/multiversx/mx-chain-go/storage/factory" "github.com/multiversx/mx-chain-go/storage/storageunit" trieStatistics "github.com/multiversx/mx-chain-go/trie/statistics" "github.com/multiversx/mx-chain-go/update/trigger" - logger "github.com/multiversx/mx-chain-logger-go" ) type nextOperationForNode int @@ -321,7 +323,8 @@ func (nr *nodeRunner) executeOneComponentCreationCycle( } log.Debug("creating bootstrap components") - managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents) + interceptedDataCache := make(map[string]storage.Cacher) + managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents, interceptedDataCache) if err != nil { return true, err } @@ -432,6 +435,7 @@ func (nr *nodeRunner) executeOneComponentCreationCycle( managedStatusCoreComponents, gasScheduleNotifier, nodesCoordinatorInstance, + interceptedDataCache, ) if err != nil { return true, err @@ -1157,6 +1161,7 @@ func (nr *nodeRunner) CreateManagedProcessComponents( statusCoreComponents mainFactory.StatusCoreComponentsHolder, gasScheduleNotifier core.GasScheduleNotifier, nodesCoordinator nodesCoordinator.NodesCoordinator, + interceptedDataCacheMap map[string]storage.Cacher, ) (mainFactory.ProcessComponentsHandler, error) { configs := nr.configs configurationPaths := nr.configs.ConfigurationPathsHolder @@ -1264,6 +1269,7 @@ func (nr *nodeRunner) CreateManagedProcessComponents( HistoryRepo: historyRepository, FlagsConfig: *configs.FlagsConfig, TxExecutionOrderHandler: txExecutionOrderHandler, + InterceptedDataCacheMap: interceptedDataCacheMap, } processComponentsFactory, err := processComp.NewProcessComponentsFactory(processArgs) if err != nil { @@ -1377,18 +1383,20 @@ func (nr *nodeRunner) CreateManagedBootstrapComponents( coreComponents mainFactory.CoreComponentsHolder, cryptoComponents mainFactory.CryptoComponentsHolder, networkComponents mainFactory.NetworkComponentsHolder, + interceptedDataCacheMap map[string]storage.Cacher, ) (mainFactory.BootstrapComponentsHandler, error) { bootstrapComponentsFactoryArgs := bootstrapComp.BootstrapComponentsFactoryArgs{ - Config: *nr.configs.GeneralConfig, - PrefConfig: *nr.configs.PreferencesConfig, - ImportDbConfig: *nr.configs.ImportDbConfig, - FlagsConfig: *nr.configs.FlagsConfig, - WorkingDir: nr.configs.FlagsConfig.DbDir, - CoreComponents: coreComponents, - CryptoComponents: cryptoComponents, - NetworkComponents: networkComponents, - StatusCoreComponents: statusCoreComponents, + Config: *nr.configs.GeneralConfig, + PrefConfig: *nr.configs.PreferencesConfig, + ImportDbConfig: *nr.configs.ImportDbConfig, + FlagsConfig: *nr.configs.FlagsConfig, + WorkingDir: nr.configs.FlagsConfig.DbDir, + CoreComponents: coreComponents, + CryptoComponents: cryptoComponents, + NetworkComponents: networkComponents, + StatusCoreComponents: statusCoreComponents, + InterceptedDataCacheMap: interceptedDataCacheMap, } bootstrapComponentsFactory, err := bootstrapComp.NewBootstrapComponentsFactory(bootstrapComponentsFactoryArgs) diff --git a/process/interceptors/interceptedDataVerifier.go b/process/interceptors/interceptedDataVerifier.go index 743da1d75f9..d6c1019084f 100644 --- a/process/interceptors/interceptedDataVerifier.go +++ b/process/interceptors/interceptedDataVerifier.go @@ -2,7 +2,6 @@ package interceptors import ( "errors" - "fmt" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/storage" @@ -37,22 +36,18 @@ func (idv *interceptedDataVerifier) Verify(interceptedData process.InterceptedDa if val, ok := idv.cache.Get(interceptedData.Hash()); ok { if val == ValidInterceptedData { - fmt.Println("it is in the cache: valid") return nil } - fmt.Println("it is in the cache: invalid") return ErrInvalidInterceptedData } err := interceptedData.CheckValidity() if err != nil { - fmt.Println("wasnt' in the cache: invalid") idv.cache.Put(interceptedData.Hash(), InvalidInterceptedData, 8) return ErrInvalidInterceptedData } - fmt.Println("wasnt' in the cache: valid") idv.cache.Put(interceptedData.Hash(), ValidInterceptedData, 100) return nil } diff --git a/process/interceptors/singleDataInterceptor.go b/process/interceptors/singleDataInterceptor.go index b23db11ff1f..7e5a4257fd6 100644 --- a/process/interceptors/singleDataInterceptor.go +++ b/process/interceptors/singleDataInterceptor.go @@ -2,7 +2,6 @@ package interceptors import ( "errors" - "fmt" "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" @@ -57,7 +56,6 @@ func NewSingleDataInterceptor(arg ArgSingleDataInterceptor) (*SingleDataIntercep return nil, process.ErrNilPreferredPeersHolder } if check.IfNil(arg.InterceptedDataVerifier) { - fmt.Println(arg.Topic) return nil, process.ErrNilInterceptedDataVerifier } if len(arg.CurrentPeerId) == 0 { @@ -104,7 +102,6 @@ func (sdi *SingleDataInterceptor) ProcessReceivedMessage(message p2p.MessageP2P, sdi.receivedDebugInterceptedData(interceptedData) err = sdi.interceptedDataVerifier.Verify(interceptedData) - fmt.Println(err) if err != nil { sdi.throttler.EndProcessing() sdi.processDebugInterceptedData(interceptedData, err) diff --git a/update/factory/exportHandlerFactory.go b/update/factory/exportHandlerFactory.go index c13f25f3f5a..f743c6f7bbe 100644 --- a/update/factory/exportHandlerFactory.go +++ b/update/factory/exportHandlerFactory.go @@ -8,6 +8,8 @@ import ( "time" "github.com/multiversx/mx-chain-core-go/core/check" + logger "github.com/multiversx/mx-chain-logger-go" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/config" "github.com/multiversx/mx-chain-go/dataRetriever" @@ -30,7 +32,6 @@ import ( "github.com/multiversx/mx-chain-go/update/genesis" "github.com/multiversx/mx-chain-go/update/storing" "github.com/multiversx/mx-chain-go/update/sync" - logger "github.com/multiversx/mx-chain-logger-go" ) var log = logger.GetOrCreate("update/factory") @@ -69,6 +70,7 @@ type ArgsExporter struct { TrieSyncerVersion int CheckNodesOnDisk bool NodeOperationMode common.NodeOperation + InterceptedDataCacheMap map[string]storage.Cacher } type exportHandlerFactory struct { @@ -108,6 +110,7 @@ type exportHandlerFactory struct { trieSyncerVersion int checkNodesOnDisk bool nodeOperationMode common.NodeOperation + interceptedDataCacheMap map[string]storage.Cacher } // NewExportHandlerFactory creates an exporter factory @@ -266,6 +269,7 @@ func NewExportHandlerFactory(args ArgsExporter) (*exportHandlerFactory, error) { checkNodesOnDisk: args.CheckNodesOnDisk, statusCoreComponents: args.StatusCoreComponents, nodeOperationMode: args.NodeOperationMode, + interceptedDataCacheMap: args.InterceptedDataCacheMap, } return e, nil @@ -588,6 +592,7 @@ func (e *exportHandlerFactory) createInterceptors() error { FullArchiveInterceptorsContainer: e.fullArchiveInterceptorsContainer, AntifloodHandler: e.networkComponents.InputAntiFloodHandler(), NodeOperationMode: e.nodeOperationMode, + InterceptedDataCache: e.interceptedDataCacheMap, } fullSyncInterceptors, err := NewFullSyncInterceptorsContainerFactory(argsInterceptors) if err != nil { diff --git a/update/factory/fullSyncInterceptors.go b/update/factory/fullSyncInterceptors.go index ae39982c15e..358056a7228 100644 --- a/update/factory/fullSyncInterceptors.go +++ b/update/factory/fullSyncInterceptors.go @@ -21,6 +21,7 @@ import ( "github.com/multiversx/mx-chain-go/sharding" "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator" "github.com/multiversx/mx-chain-go/state" + "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/storage/cache" "github.com/multiversx/mx-chain-go/update" "github.com/multiversx/mx-chain-go/update/disabled" @@ -51,6 +52,7 @@ type fullSyncInterceptorsContainerFactory struct { antifloodHandler process.P2PAntifloodHandler preferredPeersHolder update.PreferredPeersHolderHandler nodeOperationMode common.NodeOperation + interceptedDataCache map[string]storage.Cacher } // ArgsNewFullSyncInterceptorsContainerFactory holds the arguments needed for fullSyncInterceptorsContainerFactory @@ -78,6 +80,7 @@ type ArgsNewFullSyncInterceptorsContainerFactory struct { FullArchiveInterceptorsContainer process.InterceptorsContainer AntifloodHandler process.P2PAntifloodHandler NodeOperationMode common.NodeOperation + InterceptedDataCache map[string]storage.Cacher } // NewFullSyncInterceptorsContainerFactory is responsible for creating a new interceptors factory object @@ -886,7 +889,7 @@ func (ficf *fullSyncInterceptorsContainerFactory) createCacheForInterceptor(topi return nil, err } - //ficf.processedMessagesCacheMap[topic] = internalCache + ficf.interceptedDataCache[topic] = internalCache verifier := interceptors.NewInterceptedDataVerifier(internalCache) return verifier, nil } From f59fc85275642ced3e8d9d6e9919a772fab57376 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Fri, 20 Sep 2024 15:20:05 +0300 Subject: [PATCH 19/37] cosmetic changes. --- epochStart/bootstrap/syncEpochStartMeta.go | 40 +++++++++++++++------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/epochStart/bootstrap/syncEpochStartMeta.go b/epochStart/bootstrap/syncEpochStartMeta.go index bf9672a2655..bebd8cc68cd 100644 --- a/epochStart/bootstrap/syncEpochStartMeta.go +++ b/epochStart/bootstrap/syncEpochStartMeta.go @@ -22,15 +22,21 @@ import ( "github.com/multiversx/mx-chain-go/storage/cache" ) +const ( + cacheDefaultSpan = 30 * time.Second + cacheDefaultExpiry = 30 * time.Second +) + var _ epochStart.StartOfEpochMetaSyncer = (*epochStartMetaSyncer)(nil) type epochStartMetaSyncer struct { - requestHandler RequestHandler - messenger Messenger - marshalizer marshal.Marshalizer - hasher hashing.Hasher - singleDataInterceptor process.Interceptor - metaBlockProcessor EpochStartMetaBlockInterceptorProcessor + requestHandler RequestHandler + messenger Messenger + marshalizer marshal.Marshalizer + hasher hashing.Hasher + singleDataInterceptor process.Interceptor + metaBlockProcessor EpochStartMetaBlockInterceptorProcessor + interceptedDataCacheMap map[string]storage.Cacher } // ArgsNewEpochStartMetaSyncer - @@ -93,16 +99,10 @@ func NewEpochStartMetaSyncer(args ArgsNewEpochStartMetaSyncer) (*epochStartMetaS return nil, err } - //TODO: maybe move this into a function - internalCache, err := cache.NewTimeCacher(cache.ArgTimeCacher{ - DefaultSpan: 30 * time.Second, - CacheExpiry: 30 * time.Second, - }) + interceptedDataVerifier, err := e.createCacheForInterceptor(factory.MetachainBlocksTopic) if err != nil { return nil, err } - args.InterceptedDataCache[factory.MetachainBlocksTopic] = internalCache - interceptedDataVerifier := interceptors.NewInterceptedDataVerifier(internalCache) e.singleDataInterceptor, err = interceptors.NewSingleDataInterceptor( interceptors.ArgSingleDataInterceptor{ @@ -168,6 +168,20 @@ func (e *epochStartMetaSyncer) initTopicForEpochStartMetaBlockInterceptor() erro return nil } +func (e *epochStartMetaSyncer) createCacheForInterceptor(topic string) (process.InterceptedDataVerifier, error) { + internalCache, err := cache.NewTimeCacher(cache.ArgTimeCacher{ + DefaultSpan: cacheDefaultSpan, + CacheExpiry: cacheDefaultExpiry, + }) + if err != nil { + return nil, err + } + + e.interceptedDataCacheMap[topic] = internalCache + verifier := interceptors.NewInterceptedDataVerifier(internalCache) + return verifier, nil +} + // IsInterfaceNil returns true if underlying object is nil func (e *epochStartMetaSyncer) IsInterfaceNil() bool { return e == nil From d00c7aba8ca98328c1b785e24254acb1e9514c23 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Fri, 20 Sep 2024 15:35:15 +0300 Subject: [PATCH 20/37] fix integration tests. --- epochStart/bootstrap/storageProcess_test.go | 1 + .../bootstrapComponents/bootstrapComponents_test.go | 7 +++++-- .../consensusComponents/consensusComponents_test.go | 8 ++++++-- .../factory/dataComponents/dataComponents_test.go | 7 +++++-- .../heartbeatComponents/heartbeatComponents_test.go | 8 ++++++-- .../factory/processComponents/processComponents_test.go | 8 ++++++-- .../factory/stateComponents/stateComponents_test.go | 7 +++++-- .../factory/statusComponents/statusComponents_test.go | 8 ++++++-- 8 files changed, 40 insertions(+), 14 deletions(-) diff --git a/epochStart/bootstrap/storageProcess_test.go b/epochStart/bootstrap/storageProcess_test.go index c61ef9f279f..755c6155421 100644 --- a/epochStart/bootstrap/storageProcess_test.go +++ b/epochStart/bootstrap/storageProcess_test.go @@ -128,6 +128,7 @@ func TestStorageEpochStartBootstrap_BootstrapMetablockNotFound(t *testing.T) { } args.GeneralConfig = testscommon.GetGeneralConfig() args.GeneralConfig.EpochStartConfig.RoundsPerEpoch = roundsPerEpoch + args.InterceptedDataCache = make(map[string]storage.Cacher) sesb, _ := NewStorageEpochStartBootstrap(args) params, err := sesb.Bootstrap() diff --git a/integrationTests/factory/bootstrapComponents/bootstrapComponents_test.go b/integrationTests/factory/bootstrapComponents/bootstrapComponents_test.go index 6c525ff9f12..62cbd9434a8 100644 --- a/integrationTests/factory/bootstrapComponents/bootstrapComponents_test.go +++ b/integrationTests/factory/bootstrapComponents/bootstrapComponents_test.go @@ -6,10 +6,12 @@ import ( "time" "github.com/multiversx/mx-chain-core-go/data/endProcess" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-go/integrationTests/factory" "github.com/multiversx/mx-chain-go/node" + "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/testscommon/goroutines" - "github.com/stretchr/testify/require" ) // ------------ Test BootstrapComponents -------------------- @@ -36,7 +38,8 @@ func TestBootstrapComponents_Create_Close_ShouldWork(t *testing.T) { require.Nil(t, err) managedNetworkComponents, err := nr.CreateManagedNetworkComponents(managedCoreComponents, managedStatusCoreComponents, managedCryptoComponents) require.Nil(t, err) - managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents) + interceptedDataCacheMap := make(map[string]storage.Cacher) + managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents, interceptedDataCacheMap) require.Nil(t, err) require.NotNil(t, managedBootstrapComponents) diff --git a/integrationTests/factory/consensusComponents/consensusComponents_test.go b/integrationTests/factory/consensusComponents/consensusComponents_test.go index 1e32c0c574b..926f1deac78 100644 --- a/integrationTests/factory/consensusComponents/consensusComponents_test.go +++ b/integrationTests/factory/consensusComponents/consensusComponents_test.go @@ -6,13 +6,15 @@ import ( "time" "github.com/multiversx/mx-chain-core-go/data/endProcess" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-go/common/forking" "github.com/multiversx/mx-chain-go/dataRetriever" bootstrapComp "github.com/multiversx/mx-chain-go/factory/bootstrap" "github.com/multiversx/mx-chain-go/integrationTests/factory" "github.com/multiversx/mx-chain-go/node" + "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/testscommon/goroutines" - "github.com/stretchr/testify/require" ) // ------------ Test TestConsensusComponents -------------------- @@ -39,7 +41,8 @@ func TestConsensusComponents_Close_ShouldWork(t *testing.T) { require.Nil(t, err) managedNetworkComponents, err := nr.CreateManagedNetworkComponents(managedCoreComponents, managedStatusCoreComponents, managedCryptoComponents) require.Nil(t, err) - managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents) + interceptedDataCacheMap := make(map[string]storage.Cacher) + managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents, interceptedDataCacheMap) require.Nil(t, err) managedDataComponents, err := nr.CreateManagedDataComponents(managedStatusCoreComponents, managedCoreComponents, managedBootstrapComponents, managedCryptoComponents) require.Nil(t, err) @@ -103,6 +106,7 @@ func TestConsensusComponents_Close_ShouldWork(t *testing.T) { managedStatusCoreComponents, gasScheduleNotifier, nodesCoordinator, + interceptedDataCacheMap, ) require.Nil(t, err) time.Sleep(2 * time.Second) diff --git a/integrationTests/factory/dataComponents/dataComponents_test.go b/integrationTests/factory/dataComponents/dataComponents_test.go index c28a41c6543..ad9459fe3b4 100644 --- a/integrationTests/factory/dataComponents/dataComponents_test.go +++ b/integrationTests/factory/dataComponents/dataComponents_test.go @@ -6,10 +6,12 @@ import ( "time" "github.com/multiversx/mx-chain-core-go/data/endProcess" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-go/integrationTests/factory" "github.com/multiversx/mx-chain-go/node" + "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/testscommon/goroutines" - "github.com/stretchr/testify/require" ) func TestDataComponents_Create_Close_ShouldWork(t *testing.T) { @@ -36,7 +38,8 @@ func TestDataComponents_Create_Close_ShouldWork(t *testing.T) { require.Nil(t, err) managedNetworkComponents, err := nr.CreateManagedNetworkComponents(managedCoreComponents, managedStatusCoreComponents, managedCryptoComponents) require.Nil(t, err) - managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents) + interceptedDataCacheMap := make(map[string]storage.Cacher) + managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents, interceptedDataCacheMap) require.Nil(t, err) managedDataComponents, err := nr.CreateManagedDataComponents(managedStatusCoreComponents, managedCoreComponents, managedBootstrapComponents, managedCryptoComponents) require.Nil(t, err) diff --git a/integrationTests/factory/heartbeatComponents/heartbeatComponents_test.go b/integrationTests/factory/heartbeatComponents/heartbeatComponents_test.go index 1c541f524ff..8b7086017d7 100644 --- a/integrationTests/factory/heartbeatComponents/heartbeatComponents_test.go +++ b/integrationTests/factory/heartbeatComponents/heartbeatComponents_test.go @@ -6,13 +6,15 @@ import ( "time" "github.com/multiversx/mx-chain-core-go/data/endProcess" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-go/common/forking" "github.com/multiversx/mx-chain-go/dataRetriever" bootstrapComp "github.com/multiversx/mx-chain-go/factory/bootstrap" "github.com/multiversx/mx-chain-go/integrationTests/factory" "github.com/multiversx/mx-chain-go/node" + "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/testscommon/goroutines" - "github.com/stretchr/testify/require" ) // ------------ Test TestHeartbeatComponents -------------------- @@ -39,7 +41,8 @@ func TestHeartbeatComponents_Close_ShouldWork(t *testing.T) { require.Nil(t, err) managedNetworkComponents, err := nr.CreateManagedNetworkComponents(managedCoreComponents, managedStatusCoreComponents, managedCryptoComponents) require.Nil(t, err) - managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents) + interceptedDataCacheMap := make(map[string]storage.Cacher) + managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents, interceptedDataCacheMap) require.Nil(t, err) managedDataComponents, err := nr.CreateManagedDataComponents(managedStatusCoreComponents, managedCoreComponents, managedBootstrapComponents, managedCryptoComponents) require.Nil(t, err) @@ -103,6 +106,7 @@ func TestHeartbeatComponents_Close_ShouldWork(t *testing.T) { managedStatusCoreComponents, gasScheduleNotifier, nodesCoordinator, + interceptedDataCacheMap, ) require.Nil(t, err) time.Sleep(2 * time.Second) diff --git a/integrationTests/factory/processComponents/processComponents_test.go b/integrationTests/factory/processComponents/processComponents_test.go index 897a1289d2c..42a59b274b7 100644 --- a/integrationTests/factory/processComponents/processComponents_test.go +++ b/integrationTests/factory/processComponents/processComponents_test.go @@ -6,13 +6,15 @@ import ( "time" "github.com/multiversx/mx-chain-core-go/data/endProcess" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-go/common/forking" "github.com/multiversx/mx-chain-go/dataRetriever" bootstrapComp "github.com/multiversx/mx-chain-go/factory/bootstrap" "github.com/multiversx/mx-chain-go/integrationTests/factory" "github.com/multiversx/mx-chain-go/node" + "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/testscommon/goroutines" - "github.com/stretchr/testify/require" ) // ------------ Test TestProcessComponents -------------------- @@ -40,7 +42,8 @@ func TestProcessComponents_Close_ShouldWork(t *testing.T) { require.Nil(t, err) managedNetworkComponents, err := nr.CreateManagedNetworkComponents(managedCoreComponents, managedStatusCoreComponents, managedCryptoComponents) require.Nil(t, err) - managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents) + interceptedDataCacheMap := make(map[string]storage.Cacher) + managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents, interceptedDataCacheMap) require.Nil(t, err) managedDataComponents, err := nr.CreateManagedDataComponents(managedStatusCoreComponents, managedCoreComponents, managedBootstrapComponents, managedCryptoComponents) require.Nil(t, err) @@ -102,6 +105,7 @@ func TestProcessComponents_Close_ShouldWork(t *testing.T) { managedStatusCoreComponents, gasScheduleNotifier, nodesCoordinator, + interceptedDataCacheMap, ) require.Nil(t, err) require.NotNil(t, managedProcessComponents) diff --git a/integrationTests/factory/stateComponents/stateComponents_test.go b/integrationTests/factory/stateComponents/stateComponents_test.go index 3c942f54e53..ff011c7a803 100644 --- a/integrationTests/factory/stateComponents/stateComponents_test.go +++ b/integrationTests/factory/stateComponents/stateComponents_test.go @@ -6,10 +6,12 @@ import ( "time" "github.com/multiversx/mx-chain-core-go/data/endProcess" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-go/integrationTests/factory" "github.com/multiversx/mx-chain-go/node" + "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/testscommon/goroutines" - "github.com/stretchr/testify/require" ) func TestStateComponents_Create_Close_ShouldWork(t *testing.T) { @@ -36,7 +38,8 @@ func TestStateComponents_Create_Close_ShouldWork(t *testing.T) { require.Nil(t, err) managedNetworkComponents, err := nr.CreateManagedNetworkComponents(managedCoreComponents, managedStatusCoreComponents, managedCryptoComponents) require.Nil(t, err) - managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents) + interceptedDataCacheMap := make(map[string]storage.Cacher) + managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents, interceptedDataCacheMap) require.Nil(t, err) managedDataComponents, err := nr.CreateManagedDataComponents(managedStatusCoreComponents, managedCoreComponents, managedBootstrapComponents, managedCryptoComponents) require.Nil(t, err) diff --git a/integrationTests/factory/statusComponents/statusComponents_test.go b/integrationTests/factory/statusComponents/statusComponents_test.go index 85cfbd155f7..3c3ef519d1f 100644 --- a/integrationTests/factory/statusComponents/statusComponents_test.go +++ b/integrationTests/factory/statusComponents/statusComponents_test.go @@ -6,13 +6,15 @@ import ( "time" "github.com/multiversx/mx-chain-core-go/data/endProcess" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-go/common/forking" "github.com/multiversx/mx-chain-go/dataRetriever" bootstrapComp "github.com/multiversx/mx-chain-go/factory/bootstrap" "github.com/multiversx/mx-chain-go/integrationTests/factory" "github.com/multiversx/mx-chain-go/node" + "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/testscommon/goroutines" - "github.com/stretchr/testify/require" ) // ------------ Test StatusComponents -------------------- @@ -40,7 +42,8 @@ func TestStatusComponents_Create_Close_ShouldWork(t *testing.T) { require.Nil(t, err) managedNetworkComponents, err := nr.CreateManagedNetworkComponents(managedCoreComponents, managedStatusCoreComponents, managedCryptoComponents) require.Nil(t, err) - managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents) + interceptedDataCacheMap := make(map[string]storage.Cacher) + managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents, interceptedDataCacheMap) require.Nil(t, err) managedDataComponents, err := nr.CreateManagedDataComponents(managedStatusCoreComponents, managedCoreComponents, managedBootstrapComponents, managedCryptoComponents) require.Nil(t, err) @@ -104,6 +107,7 @@ func TestStatusComponents_Create_Close_ShouldWork(t *testing.T) { managedStatusCoreComponents, gasScheduleNotifier, nodesCoordinator, + interceptedDataCacheMap, ) require.Nil(t, err) time.Sleep(2 * time.Second) From f21e24f79a9fa733fc081180a63106f36ad8e176 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Fri, 20 Sep 2024 16:21:51 +0300 Subject: [PATCH 21/37] fix some more tests. --- epochStart/bootstrap/storageProcess.go | 2 ++ epochStart/bootstrap/syncEpochStartMeta.go | 11 ++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/epochStart/bootstrap/storageProcess.go b/epochStart/bootstrap/storageProcess.go index 2e57801ef89..5fe62e8c1b1 100644 --- a/epochStart/bootstrap/storageProcess.go +++ b/epochStart/bootstrap/storageProcess.go @@ -11,6 +11,7 @@ import ( "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/endProcess" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/config" "github.com/multiversx/mx-chain-go/dataRetriever" @@ -187,6 +188,7 @@ func (sesb *storageEpochStartBootstrap) prepareComponentsToSync() error { StartInEpochConfig: sesb.generalConfig.EpochStartConfig, HeaderIntegrityVerifier: sesb.headerIntegrityVerifier, MetaBlockProcessor: metablockProcessor, + InterceptedDataCache: sesb.interceptedDataCache, } sesb.epochStartMetaBlockSyncer, err = NewEpochStartMetaSyncer(argsEpochStartSyncer) diff --git a/epochStart/bootstrap/syncEpochStartMeta.go b/epochStart/bootstrap/syncEpochStartMeta.go index bebd8cc68cd..922fdf8477d 100644 --- a/epochStart/bootstrap/syncEpochStartMeta.go +++ b/epochStart/bootstrap/syncEpochStartMeta.go @@ -74,11 +74,12 @@ func NewEpochStartMetaSyncer(args ArgsNewEpochStartMetaSyncer) (*epochStartMetaS } e := &epochStartMetaSyncer{ - requestHandler: args.RequestHandler, - messenger: args.Messenger, - marshalizer: args.CoreComponentsHolder.InternalMarshalizer(), - hasher: args.CoreComponentsHolder.Hasher(), - metaBlockProcessor: args.MetaBlockProcessor, + requestHandler: args.RequestHandler, + messenger: args.Messenger, + marshalizer: args.CoreComponentsHolder.InternalMarshalizer(), + hasher: args.CoreComponentsHolder.Hasher(), + metaBlockProcessor: args.MetaBlockProcessor, + interceptedDataCacheMap: args.InterceptedDataCache, } argsInterceptedDataFactory := interceptorsFactory.ArgInterceptedDataFactory{ From f40d222a63d4be7b90ddb81fe0d970e81b842572 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Mon, 23 Sep 2024 09:21:03 +0300 Subject: [PATCH 22/37] fix some unit tests. --- epochStart/bootstrap/syncEpochStartMeta_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/epochStart/bootstrap/syncEpochStartMeta_test.go b/epochStart/bootstrap/syncEpochStartMeta_test.go index 169b20a656e..a99722d9422 100644 --- a/epochStart/bootstrap/syncEpochStartMeta_test.go +++ b/epochStart/bootstrap/syncEpochStartMeta_test.go @@ -9,17 +9,19 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-go/config" "github.com/multiversx/mx-chain-go/epochStart" "github.com/multiversx/mx-chain-go/epochStart/mock" "github.com/multiversx/mx-chain-go/p2p" + "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/testscommon" "github.com/multiversx/mx-chain-go/testscommon/cryptoMocks" "github.com/multiversx/mx-chain-go/testscommon/economicsmocks" "github.com/multiversx/mx-chain-go/testscommon/hashingMocks" "github.com/multiversx/mx-chain-go/testscommon/p2pmocks" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func TestNewEpochStartMetaSyncer_NilsShouldError(t *testing.T) { @@ -161,5 +163,6 @@ func getEpochStartSyncerArgs() ArgsNewEpochStartMetaSyncer { }, HeaderIntegrityVerifier: &mock.HeaderIntegrityVerifierStub{}, MetaBlockProcessor: &mock.EpochStartMetaBlockProcessorStub{}, + InterceptedDataCache: make(map[string]storage.Cacher), } } From 3a327c4dfdcc0fb55b4fc5332fc992e9c9921455 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Mon, 23 Sep 2024 09:31:17 +0300 Subject: [PATCH 23/37] fix nil map in tests. --- epochStart/bootstrap/process_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/epochStart/bootstrap/process_test.go b/epochStart/bootstrap/process_test.go index 7878f3842be..8807ee77814 100644 --- a/epochStart/bootstrap/process_test.go +++ b/epochStart/bootstrap/process_test.go @@ -253,6 +253,7 @@ func createMockEpochStartBootstrapArgs( }, TrieSyncStatisticsProvider: &testscommon.SizeSyncStatisticsHandlerStub{}, StateStatsHandler: disabledStatistics.NewStateStatistics(), + InterceptedDataCache: make(map[string]storage.Cacher), } } From cde6c806f5ed06533282435ad32996570efc6fdc Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Wed, 25 Sep 2024 16:19:58 +0300 Subject: [PATCH 24/37] cosmetic changes. --- process/block/interceptedBlocks/interceptedMetaBlockHeader.go | 2 +- .../interceptedBlocks/interceptedMetaBlockHeader_test.go | 2 +- process/interceptors/multiDataInterceptor.go | 4 +--- process/transaction/interceptedTransaction_test.go | 2 +- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/process/block/interceptedBlocks/interceptedMetaBlockHeader.go b/process/block/interceptedBlocks/interceptedMetaBlockHeader.go index 29e8dae4228..1d2917c7cb8 100644 --- a/process/block/interceptedBlocks/interceptedMetaBlockHeader.go +++ b/process/block/interceptedBlocks/interceptedMetaBlockHeader.go @@ -100,7 +100,7 @@ func (imh *InterceptedMetaHeader) CheckValidity() error { } if imh.isMetaHeaderEpochOutOfRange() { - log.Trace("InterceptedMetaHeader.Verify", + log.Trace("InterceptedMetaHeader.CheckValidity", "trigger epoch", imh.epochStartTrigger.Epoch(), "metaBlock epoch", imh.hdr.GetEpoch(), "error", process.ErrMetaHeaderEpochOutOfRange) diff --git a/process/block/interceptedBlocks/interceptedMetaBlockHeader_test.go b/process/block/interceptedBlocks/interceptedMetaBlockHeader_test.go index a3776269e21..b895a6a81cc 100644 --- a/process/block/interceptedBlocks/interceptedMetaBlockHeader_test.go +++ b/process/block/interceptedBlocks/interceptedMetaBlockHeader_test.go @@ -99,7 +99,7 @@ func TestNewInterceptedMetaHeader_ShouldWork(t *testing.T) { assert.Nil(t, err) } -//------- Verify +//------- CheckValidity func TestInterceptedMetaHeader_CheckValidityNilPubKeyBitmapShouldErr(t *testing.T) { t.Parallel() diff --git a/process/interceptors/multiDataInterceptor.go b/process/interceptors/multiDataInterceptor.go index 2bf6f29918b..923c9b360e9 100644 --- a/process/interceptors/multiDataInterceptor.go +++ b/process/interceptors/multiDataInterceptor.go @@ -159,10 +159,8 @@ func (mdi *MultiDataInterceptor) ProcessReceivedMessage(message p2p.MessageP2P, for index, dataBuff := range multiDataBuff { var interceptedData process.InterceptedData interceptedData, err = mdi.interceptedData(dataBuff, message.Peer(), fromConnectedPeer) + listInterceptedData[index] = interceptedData - if !errors.Is(err, ErrInvalidInterceptedData) { - listInterceptedData[index] = interceptedData - } if err != nil { mdi.throttler.EndProcessing() return err diff --git a/process/transaction/interceptedTransaction_test.go b/process/transaction/interceptedTransaction_test.go index de5536ea886..1312f5cba4f 100644 --- a/process/transaction/interceptedTransaction_test.go +++ b/process/transaction/interceptedTransaction_test.go @@ -612,7 +612,7 @@ func TestNewInterceptedTransaction_ShouldWork(t *testing.T) { assert.Equal(t, tx, txi.Transaction()) } -// ------- Verify +// ------- CheckValidity func TestInterceptedTransaction_CheckValidityNilSignatureShouldErr(t *testing.T) { t.Parallel() From 9504263b8583877b7a85a7e7f086efab54885537 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Wed, 25 Sep 2024 17:21:33 +0300 Subject: [PATCH 25/37] refactored map of caches for intercepted data into a new component. --- cmd/node/config/config.toml | 4 + config/config.go | 8 + .../epochStartInterceptorsContainerFactory.go | 98 ++++----- epochStart/bootstrap/process.go | 28 +-- epochStart/bootstrap/storageProcess.go | 22 +- epochStart/bootstrap/syncEpochStartMeta.go | 73 +++---- factory/bootstrap/bootstrapComponents.go | 22 +- factory/processing/processComponents.go | 198 +++++++++--------- go.mod | 2 +- go.sum | 4 +- node/interface.go | 3 +- node/nodeRunner.go | 91 ++++---- process/factory/interceptorscontainer/args.go | 63 +++--- .../baseInterceptorsContainerFactory.go | 101 +++++---- .../metaInterceptorsContainerFactory.go | 48 ++--- .../shardInterceptorsContainerFactory.go | 46 ++-- .../factory/interceptedDataVerifierFactory.go | 48 +++++ .../interceptors/interceptedDataVerifier.go | 22 +- process/interface.go | 4 + update/factory/exportHandlerFactory.go | 8 +- update/factory/fullSyncInterceptors.go | 2 +- 21 files changed, 474 insertions(+), 421 deletions(-) create mode 100644 process/interceptors/factory/interceptedDataVerifierFactory.go diff --git a/cmd/node/config/config.toml b/cmd/node/config/config.toml index f415dd8e426..688f688b7e2 100644 --- a/cmd/node/config/config.toml +++ b/cmd/node/config/config.toml @@ -958,3 +958,7 @@ # All validators will broadcast the message right away { EndIndex = 0, DelayInMilliseconds = 0 }, ] + +[InterceptedDataVerifier] + CacheSpanInSec = 30 + CacheExpiryInSec = 30 diff --git a/config/config.go b/config/config.go index 19da7e2c0c8..9607c9dc330 100644 --- a/config/config.go +++ b/config/config.go @@ -229,6 +229,8 @@ type Config struct { PoolsCleanersConfig PoolsCleanersConfig Redundancy RedundancyConfig ConsensusGradualBroadcast ConsensusGradualBroadcastConfig + + InterceptedDataVerifier InterceptedDataVerifierConfig } // PeersRatingConfig will hold settings related to peers rating @@ -679,3 +681,9 @@ type IndexBroadcastDelay struct { type ConsensusGradualBroadcastConfig struct { GradualIndexBroadcastDelay []IndexBroadcastDelay } + +// InterceptedDataVerifierConfig holds the configuration for the intercepted data verifier +type InterceptedDataVerifierConfig struct { + CacheSpanInSec uint64 + CacheExpiryInSec uint64 +} diff --git a/epochStart/bootstrap/factory/epochStartInterceptorsContainerFactory.go b/epochStart/bootstrap/factory/epochStartInterceptorsContainerFactory.go index fb6ca753834..8700b1daa24 100644 --- a/epochStart/bootstrap/factory/epochStartInterceptorsContainerFactory.go +++ b/epochStart/bootstrap/factory/epochStartInterceptorsContainerFactory.go @@ -17,7 +17,6 @@ import ( "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/factory/interceptorscontainer" "github.com/multiversx/mx-chain-go/sharding" - "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/storage/cache" "github.com/multiversx/mx-chain-go/update" ) @@ -27,23 +26,24 @@ const timeSpanForBadHeaders = time.Minute // ArgsEpochStartInterceptorContainer holds the arguments needed for creating a new epoch start interceptors // container factory type ArgsEpochStartInterceptorContainer struct { - CoreComponents process.CoreComponentsHolder - CryptoComponents process.CryptoComponentsHolder - Config config.Config - ShardCoordinator sharding.Coordinator - MainMessenger process.TopicHandler - FullArchiveMessenger process.TopicHandler - DataPool dataRetriever.PoolsHolder - WhiteListHandler update.WhiteListHandler - WhiteListerVerifiedTxs update.WhiteListHandler - AddressPubkeyConv core.PubkeyConverter - NonceConverter typeConverters.Uint64ByteSliceConverter - ChainID []byte - ArgumentsParser process.ArgumentsParser - HeaderIntegrityVerifier process.HeaderIntegrityVerifier - RequestHandler process.RequestHandler - SignaturesHandler process.SignaturesHandler - NodeOperationMode common.NodeOperation + CoreComponents process.CoreComponentsHolder + CryptoComponents process.CryptoComponentsHolder + Config config.Config + ShardCoordinator sharding.Coordinator + MainMessenger process.TopicHandler + FullArchiveMessenger process.TopicHandler + DataPool dataRetriever.PoolsHolder + WhiteListHandler update.WhiteListHandler + WhiteListerVerifiedTxs update.WhiteListHandler + AddressPubkeyConv core.PubkeyConverter + NonceConverter typeConverters.Uint64ByteSliceConverter + ChainID []byte + ArgumentsParser process.ArgumentsParser + HeaderIntegrityVerifier process.HeaderIntegrityVerifier + RequestHandler process.RequestHandler + SignaturesHandler process.SignaturesHandler + NodeOperationMode common.NodeOperation + InterceptedDataVerifierFactory process.InterceptedDataVerifierFactory } // NewEpochStartInterceptorsContainer will return a real interceptors container factory, but with many disabled components @@ -80,37 +80,37 @@ func NewEpochStartInterceptorsContainer(args ArgsEpochStartInterceptorContainer) hardforkTrigger := disabledFactory.HardforkTrigger() containerFactoryArgs := interceptorscontainer.CommonInterceptorsContainerFactoryArgs{ - CoreComponents: args.CoreComponents, - CryptoComponents: cryptoComponents, - Accounts: accountsAdapter, - ShardCoordinator: args.ShardCoordinator, - NodesCoordinator: nodesCoordinator, - MainMessenger: args.MainMessenger, - FullArchiveMessenger: args.FullArchiveMessenger, - Store: storer, - DataPool: args.DataPool, - MaxTxNonceDeltaAllowed: common.MaxTxNonceDeltaAllowed, - TxFeeHandler: feeHandler, - BlockBlackList: blackListHandler, - HeaderSigVerifier: headerSigVerifier, - HeaderIntegrityVerifier: args.HeaderIntegrityVerifier, - ValidityAttester: validityAttester, - EpochStartTrigger: epochStartTrigger, - WhiteListHandler: args.WhiteListHandler, - WhiteListerVerifiedTxs: args.WhiteListerVerifiedTxs, - AntifloodHandler: antiFloodHandler, - ArgumentsParser: args.ArgumentsParser, - PreferredPeersHolder: disabled.NewPreferredPeersHolder(), - SizeCheckDelta: uint32(sizeCheckDelta), - RequestHandler: args.RequestHandler, - PeerSignatureHandler: cryptoComponents.PeerSignatureHandler(), - SignaturesHandler: args.SignaturesHandler, - HeartbeatExpiryTimespanInSec: args.Config.HeartbeatV2.HeartbeatExpiryTimespanInSec, - MainPeerShardMapper: peerShardMapper, - FullArchivePeerShardMapper: fullArchivePeerShardMapper, - HardforkTrigger: hardforkTrigger, - NodeOperationMode: args.NodeOperationMode, - ProcessedMessagesCacheMap: make(map[string]storage.Cacher), + CoreComponents: args.CoreComponents, + CryptoComponents: cryptoComponents, + Accounts: accountsAdapter, + ShardCoordinator: args.ShardCoordinator, + NodesCoordinator: nodesCoordinator, + MainMessenger: args.MainMessenger, + FullArchiveMessenger: args.FullArchiveMessenger, + Store: storer, + DataPool: args.DataPool, + MaxTxNonceDeltaAllowed: common.MaxTxNonceDeltaAllowed, + TxFeeHandler: feeHandler, + BlockBlackList: blackListHandler, + HeaderSigVerifier: headerSigVerifier, + HeaderIntegrityVerifier: args.HeaderIntegrityVerifier, + ValidityAttester: validityAttester, + EpochStartTrigger: epochStartTrigger, + WhiteListHandler: args.WhiteListHandler, + WhiteListerVerifiedTxs: args.WhiteListerVerifiedTxs, + AntifloodHandler: antiFloodHandler, + ArgumentsParser: args.ArgumentsParser, + PreferredPeersHolder: disabled.NewPreferredPeersHolder(), + SizeCheckDelta: uint32(sizeCheckDelta), + RequestHandler: args.RequestHandler, + PeerSignatureHandler: cryptoComponents.PeerSignatureHandler(), + SignaturesHandler: args.SignaturesHandler, + HeartbeatExpiryTimespanInSec: args.Config.HeartbeatV2.HeartbeatExpiryTimespanInSec, + MainPeerShardMapper: peerShardMapper, + FullArchivePeerShardMapper: fullArchivePeerShardMapper, + HardforkTrigger: hardforkTrigger, + NodeOperationMode: args.NodeOperationMode, + InterceptedDataVerifierFactory: args.InterceptedDataVerifierFactory, } interceptorsContainerFactory, err := interceptorscontainer.NewMetaInterceptorsContainerFactory(containerFactoryArgs) diff --git a/epochStart/bootstrap/process.go b/epochStart/bootstrap/process.go index ac6b7fb371b..62ab816ce33 100644 --- a/epochStart/bootstrap/process.go +++ b/epochStart/bootstrap/process.go @@ -154,7 +154,7 @@ type epochStartBootstrap struct { startEpoch uint32 shuffledOut bool - interceptedDataCache map[string]storage.Cacher + interceptedDataVerifierFactory process.InterceptedDataVerifierFactory } type baseDataInStorage struct { @@ -193,7 +193,7 @@ type ArgsEpochStartBootstrap struct { NodeProcessingMode common.NodeProcessingMode StateStatsHandler common.StateStatisticsHandler NodesCoordinatorRegistryFactory nodesCoordinator.NodesCoordinatorRegistryFactory - InterceptedDataCache map[string]storage.Cacher + InterceptedDataVerifierFactory process.InterceptedDataVerifierFactory } type dataToSync struct { @@ -246,7 +246,7 @@ func NewEpochStartBootstrap(args ArgsEpochStartBootstrap) (*epochStartBootstrap, stateStatsHandler: args.StateStatsHandler, startEpoch: args.GeneralConfig.EpochStartConfig.GenesisEpoch, nodesCoordinatorRegistryFactory: args.NodesCoordinatorRegistryFactory, - interceptedDataCache: args.InterceptedDataCache, + interceptedDataVerifierFactory: args.InterceptedDataVerifierFactory, } if epochStartProvider.prefsConfig.FullArchive { @@ -558,17 +558,17 @@ func (e *epochStartBootstrap) prepareComponentsToSyncFromNetwork() error { } argsEpochStartSyncer := ArgsNewEpochStartMetaSyncer{ - CoreComponentsHolder: e.coreComponentsHolder, - CryptoComponentsHolder: e.cryptoComponentsHolder, - RequestHandler: e.requestHandler, - Messenger: e.mainMessenger, - ShardCoordinator: e.shardCoordinator, - EconomicsData: e.economicsData, - WhitelistHandler: e.whiteListHandler, - StartInEpochConfig: epochStartConfig, - HeaderIntegrityVerifier: e.headerIntegrityVerifier, - MetaBlockProcessor: metaBlockProcessor, - InterceptedDataCache: e.interceptedDataCache, + CoreComponentsHolder: e.coreComponentsHolder, + CryptoComponentsHolder: e.cryptoComponentsHolder, + RequestHandler: e.requestHandler, + Messenger: e.mainMessenger, + ShardCoordinator: e.shardCoordinator, + EconomicsData: e.economicsData, + WhitelistHandler: e.whiteListHandler, + StartInEpochConfig: epochStartConfig, + HeaderIntegrityVerifier: e.headerIntegrityVerifier, + MetaBlockProcessor: metaBlockProcessor, + InterceptedDataVerifierFactory: e.interceptedDataVerifierFactory, } e.epochStartMetaBlockSyncer, err = NewEpochStartMetaSyncer(argsEpochStartSyncer) if err != nil { diff --git a/epochStart/bootstrap/storageProcess.go b/epochStart/bootstrap/storageProcess.go index 5fe62e8c1b1..0ec16f6548d 100644 --- a/epochStart/bootstrap/storageProcess.go +++ b/epochStart/bootstrap/storageProcess.go @@ -178,17 +178,17 @@ func (sesb *storageEpochStartBootstrap) prepareComponentsToSync() error { } argsEpochStartSyncer := ArgsNewEpochStartMetaSyncer{ - CoreComponentsHolder: sesb.coreComponentsHolder, - CryptoComponentsHolder: sesb.cryptoComponentsHolder, - RequestHandler: sesb.requestHandler, - Messenger: sesb.mainMessenger, - ShardCoordinator: sesb.shardCoordinator, - EconomicsData: sesb.economicsData, - WhitelistHandler: sesb.whiteListHandler, - StartInEpochConfig: sesb.generalConfig.EpochStartConfig, - HeaderIntegrityVerifier: sesb.headerIntegrityVerifier, - MetaBlockProcessor: metablockProcessor, - InterceptedDataCache: sesb.interceptedDataCache, + CoreComponentsHolder: sesb.coreComponentsHolder, + CryptoComponentsHolder: sesb.cryptoComponentsHolder, + RequestHandler: sesb.requestHandler, + Messenger: sesb.mainMessenger, + ShardCoordinator: sesb.shardCoordinator, + EconomicsData: sesb.economicsData, + WhitelistHandler: sesb.whiteListHandler, + StartInEpochConfig: sesb.generalConfig.EpochStartConfig, + HeaderIntegrityVerifier: sesb.headerIntegrityVerifier, + MetaBlockProcessor: metablockProcessor, + InterceptedDataVerifierFactory: sesb.interceptedDataVerifierFactory, } sesb.epochStartMetaBlockSyncer, err = NewEpochStartMetaSyncer(argsEpochStartSyncer) diff --git a/epochStart/bootstrap/syncEpochStartMeta.go b/epochStart/bootstrap/syncEpochStartMeta.go index 922fdf8477d..8b059909fb1 100644 --- a/epochStart/bootstrap/syncEpochStartMeta.go +++ b/epochStart/bootstrap/syncEpochStartMeta.go @@ -18,41 +18,34 @@ import ( "github.com/multiversx/mx-chain-go/process/interceptors" interceptorsFactory "github.com/multiversx/mx-chain-go/process/interceptors/factory" "github.com/multiversx/mx-chain-go/sharding" - "github.com/multiversx/mx-chain-go/storage" - "github.com/multiversx/mx-chain-go/storage/cache" -) - -const ( - cacheDefaultSpan = 30 * time.Second - cacheDefaultExpiry = 30 * time.Second ) var _ epochStart.StartOfEpochMetaSyncer = (*epochStartMetaSyncer)(nil) type epochStartMetaSyncer struct { - requestHandler RequestHandler - messenger Messenger - marshalizer marshal.Marshalizer - hasher hashing.Hasher - singleDataInterceptor process.Interceptor - metaBlockProcessor EpochStartMetaBlockInterceptorProcessor - interceptedDataCacheMap map[string]storage.Cacher + requestHandler RequestHandler + messenger Messenger + marshalizer marshal.Marshalizer + hasher hashing.Hasher + singleDataInterceptor process.Interceptor + metaBlockProcessor EpochStartMetaBlockInterceptorProcessor + interceptedDataVerifierFactory process.InterceptedDataVerifierFactory } // ArgsNewEpochStartMetaSyncer - type ArgsNewEpochStartMetaSyncer struct { - CoreComponentsHolder process.CoreComponentsHolder - CryptoComponentsHolder process.CryptoComponentsHolder - RequestHandler RequestHandler - Messenger Messenger - ShardCoordinator sharding.Coordinator - EconomicsData process.EconomicsDataHandler - WhitelistHandler process.WhiteListHandler - StartInEpochConfig config.EpochStartConfig - ArgsParser process.ArgumentsParser - HeaderIntegrityVerifier process.HeaderIntegrityVerifier - MetaBlockProcessor EpochStartMetaBlockInterceptorProcessor - InterceptedDataCache map[string]storage.Cacher + CoreComponentsHolder process.CoreComponentsHolder + CryptoComponentsHolder process.CryptoComponentsHolder + RequestHandler RequestHandler + Messenger Messenger + ShardCoordinator sharding.Coordinator + EconomicsData process.EconomicsDataHandler + WhitelistHandler process.WhiteListHandler + StartInEpochConfig config.EpochStartConfig + ArgsParser process.ArgumentsParser + HeaderIntegrityVerifier process.HeaderIntegrityVerifier + MetaBlockProcessor EpochStartMetaBlockInterceptorProcessor + InterceptedDataVerifierFactory process.InterceptedDataVerifierFactory } // NewEpochStartMetaSyncer will return a new instance of epochStartMetaSyncer @@ -74,12 +67,12 @@ func NewEpochStartMetaSyncer(args ArgsNewEpochStartMetaSyncer) (*epochStartMetaS } e := &epochStartMetaSyncer{ - requestHandler: args.RequestHandler, - messenger: args.Messenger, - marshalizer: args.CoreComponentsHolder.InternalMarshalizer(), - hasher: args.CoreComponentsHolder.Hasher(), - metaBlockProcessor: args.MetaBlockProcessor, - interceptedDataCacheMap: args.InterceptedDataCache, + requestHandler: args.RequestHandler, + messenger: args.Messenger, + marshalizer: args.CoreComponentsHolder.InternalMarshalizer(), + hasher: args.CoreComponentsHolder.Hasher(), + metaBlockProcessor: args.MetaBlockProcessor, + interceptedDataVerifierFactory: args.InterceptedDataVerifierFactory, } argsInterceptedDataFactory := interceptorsFactory.ArgInterceptedDataFactory{ @@ -100,7 +93,7 @@ func NewEpochStartMetaSyncer(args ArgsNewEpochStartMetaSyncer) (*epochStartMetaS return nil, err } - interceptedDataVerifier, err := e.createCacheForInterceptor(factory.MetachainBlocksTopic) + interceptedDataVerifier, err := e.interceptedDataVerifierFactory.Create(factory.MetachainBlocksTopic) if err != nil { return nil, err } @@ -169,20 +162,6 @@ func (e *epochStartMetaSyncer) initTopicForEpochStartMetaBlockInterceptor() erro return nil } -func (e *epochStartMetaSyncer) createCacheForInterceptor(topic string) (process.InterceptedDataVerifier, error) { - internalCache, err := cache.NewTimeCacher(cache.ArgTimeCacher{ - DefaultSpan: cacheDefaultSpan, - CacheExpiry: cacheDefaultExpiry, - }) - if err != nil { - return nil, err - } - - e.interceptedDataCacheMap[topic] = internalCache - verifier := interceptors.NewInterceptedDataVerifier(internalCache) - return verifier, nil -} - // IsInterfaceNil returns true if underlying object is nil func (e *epochStartMetaSyncer) IsInterfaceNil() bool { return e == nil diff --git a/factory/bootstrap/bootstrapComponents.go b/factory/bootstrap/bootstrapComponents.go index cb325cb2e71..1c3500f9599 100644 --- a/factory/bootstrap/bootstrapComponents.go +++ b/factory/bootstrap/bootstrapComponents.go @@ -32,17 +32,17 @@ var log = logger.GetOrCreate("factory") // BootstrapComponentsFactoryArgs holds the arguments needed to create a bootstrap components factory type BootstrapComponentsFactoryArgs struct { - Config config.Config - RoundConfig config.RoundConfig - PrefConfig config.Preferences - ImportDbConfig config.ImportDbConfig - FlagsConfig config.ContextFlagsConfig - WorkingDir string - CoreComponents factory.CoreComponentsHolder - CryptoComponents factory.CryptoComponentsHolder - NetworkComponents factory.NetworkComponentsHolder - StatusCoreComponents factory.StatusCoreComponentsHolder - InterceptedDataCacheMap map[string]storage.Cacher + Config config.Config + RoundConfig config.RoundConfig + PrefConfig config.Preferences + ImportDbConfig config.ImportDbConfig + FlagsConfig config.ContextFlagsConfig + WorkingDir string + CoreComponents factory.CoreComponentsHolder + CryptoComponents factory.CryptoComponentsHolder + NetworkComponents factory.NetworkComponentsHolder + StatusCoreComponents factory.StatusCoreComponentsHolder + InterceptedDataVerifierFactory process.InterceptedDataVerifierFactory } type bootstrapComponentsFactory struct { diff --git a/factory/processing/processComponents.go b/factory/processing/processComponents.go index 1df74b3f9dd..fbab1357680 100644 --- a/factory/processing/processComponents.go +++ b/factory/processing/processComponents.go @@ -169,7 +169,7 @@ type ProcessComponentsFactoryArgs struct { GenesisNonce uint64 GenesisRound uint64 - InterceptedDataCacheMap map[string]storage.Cacher + InterceptedDataVerifierFactory process.InterceptedDataVerifierFactory } type processComponentsFactory struct { @@ -211,7 +211,7 @@ type processComponentsFactory struct { genesisNonce uint64 genesisRound uint64 - interceptedDataCacheMap map[string]storage.Cacher + interceptedDataVerifierFactory process.InterceptedDataVerifierFactory } // NewProcessComponentsFactory will return a new instance of processComponentsFactory @@ -222,37 +222,37 @@ func NewProcessComponentsFactory(args ProcessComponentsFactoryArgs) (*processCom } return &processComponentsFactory{ - config: args.Config, - epochConfig: args.EpochConfig, - prefConfigs: args.PrefConfigs, - importDBConfig: args.ImportDBConfig, - economicsConfig: args.EconomicsConfig, - accountsParser: args.AccountsParser, - smartContractParser: args.SmartContractParser, - gasSchedule: args.GasSchedule, - nodesCoordinator: args.NodesCoordinator, - data: args.Data, - coreData: args.CoreData, - crypto: args.Crypto, - state: args.State, - network: args.Network, - bootstrapComponents: args.BootstrapComponents, - statusComponents: args.StatusComponents, - requestedItemsHandler: args.RequestedItemsHandler, - whiteListHandler: args.WhiteListHandler, - whiteListerVerifiedTxs: args.WhiteListerVerifiedTxs, - maxRating: args.MaxRating, - systemSCConfig: args.SystemSCConfig, - importStartHandler: args.ImportStartHandler, - historyRepo: args.HistoryRepo, - epochNotifier: args.CoreData.EpochNotifier(), - statusCoreComponents: args.StatusCoreComponents, - flagsConfig: args.FlagsConfig, - txExecutionOrderHandler: args.TxExecutionOrderHandler, - genesisNonce: args.GenesisNonce, - genesisRound: args.GenesisRound, - roundConfig: args.RoundConfig, - interceptedDataCacheMap: args.InterceptedDataCacheMap, + config: args.Config, + epochConfig: args.EpochConfig, + prefConfigs: args.PrefConfigs, + importDBConfig: args.ImportDBConfig, + economicsConfig: args.EconomicsConfig, + accountsParser: args.AccountsParser, + smartContractParser: args.SmartContractParser, + gasSchedule: args.GasSchedule, + nodesCoordinator: args.NodesCoordinator, + data: args.Data, + coreData: args.CoreData, + crypto: args.Crypto, + state: args.State, + network: args.Network, + bootstrapComponents: args.BootstrapComponents, + statusComponents: args.StatusComponents, + requestedItemsHandler: args.RequestedItemsHandler, + whiteListHandler: args.WhiteListHandler, + whiteListerVerifiedTxs: args.WhiteListerVerifiedTxs, + maxRating: args.MaxRating, + systemSCConfig: args.SystemSCConfig, + importStartHandler: args.ImportStartHandler, + historyRepo: args.HistoryRepo, + epochNotifier: args.CoreData.EpochNotifier(), + statusCoreComponents: args.StatusCoreComponents, + flagsConfig: args.FlagsConfig, + txExecutionOrderHandler: args.TxExecutionOrderHandler, + genesisNonce: args.GenesisNonce, + genesisRound: args.GenesisRound, + roundConfig: args.RoundConfig, + interceptedDataVerifierFactory: args.InterceptedDataVerifierFactory, }, nil } @@ -1506,8 +1506,6 @@ func (pcf *processComponentsFactory) newInterceptorContainerFactory( nodeOperationMode = common.FullArchiveMode } - processedMessagesCacheMap := make(map[string]storage.Cacher) - shardCoordinator := pcf.bootstrapComponents.ShardCoordinator() if shardCoordinator.SelfId() < shardCoordinator.NumberOfShards() { return pcf.newShardInterceptorContainerFactory( @@ -1520,7 +1518,6 @@ func (pcf *processComponentsFactory) newInterceptorContainerFactory( fullArchivePeerShardMapper, hardforkTrigger, nodeOperationMode, - processedMessagesCacheMap, ) } if shardCoordinator.SelfId() == core.MetachainShardId { @@ -1534,7 +1531,6 @@ func (pcf *processComponentsFactory) newInterceptorContainerFactory( fullArchivePeerShardMapper, hardforkTrigger, nodeOperationMode, - processedMessagesCacheMap, ) } @@ -1674,41 +1670,40 @@ func (pcf *processComponentsFactory) newShardInterceptorContainerFactory( fullArchivePeerShardMapper *networksharding.PeerShardMapper, hardforkTrigger factory.HardforkTrigger, nodeOperationMode common.NodeOperation, - processedMessagesCacheMap map[string]storage.Cacher, ) (process.InterceptorsContainerFactory, process.TimeCacher, error) { headerBlackList := cache.NewTimeCache(timeSpanForBadHeaders) shardInterceptorsContainerFactoryArgs := interceptorscontainer.CommonInterceptorsContainerFactoryArgs{ - CoreComponents: pcf.coreData, - CryptoComponents: pcf.crypto, - Accounts: pcf.state.AccountsAdapter(), - ShardCoordinator: pcf.bootstrapComponents.ShardCoordinator(), - NodesCoordinator: pcf.nodesCoordinator, - MainMessenger: pcf.network.NetworkMessenger(), - FullArchiveMessenger: pcf.network.FullArchiveNetworkMessenger(), - Store: pcf.data.StorageService(), - DataPool: pcf.data.Datapool(), - MaxTxNonceDeltaAllowed: common.MaxTxNonceDeltaAllowed, - TxFeeHandler: pcf.coreData.EconomicsData(), - BlockBlackList: headerBlackList, - HeaderSigVerifier: headerSigVerifier, - HeaderIntegrityVerifier: headerIntegrityVerifier, - ValidityAttester: validityAttester, - EpochStartTrigger: epochStartTrigger, - WhiteListHandler: pcf.whiteListHandler, - WhiteListerVerifiedTxs: pcf.whiteListerVerifiedTxs, - AntifloodHandler: pcf.network.InputAntiFloodHandler(), - ArgumentsParser: smartContract.NewArgumentParser(), - PreferredPeersHolder: pcf.network.PreferredPeersHolderHandler(), - SizeCheckDelta: pcf.config.Marshalizer.SizeCheckDelta, - RequestHandler: requestHandler, - PeerSignatureHandler: pcf.crypto.PeerSignatureHandler(), - SignaturesHandler: pcf.network.NetworkMessenger(), - HeartbeatExpiryTimespanInSec: pcf.config.HeartbeatV2.HeartbeatExpiryTimespanInSec, - MainPeerShardMapper: mainPeerShardMapper, - FullArchivePeerShardMapper: fullArchivePeerShardMapper, - HardforkTrigger: hardforkTrigger, - NodeOperationMode: nodeOperationMode, - ProcessedMessagesCacheMap: processedMessagesCacheMap, + CoreComponents: pcf.coreData, + CryptoComponents: pcf.crypto, + Accounts: pcf.state.AccountsAdapter(), + ShardCoordinator: pcf.bootstrapComponents.ShardCoordinator(), + NodesCoordinator: pcf.nodesCoordinator, + MainMessenger: pcf.network.NetworkMessenger(), + FullArchiveMessenger: pcf.network.FullArchiveNetworkMessenger(), + Store: pcf.data.StorageService(), + DataPool: pcf.data.Datapool(), + MaxTxNonceDeltaAllowed: common.MaxTxNonceDeltaAllowed, + TxFeeHandler: pcf.coreData.EconomicsData(), + BlockBlackList: headerBlackList, + HeaderSigVerifier: headerSigVerifier, + HeaderIntegrityVerifier: headerIntegrityVerifier, + ValidityAttester: validityAttester, + EpochStartTrigger: epochStartTrigger, + WhiteListHandler: pcf.whiteListHandler, + WhiteListerVerifiedTxs: pcf.whiteListerVerifiedTxs, + AntifloodHandler: pcf.network.InputAntiFloodHandler(), + ArgumentsParser: smartContract.NewArgumentParser(), + PreferredPeersHolder: pcf.network.PreferredPeersHolderHandler(), + SizeCheckDelta: pcf.config.Marshalizer.SizeCheckDelta, + RequestHandler: requestHandler, + PeerSignatureHandler: pcf.crypto.PeerSignatureHandler(), + SignaturesHandler: pcf.network.NetworkMessenger(), + HeartbeatExpiryTimespanInSec: pcf.config.HeartbeatV2.HeartbeatExpiryTimespanInSec, + MainPeerShardMapper: mainPeerShardMapper, + FullArchivePeerShardMapper: fullArchivePeerShardMapper, + HardforkTrigger: hardforkTrigger, + NodeOperationMode: nodeOperationMode, + InterceptedDataVerifierFactory: pcf.interceptedDataVerifierFactory, } interceptorContainerFactory, err := interceptorscontainer.NewShardInterceptorsContainerFactory(shardInterceptorsContainerFactoryArgs) @@ -1729,41 +1724,40 @@ func (pcf *processComponentsFactory) newMetaInterceptorContainerFactory( fullArchivePeerShardMapper *networksharding.PeerShardMapper, hardforkTrigger factory.HardforkTrigger, nodeOperationMode common.NodeOperation, - processedMessageCacheMap map[string]storage.Cacher, ) (process.InterceptorsContainerFactory, process.TimeCacher, error) { headerBlackList := cache.NewTimeCache(timeSpanForBadHeaders) metaInterceptorsContainerFactoryArgs := interceptorscontainer.CommonInterceptorsContainerFactoryArgs{ - CoreComponents: pcf.coreData, - CryptoComponents: pcf.crypto, - ShardCoordinator: pcf.bootstrapComponents.ShardCoordinator(), - NodesCoordinator: pcf.nodesCoordinator, - MainMessenger: pcf.network.NetworkMessenger(), - FullArchiveMessenger: pcf.network.FullArchiveNetworkMessenger(), - Store: pcf.data.StorageService(), - DataPool: pcf.data.Datapool(), - Accounts: pcf.state.AccountsAdapter(), - MaxTxNonceDeltaAllowed: common.MaxTxNonceDeltaAllowed, - TxFeeHandler: pcf.coreData.EconomicsData(), - BlockBlackList: headerBlackList, - HeaderSigVerifier: headerSigVerifier, - HeaderIntegrityVerifier: headerIntegrityVerifier, - ValidityAttester: validityAttester, - EpochStartTrigger: epochStartTrigger, - WhiteListHandler: pcf.whiteListHandler, - WhiteListerVerifiedTxs: pcf.whiteListerVerifiedTxs, - AntifloodHandler: pcf.network.InputAntiFloodHandler(), - ArgumentsParser: smartContract.NewArgumentParser(), - SizeCheckDelta: pcf.config.Marshalizer.SizeCheckDelta, - PreferredPeersHolder: pcf.network.PreferredPeersHolderHandler(), - RequestHandler: requestHandler, - PeerSignatureHandler: pcf.crypto.PeerSignatureHandler(), - SignaturesHandler: pcf.network.NetworkMessenger(), - HeartbeatExpiryTimespanInSec: pcf.config.HeartbeatV2.HeartbeatExpiryTimespanInSec, - MainPeerShardMapper: mainPeerShardMapper, - FullArchivePeerShardMapper: fullArchivePeerShardMapper, - HardforkTrigger: hardforkTrigger, - NodeOperationMode: nodeOperationMode, - ProcessedMessagesCacheMap: processedMessageCacheMap, + CoreComponents: pcf.coreData, + CryptoComponents: pcf.crypto, + ShardCoordinator: pcf.bootstrapComponents.ShardCoordinator(), + NodesCoordinator: pcf.nodesCoordinator, + MainMessenger: pcf.network.NetworkMessenger(), + FullArchiveMessenger: pcf.network.FullArchiveNetworkMessenger(), + Store: pcf.data.StorageService(), + DataPool: pcf.data.Datapool(), + Accounts: pcf.state.AccountsAdapter(), + MaxTxNonceDeltaAllowed: common.MaxTxNonceDeltaAllowed, + TxFeeHandler: pcf.coreData.EconomicsData(), + BlockBlackList: headerBlackList, + HeaderSigVerifier: headerSigVerifier, + HeaderIntegrityVerifier: headerIntegrityVerifier, + ValidityAttester: validityAttester, + EpochStartTrigger: epochStartTrigger, + WhiteListHandler: pcf.whiteListHandler, + WhiteListerVerifiedTxs: pcf.whiteListerVerifiedTxs, + AntifloodHandler: pcf.network.InputAntiFloodHandler(), + ArgumentsParser: smartContract.NewArgumentParser(), + SizeCheckDelta: pcf.config.Marshalizer.SizeCheckDelta, + PreferredPeersHolder: pcf.network.PreferredPeersHolderHandler(), + RequestHandler: requestHandler, + PeerSignatureHandler: pcf.crypto.PeerSignatureHandler(), + SignaturesHandler: pcf.network.NetworkMessenger(), + HeartbeatExpiryTimespanInSec: pcf.config.HeartbeatV2.HeartbeatExpiryTimespanInSec, + MainPeerShardMapper: mainPeerShardMapper, + FullArchivePeerShardMapper: fullArchivePeerShardMapper, + HardforkTrigger: hardforkTrigger, + NodeOperationMode: nodeOperationMode, + InterceptedDataVerifierFactory: pcf.interceptedDataVerifierFactory, } interceptorContainerFactory, err := interceptorscontainer.NewMetaInterceptorsContainerFactory(metaInterceptorsContainerFactoryArgs) @@ -1863,7 +1857,7 @@ func (pcf *processComponentsFactory) createExportFactoryHandler( NumConcurrentTrieSyncers: pcf.config.TrieSync.NumConcurrentTrieSyncers, TrieSyncerVersion: pcf.config.TrieSync.TrieSyncerVersion, NodeOperationMode: nodeOperationMode, - InterceptedDataCacheMap: pcf.interceptedDataCacheMap, + InterceptedDataVerifierFactory: pcf.interceptedDataVerifierFactory, } return updateFactory.NewExportHandlerFactory(argsExporter) } diff --git a/go.mod b/go.mod index c02ddde8a66..0b1ef7d9632 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/klauspost/cpuid/v2 v2.2.5 github.com/mitchellh/mapstructure v1.5.0 github.com/multiversx/mx-chain-communication-go v1.0.15-0.20240508074652-e128a1c05c8e - github.com/multiversx/mx-chain-core-go v1.2.21-0.20240917083438-99280b4dc9b1 + github.com/multiversx/mx-chain-core-go v1.2.21-0.20240925111815-120b0b610b5a github.com/multiversx/mx-chain-crypto-go v1.2.12-0.20240508074452-cc21c1b505df github.com/multiversx/mx-chain-es-indexer-go v1.7.2-0.20240619122842-05143459c554 github.com/multiversx/mx-chain-logger-go v1.0.15-0.20240508072523-3f00a726af57 diff --git a/go.sum b/go.sum index c166db57fb2..4e99b95dc3d 100644 --- a/go.sum +++ b/go.sum @@ -387,8 +387,8 @@ github.com/multiversx/concurrent-map v0.1.4 h1:hdnbM8VE4b0KYJaGY5yJS2aNIW9TFFsUY github.com/multiversx/concurrent-map v0.1.4/go.mod h1:8cWFRJDOrWHOTNSqgYCUvwT7c7eFQ4U2vKMOp4A/9+o= github.com/multiversx/mx-chain-communication-go v1.0.15-0.20240508074652-e128a1c05c8e h1:Tsmwhu+UleE+l3buPuqXSKTqfu5FbPmzQ4MjMoUvCWA= github.com/multiversx/mx-chain-communication-go v1.0.15-0.20240508074652-e128a1c05c8e/go.mod h1:2yXl18wUbuV3cRZr7VHxM1xo73kTaC1WUcu2kx8R034= -github.com/multiversx/mx-chain-core-go v1.2.21-0.20240917083438-99280b4dc9b1 h1:AObGM2gvQrbFH45HrWBfhgpPRMAQkcAEsZrBN+Vi7ew= -github.com/multiversx/mx-chain-core-go v1.2.21-0.20240917083438-99280b4dc9b1/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= +github.com/multiversx/mx-chain-core-go v1.2.21-0.20240925111815-120b0b610b5a h1:YsPfyNONJsERG+MzJIHRZW6mVIHkUFc8YeKsb20YhhA= +github.com/multiversx/mx-chain-core-go v1.2.21-0.20240925111815-120b0b610b5a/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= github.com/multiversx/mx-chain-crypto-go v1.2.12-0.20240508074452-cc21c1b505df h1:clihfi78bMEOWk/qw6WA4uQbCM2e2NGliqswLAvw19k= github.com/multiversx/mx-chain-crypto-go v1.2.12-0.20240508074452-cc21c1b505df/go.mod h1:gtJYB4rR21KBSqJlazn+2z6f9gFSqQP3KvAgL7Qgxw4= github.com/multiversx/mx-chain-es-indexer-go v1.7.2-0.20240619122842-05143459c554 h1:Fv8BfzJSzdovmoh9Jh/by++0uGsOVBlMP3XiN5Svkn4= diff --git a/node/interface.go b/node/interface.go index 236e7a131e3..05330285fb6 100644 --- a/node/interface.go +++ b/node/interface.go @@ -4,8 +4,9 @@ import ( "io" "github.com/multiversx/mx-chain-core-go/core" - "github.com/multiversx/mx-chain-go/update" vmcommon "github.com/multiversx/mx-chain-vm-common-go" + + "github.com/multiversx/mx-chain-go/update" ) // NetworkShardingCollector defines the updating methods used by the network sharding component diff --git a/node/nodeRunner.go b/node/nodeRunner.go index 23dd7a6fc15..478e0ab4899 100644 --- a/node/nodeRunner.go +++ b/node/nodeRunner.go @@ -56,9 +56,9 @@ import ( "github.com/multiversx/mx-chain-go/outport" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/interceptors" + "github.com/multiversx/mx-chain-go/process/interceptors/factory" "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator" "github.com/multiversx/mx-chain-go/state/syncer" - "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/storage/cache" storageFactory "github.com/multiversx/mx-chain-go/storage/factory" "github.com/multiversx/mx-chain-go/storage/storageunit" @@ -323,8 +323,11 @@ func (nr *nodeRunner) executeOneComponentCreationCycle( } log.Debug("creating bootstrap components") - interceptedDataCache := make(map[string]storage.Cacher) - managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents, interceptedDataCache) + interceptedDataVerifierFactory := factory.NewInterceptedDataVerifierFactory(factory.InterceptedDataVerifierFactoryArgs{ + CacheSpan: time.Duration(nr.configs.GeneralConfig.InterceptedDataVerifier.CacheExpiryInSec), + CacheExpiry: time.Duration(nr.configs.GeneralConfig.InterceptedDataVerifier.CacheExpiryInSec), + }) + managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents, interceptedDataVerifierFactory) if err != nil { return true, err } @@ -435,7 +438,7 @@ func (nr *nodeRunner) executeOneComponentCreationCycle( managedStatusCoreComponents, gasScheduleNotifier, nodesCoordinatorInstance, - interceptedDataCache, + interceptedDataVerifierFactory, ) if err != nil { return true, err @@ -1161,7 +1164,7 @@ func (nr *nodeRunner) CreateManagedProcessComponents( statusCoreComponents mainFactory.StatusCoreComponentsHolder, gasScheduleNotifier core.GasScheduleNotifier, nodesCoordinator nodesCoordinator.NodesCoordinator, - interceptedDataCacheMap map[string]storage.Cacher, + interceptedDataVerifierFactory process.InterceptedDataVerifierFactory, ) (mainFactory.ProcessComponentsHandler, error) { configs := nr.configs configurationPaths := nr.configs.ConfigurationPathsHolder @@ -1242,34 +1245,34 @@ func (nr *nodeRunner) CreateManagedProcessComponents( txExecutionOrderHandler := ordering.NewOrderedCollection() processArgs := processComp.ProcessComponentsFactoryArgs{ - Config: *configs.GeneralConfig, - EpochConfig: *configs.EpochConfig, - RoundConfig: *configs.RoundConfig, - PrefConfigs: *configs.PreferencesConfig, - ImportDBConfig: *configs.ImportDbConfig, - EconomicsConfig: *configs.EconomicsConfig, - AccountsParser: accountsParser, - SmartContractParser: smartContractParser, - GasSchedule: gasScheduleNotifier, - NodesCoordinator: nodesCoordinator, - Data: dataComponents, - CoreData: coreComponents, - Crypto: cryptoComponents, - State: stateComponents, - Network: networkComponents, - BootstrapComponents: bootstrapComponents, - StatusComponents: statusComponents, - StatusCoreComponents: statusCoreComponents, - RequestedItemsHandler: requestedItemsHandler, - WhiteListHandler: whiteListRequest, - WhiteListerVerifiedTxs: whiteListerVerifiedTxs, - MaxRating: configs.RatingsConfig.General.MaxRating, - SystemSCConfig: configs.SystemSCConfig, - ImportStartHandler: importStartHandler, - HistoryRepo: historyRepository, - FlagsConfig: *configs.FlagsConfig, - TxExecutionOrderHandler: txExecutionOrderHandler, - InterceptedDataCacheMap: interceptedDataCacheMap, + Config: *configs.GeneralConfig, + EpochConfig: *configs.EpochConfig, + RoundConfig: *configs.RoundConfig, + PrefConfigs: *configs.PreferencesConfig, + ImportDBConfig: *configs.ImportDbConfig, + EconomicsConfig: *configs.EconomicsConfig, + AccountsParser: accountsParser, + SmartContractParser: smartContractParser, + GasSchedule: gasScheduleNotifier, + NodesCoordinator: nodesCoordinator, + Data: dataComponents, + CoreData: coreComponents, + Crypto: cryptoComponents, + State: stateComponents, + Network: networkComponents, + BootstrapComponents: bootstrapComponents, + StatusComponents: statusComponents, + StatusCoreComponents: statusCoreComponents, + RequestedItemsHandler: requestedItemsHandler, + WhiteListHandler: whiteListRequest, + WhiteListerVerifiedTxs: whiteListerVerifiedTxs, + MaxRating: configs.RatingsConfig.General.MaxRating, + SystemSCConfig: configs.SystemSCConfig, + ImportStartHandler: importStartHandler, + HistoryRepo: historyRepository, + FlagsConfig: *configs.FlagsConfig, + TxExecutionOrderHandler: txExecutionOrderHandler, + InterceptedDataVerifierFactory: interceptedDataVerifierFactory, } processComponentsFactory, err := processComp.NewProcessComponentsFactory(processArgs) if err != nil { @@ -1383,20 +1386,20 @@ func (nr *nodeRunner) CreateManagedBootstrapComponents( coreComponents mainFactory.CoreComponentsHolder, cryptoComponents mainFactory.CryptoComponentsHolder, networkComponents mainFactory.NetworkComponentsHolder, - interceptedDataCacheMap map[string]storage.Cacher, + interceptedDataVerifierFactory process.InterceptedDataVerifierFactory, ) (mainFactory.BootstrapComponentsHandler, error) { bootstrapComponentsFactoryArgs := bootstrapComp.BootstrapComponentsFactoryArgs{ - Config: *nr.configs.GeneralConfig, - PrefConfig: *nr.configs.PreferencesConfig, - ImportDbConfig: *nr.configs.ImportDbConfig, - FlagsConfig: *nr.configs.FlagsConfig, - WorkingDir: nr.configs.FlagsConfig.DbDir, - CoreComponents: coreComponents, - CryptoComponents: cryptoComponents, - NetworkComponents: networkComponents, - StatusCoreComponents: statusCoreComponents, - InterceptedDataCacheMap: interceptedDataCacheMap, + Config: *nr.configs.GeneralConfig, + PrefConfig: *nr.configs.PreferencesConfig, + ImportDbConfig: *nr.configs.ImportDbConfig, + FlagsConfig: *nr.configs.FlagsConfig, + WorkingDir: nr.configs.FlagsConfig.DbDir, + CoreComponents: coreComponents, + CryptoComponents: cryptoComponents, + NetworkComponents: networkComponents, + StatusCoreComponents: statusCoreComponents, + InterceptedDataVerifierFactory: interceptedDataVerifierFactory, } bootstrapComponentsFactory, err := bootstrapComp.NewBootstrapComponentsFactory(bootstrapComponentsFactoryArgs) diff --git a/process/factory/interceptorscontainer/args.go b/process/factory/interceptorscontainer/args.go index dd08954a3ff..8e98c7c18ab 100644 --- a/process/factory/interceptorscontainer/args.go +++ b/process/factory/interceptorscontainer/args.go @@ -10,40 +10,39 @@ import ( "github.com/multiversx/mx-chain-go/sharding" "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator" "github.com/multiversx/mx-chain-go/state" - "github.com/multiversx/mx-chain-go/storage" ) // CommonInterceptorsContainerFactoryArgs holds the arguments needed for the metachain/shard interceptors factories type CommonInterceptorsContainerFactoryArgs struct { - CoreComponents process.CoreComponentsHolder - CryptoComponents process.CryptoComponentsHolder - Accounts state.AccountsAdapter - ShardCoordinator sharding.Coordinator - NodesCoordinator nodesCoordinator.NodesCoordinator - MainMessenger process.TopicHandler - FullArchiveMessenger process.TopicHandler - Store dataRetriever.StorageService - DataPool dataRetriever.PoolsHolder - MaxTxNonceDeltaAllowed int - TxFeeHandler process.FeeHandler - BlockBlackList process.TimeCacher - HeaderSigVerifier process.InterceptedHeaderSigVerifier - HeaderIntegrityVerifier process.HeaderIntegrityVerifier - ValidityAttester process.ValidityAttester - EpochStartTrigger process.EpochStartTriggerHandler - WhiteListHandler process.WhiteListHandler - WhiteListerVerifiedTxs process.WhiteListHandler - AntifloodHandler process.P2PAntifloodHandler - ArgumentsParser process.ArgumentsParser - PreferredPeersHolder process.PreferredPeersHolderHandler - SizeCheckDelta uint32 - RequestHandler process.RequestHandler - PeerSignatureHandler crypto.PeerSignatureHandler - SignaturesHandler process.SignaturesHandler - HeartbeatExpiryTimespanInSec int64 - MainPeerShardMapper process.PeerShardMapper - FullArchivePeerShardMapper process.PeerShardMapper - HardforkTrigger heartbeat.HardforkTrigger - NodeOperationMode common.NodeOperation - ProcessedMessagesCacheMap map[string]storage.Cacher + CoreComponents process.CoreComponentsHolder + CryptoComponents process.CryptoComponentsHolder + Accounts state.AccountsAdapter + ShardCoordinator sharding.Coordinator + NodesCoordinator nodesCoordinator.NodesCoordinator + MainMessenger process.TopicHandler + FullArchiveMessenger process.TopicHandler + Store dataRetriever.StorageService + DataPool dataRetriever.PoolsHolder + MaxTxNonceDeltaAllowed int + TxFeeHandler process.FeeHandler + BlockBlackList process.TimeCacher + HeaderSigVerifier process.InterceptedHeaderSigVerifier + HeaderIntegrityVerifier process.HeaderIntegrityVerifier + ValidityAttester process.ValidityAttester + EpochStartTrigger process.EpochStartTriggerHandler + WhiteListHandler process.WhiteListHandler + WhiteListerVerifiedTxs process.WhiteListHandler + AntifloodHandler process.P2PAntifloodHandler + ArgumentsParser process.ArgumentsParser + PreferredPeersHolder process.PreferredPeersHolderHandler + SizeCheckDelta uint32 + RequestHandler process.RequestHandler + PeerSignatureHandler crypto.PeerSignatureHandler + SignaturesHandler process.SignaturesHandler + HeartbeatExpiryTimespanInSec int64 + MainPeerShardMapper process.PeerShardMapper + FullArchivePeerShardMapper process.PeerShardMapper + HardforkTrigger heartbeat.HardforkTrigger + NodeOperationMode common.NodeOperation + InterceptedDataVerifierFactory process.InterceptedDataVerifierFactory } diff --git a/process/factory/interceptorscontainer/baseInterceptorsContainerFactory.go b/process/factory/interceptorscontainer/baseInterceptorsContainerFactory.go index bafc3de6d7c..518ca765633 100644 --- a/process/factory/interceptorscontainer/baseInterceptorsContainerFactory.go +++ b/process/factory/interceptorscontainer/baseInterceptorsContainerFactory.go @@ -21,7 +21,6 @@ import ( "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator" "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/storage" - "github.com/multiversx/mx-chain-go/storage/cache" "github.com/multiversx/mx-chain-go/testscommon/processMocks" ) @@ -31,35 +30,33 @@ const ( minTimespanDurationInSec = int64(1) errorOnMainNetworkString = "on main network" errorOnFullArchiveNetworkString = "on full archive network" - cacheDefaultSpan = 30 * time.Second - cacheDefaultExpiry = 30 * time.Second ) type baseInterceptorsContainerFactory struct { - mainContainer process.InterceptorsContainer - fullArchiveContainer process.InterceptorsContainer - shardCoordinator sharding.Coordinator - accounts state.AccountsAdapter - store dataRetriever.StorageService - dataPool dataRetriever.PoolsHolder - mainMessenger process.TopicHandler - fullArchiveMessenger process.TopicHandler - nodesCoordinator nodesCoordinator.NodesCoordinator - blockBlackList process.TimeCacher - argInterceptorFactory *interceptorFactory.ArgInterceptedDataFactory - globalThrottler process.InterceptorThrottler - maxTxNonceDeltaAllowed int - antifloodHandler process.P2PAntifloodHandler - whiteListHandler process.WhiteListHandler - whiteListerVerifiedTxs process.WhiteListHandler - preferredPeersHolder process.PreferredPeersHolderHandler - hasher hashing.Hasher - requestHandler process.RequestHandler - mainPeerShardMapper process.PeerShardMapper - fullArchivePeerShardMapper process.PeerShardMapper - hardforkTrigger heartbeat.HardforkTrigger - nodeOperationMode common.NodeOperation - processedMessagesCacheMap map[string]storage.Cacher + mainContainer process.InterceptorsContainer + fullArchiveContainer process.InterceptorsContainer + shardCoordinator sharding.Coordinator + accounts state.AccountsAdapter + store dataRetriever.StorageService + dataPool dataRetriever.PoolsHolder + mainMessenger process.TopicHandler + fullArchiveMessenger process.TopicHandler + nodesCoordinator nodesCoordinator.NodesCoordinator + blockBlackList process.TimeCacher + argInterceptorFactory *interceptorFactory.ArgInterceptedDataFactory + globalThrottler process.InterceptorThrottler + maxTxNonceDeltaAllowed int + antifloodHandler process.P2PAntifloodHandler + whiteListHandler process.WhiteListHandler + whiteListerVerifiedTxs process.WhiteListHandler + preferredPeersHolder process.PreferredPeersHolderHandler + hasher hashing.Hasher + requestHandler process.RequestHandler + mainPeerShardMapper process.PeerShardMapper + fullArchivePeerShardMapper process.PeerShardMapper + hardforkTrigger heartbeat.HardforkTrigger + nodeOperationMode common.NodeOperation + interceptedDataVerifierFactory process.InterceptedDataVerifierFactory } func checkBaseParams( @@ -291,7 +288,7 @@ func (bicf *baseInterceptorsContainerFactory) createOneTxInterceptor(topic strin return nil, err } - interceptedDataVerifier, err := bicf.createCacheForInterceptor(topic) + interceptedDataVerifier, err := bicf.interceptedDataVerifierFactory.Create(topic) if err != nil { return nil, err } @@ -340,7 +337,7 @@ func (bicf *baseInterceptorsContainerFactory) createOneUnsignedTxInterceptor(top return nil, err } - interceptedDataVerifier, err := bicf.createCacheForInterceptor(topic) + interceptedDataVerifier, err := bicf.interceptedDataVerifierFactory.Create(topic) if err != nil { return nil, err } @@ -389,7 +386,7 @@ func (bicf *baseInterceptorsContainerFactory) createOneRewardTxInterceptor(topic return nil, err } - interceptedDataVerifier, err := bicf.createCacheForInterceptor(topic) + interceptedDataVerifier, err := bicf.interceptedDataVerifierFactory.Create(topic) if err != nil { return nil, err } @@ -438,7 +435,7 @@ func (bicf *baseInterceptorsContainerFactory) generateHeaderInterceptors() error // compose header shard topic, for example: shardBlocks_0_META identifierHdr := factory.ShardBlocksTopic + shardC.CommunicationIdentifier(core.MetachainShardId) - interceptedDataVerifier, err := bicf.createCacheForInterceptor(identifierHdr) + interceptedDataVerifier, err := bicf.interceptedDataVerifierFactory.Create(identifierHdr) if err != nil { return err } @@ -532,7 +529,7 @@ func (bicf *baseInterceptorsContainerFactory) createOneMiniBlocksInterceptor(top return nil, err } - interceptedDataVerifier, err := bicf.createCacheForInterceptor(topic) + interceptedDataVerifier, err := bicf.interceptedDataVerifierFactory.Create(topic) if err != nil { return nil, err } @@ -577,7 +574,7 @@ func (bicf *baseInterceptorsContainerFactory) generateMetachainHeaderInterceptor return err } - interceptedDataVerifier, err := bicf.createCacheForInterceptor(identifierHdr) + interceptedDataVerifier, err := bicf.interceptedDataVerifierFactory.Create(identifierHdr) if err != nil { return err } @@ -619,7 +616,7 @@ func (bicf *baseInterceptorsContainerFactory) createOneTrieNodesInterceptor(topi return nil, err } - interceptedDataVerifier, err := bicf.createCacheForInterceptor(topic) + interceptedDataVerifier, err := bicf.interceptedDataVerifierFactory.Create(topic) if err != nil { return nil, err } @@ -717,7 +714,7 @@ func (bicf *baseInterceptorsContainerFactory) generatePeerAuthenticationIntercep return err } - interceptedDataVerifier, err := bicf.createCacheForInterceptor(identifierPeerAuthentication) + interceptedDataVerifier, err := bicf.interceptedDataVerifierFactory.Create(identifierPeerAuthentication) if err != nil { return err } @@ -782,7 +779,7 @@ func (bicf *baseInterceptorsContainerFactory) createHeartbeatV2Interceptor( return nil, err } - interceptedDataVerifier, err := bicf.createCacheForInterceptor(identifier) + interceptedDataVerifier, err := bicf.interceptedDataVerifierFactory.Create(identifier) if err != nil { return nil, err } @@ -837,7 +834,7 @@ func (bicf *baseInterceptorsContainerFactory) createPeerShardInterceptor( return nil, err } - interceptedDataVerifier, err := bicf.createCacheForInterceptor(identifier) + interceptedDataVerifier, err := bicf.interceptedDataVerifierFactory.Create(identifier) if err != nil { return nil, err } @@ -880,7 +877,7 @@ func (bicf *baseInterceptorsContainerFactory) generateValidatorInfoInterceptor() return err } - interceptedDataVerifier, err := bicf.createCacheForInterceptor(identifier) + interceptedDataVerifier, err := bicf.interceptedDataVerifierFactory.Create(identifier) if err != nil { return err } @@ -924,7 +921,7 @@ func (bicf *baseInterceptorsContainerFactory) createOneShardEquivalentProofsInte return nil, err } - interceptedDataVerifier, err := bicf.createCacheForInterceptor(topic) + interceptedDataVerifier, err := bicf.interceptedDataVerifierFactory.Create(topic) if err != nil { return nil, err } @@ -962,16 +959,16 @@ func (bicf *baseInterceptorsContainerFactory) addInterceptorsToContainers(keys [ return bicf.fullArchiveContainer.AddMultiple(keys, interceptors) } -func (bicf *baseInterceptorsContainerFactory) createCacheForInterceptor(topic string) (process.InterceptedDataVerifier, error) { - internalCache, err := cache.NewTimeCacher(cache.ArgTimeCacher{ - DefaultSpan: cacheDefaultSpan, - CacheExpiry: cacheDefaultExpiry, - }) - if err != nil { - return nil, err - } - - bicf.processedMessagesCacheMap[topic] = internalCache - verifier := interceptors.NewInterceptedDataVerifier(internalCache) - return verifier, nil -} +//func (bicf *baseInterceptorsContainerFactory) createCacheForInterceptor(topic string) (process.InterceptedDataVerifier, error) { +// internalCache, err := cache.NewTimeCacher(cache.ArgTimeCacher{ +// DefaultSpan: cacheDefaultSpan, +// CacheExpiry: cacheDefaultExpiry, +// }) +// if err != nil { +// return nil, err +// } +// +// bicf.processedMessagesCacheMap[topic] = internalCache +// verifier := interceptors.NewInterceptedDataVerifier(internalCache) +// return verifier, nil +//} diff --git a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go index 42c85e85084..6d92b8a34c4 100644 --- a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go +++ b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go @@ -104,29 +104,29 @@ func NewMetaInterceptorsContainerFactory( } base := &baseInterceptorsContainerFactory{ - mainContainer: containers.NewInterceptorsContainer(), - fullArchiveContainer: containers.NewInterceptorsContainer(), - shardCoordinator: args.ShardCoordinator, - mainMessenger: args.MainMessenger, - fullArchiveMessenger: args.FullArchiveMessenger, - store: args.Store, - dataPool: args.DataPool, - nodesCoordinator: args.NodesCoordinator, - blockBlackList: args.BlockBlackList, - argInterceptorFactory: argInterceptorFactory, - maxTxNonceDeltaAllowed: args.MaxTxNonceDeltaAllowed, - accounts: args.Accounts, - antifloodHandler: args.AntifloodHandler, - whiteListHandler: args.WhiteListHandler, - whiteListerVerifiedTxs: args.WhiteListerVerifiedTxs, - preferredPeersHolder: args.PreferredPeersHolder, - hasher: args.CoreComponents.Hasher(), - requestHandler: args.RequestHandler, - mainPeerShardMapper: args.MainPeerShardMapper, - fullArchivePeerShardMapper: args.FullArchivePeerShardMapper, - hardforkTrigger: args.HardforkTrigger, - nodeOperationMode: args.NodeOperationMode, - processedMessagesCacheMap: args.ProcessedMessagesCacheMap, + mainContainer: containers.NewInterceptorsContainer(), + fullArchiveContainer: containers.NewInterceptorsContainer(), + shardCoordinator: args.ShardCoordinator, + mainMessenger: args.MainMessenger, + fullArchiveMessenger: args.FullArchiveMessenger, + store: args.Store, + dataPool: args.DataPool, + nodesCoordinator: args.NodesCoordinator, + blockBlackList: args.BlockBlackList, + argInterceptorFactory: argInterceptorFactory, + maxTxNonceDeltaAllowed: args.MaxTxNonceDeltaAllowed, + accounts: args.Accounts, + antifloodHandler: args.AntifloodHandler, + whiteListHandler: args.WhiteListHandler, + whiteListerVerifiedTxs: args.WhiteListerVerifiedTxs, + preferredPeersHolder: args.PreferredPeersHolder, + hasher: args.CoreComponents.Hasher(), + requestHandler: args.RequestHandler, + mainPeerShardMapper: args.MainPeerShardMapper, + fullArchivePeerShardMapper: args.FullArchivePeerShardMapper, + hardforkTrigger: args.HardforkTrigger, + nodeOperationMode: args.NodeOperationMode, + interceptedDataVerifierFactory: args.InterceptedDataVerifierFactory, } icf := &metaInterceptorsContainerFactory{ @@ -269,7 +269,7 @@ func (micf *metaInterceptorsContainerFactory) createOneShardHeaderInterceptor(to return nil, err } - interceptedDataVerifier, err := micf.createCacheForInterceptor(topic) + interceptedDataVerifier, err := micf.interceptedDataVerifierFactory.Create(topic) if err != nil { return nil, err } diff --git a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go index e1d0b3fbef6..bdc72590993 100644 --- a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go +++ b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go @@ -104,29 +104,29 @@ func NewShardInterceptorsContainerFactory( } base := &baseInterceptorsContainerFactory{ - mainContainer: containers.NewInterceptorsContainer(), - fullArchiveContainer: containers.NewInterceptorsContainer(), - accounts: args.Accounts, - shardCoordinator: args.ShardCoordinator, - mainMessenger: args.MainMessenger, - fullArchiveMessenger: args.FullArchiveMessenger, - store: args.Store, - dataPool: args.DataPool, - nodesCoordinator: args.NodesCoordinator, - argInterceptorFactory: argInterceptorFactory, - blockBlackList: args.BlockBlackList, - maxTxNonceDeltaAllowed: args.MaxTxNonceDeltaAllowed, - antifloodHandler: args.AntifloodHandler, - whiteListHandler: args.WhiteListHandler, - whiteListerVerifiedTxs: args.WhiteListerVerifiedTxs, - preferredPeersHolder: args.PreferredPeersHolder, - hasher: args.CoreComponents.Hasher(), - requestHandler: args.RequestHandler, - mainPeerShardMapper: args.MainPeerShardMapper, - fullArchivePeerShardMapper: args.FullArchivePeerShardMapper, - hardforkTrigger: args.HardforkTrigger, - nodeOperationMode: args.NodeOperationMode, - processedMessagesCacheMap: args.ProcessedMessagesCacheMap, + mainContainer: containers.NewInterceptorsContainer(), + fullArchiveContainer: containers.NewInterceptorsContainer(), + accounts: args.Accounts, + shardCoordinator: args.ShardCoordinator, + mainMessenger: args.MainMessenger, + fullArchiveMessenger: args.FullArchiveMessenger, + store: args.Store, + dataPool: args.DataPool, + nodesCoordinator: args.NodesCoordinator, + argInterceptorFactory: argInterceptorFactory, + blockBlackList: args.BlockBlackList, + maxTxNonceDeltaAllowed: args.MaxTxNonceDeltaAllowed, + antifloodHandler: args.AntifloodHandler, + whiteListHandler: args.WhiteListHandler, + whiteListerVerifiedTxs: args.WhiteListerVerifiedTxs, + preferredPeersHolder: args.PreferredPeersHolder, + hasher: args.CoreComponents.Hasher(), + requestHandler: args.RequestHandler, + mainPeerShardMapper: args.MainPeerShardMapper, + fullArchivePeerShardMapper: args.FullArchivePeerShardMapper, + hardforkTrigger: args.HardforkTrigger, + nodeOperationMode: args.NodeOperationMode, + interceptedDataVerifierFactory: args.InterceptedDataVerifierFactory, } icf := &shardInterceptorsContainerFactory{ diff --git a/process/interceptors/factory/interceptedDataVerifierFactory.go b/process/interceptors/factory/interceptedDataVerifierFactory.go new file mode 100644 index 00000000000..96d5a8bb2c3 --- /dev/null +++ b/process/interceptors/factory/interceptedDataVerifierFactory.go @@ -0,0 +1,48 @@ +package factory + +import ( + "time" + + "github.com/multiversx/mx-chain-go/process" + "github.com/multiversx/mx-chain-go/process/interceptors" + "github.com/multiversx/mx-chain-go/storage" + "github.com/multiversx/mx-chain-go/storage/cache" +) + +// InterceptedDataVerifierFactoryArgs holds the required arguments for InterceptedDataVerifierFactory +type InterceptedDataVerifierFactoryArgs struct { + CacheSpan time.Duration + CacheExpiry time.Duration +} + +// InterceptedDataVerifierFactory encapsulates the required arguments to create InterceptedDataVerifier +// Furthermore it will hold all such instances in an internal map. +type InterceptedDataVerifierFactory struct { + cacheSpan time.Duration + cacheExpiry time.Duration + interceptedDataVerifierMap map[string]storage.Cacher +} + +// NewInterceptedDataVerifierFactory will create a factory instance that will create instance of InterceptedDataVerifiers +func NewInterceptedDataVerifierFactory(args InterceptedDataVerifierFactoryArgs) *InterceptedDataVerifierFactory { + return &InterceptedDataVerifierFactory{ + cacheSpan: args.CacheSpan, + cacheExpiry: args.CacheExpiry, + interceptedDataVerifierMap: make(map[string]storage.Cacher), + } +} + +// Create will return an instance of InterceptedDataVerifier +func (idvf *InterceptedDataVerifierFactory) Create(topic string) (process.InterceptedDataVerifier, error) { + internalCache, err := cache.NewTimeCacher(cache.ArgTimeCacher{ + DefaultSpan: idvf.cacheSpan, + CacheExpiry: idvf.cacheExpiry, + }) + if err != nil { + return nil, err + } + + idvf.interceptedDataVerifierMap[topic] = internalCache + verifier := interceptors.NewInterceptedDataVerifier(internalCache) + return verifier, nil +} diff --git a/process/interceptors/interceptedDataVerifier.go b/process/interceptors/interceptedDataVerifier.go index d6c1019084f..00ea4dbadb6 100644 --- a/process/interceptors/interceptedDataVerifier.go +++ b/process/interceptors/interceptedDataVerifier.go @@ -19,11 +19,18 @@ var ( ) type interceptedDataVerifier struct { + //km sync.KeyRWMutexHandler cache storage.Cacher } +// NewInterceptedDataVerifier creates a new instance of intercepted data verifier func NewInterceptedDataVerifier(cache storage.Cacher) *interceptedDataVerifier { - return &interceptedDataVerifier{cache: cache} + //keyRWMutex := sync.NewKeyRWMutex() + + return &interceptedDataVerifier{ + //km: keyRWMutex, + cache: cache, + } } // Verify will check if the intercepted data has been validated before and put in the time cache. @@ -31,7 +38,7 @@ func NewInterceptedDataVerifier(cache storage.Cacher) *interceptedDataVerifier { // validation in the cache. Note that the entries are stored for a set period of time func (idv *interceptedDataVerifier) Verify(interceptedData process.InterceptedData) error { if len(interceptedData.Hash()) == 0 { - return nil + return interceptedData.CheckValidity() } if val, ok := idv.cache.Get(interceptedData.Hash()); ok { @@ -42,7 +49,7 @@ func (idv *interceptedDataVerifier) Verify(interceptedData process.InterceptedDa return ErrInvalidInterceptedData } - err := interceptedData.CheckValidity() + err := idv.checkValidity(interceptedData) if err != nil { idv.cache.Put(interceptedData.Hash(), InvalidInterceptedData, 8) return ErrInvalidInterceptedData @@ -56,3 +63,12 @@ func (idv *interceptedDataVerifier) Verify(interceptedData process.InterceptedDa func (idv *interceptedDataVerifier) IsInterfaceNil() bool { return idv == nil } + +func (idv *interceptedDataVerifier) checkValidity(interceptedData process.InterceptedData) error { + //hash := string(interceptedData.Hash()) + + //idv.km.Lock(hash) + //defer idv.km.Unlock(hash) + + return interceptedData.CheckValidity() +} diff --git a/process/interface.go b/process/interface.go index d39e91f8e8d..e12443fcb48 100644 --- a/process/interface.go +++ b/process/interface.go @@ -1406,3 +1406,7 @@ type InterceptedDataVerifier interface { Verify(interceptedData InterceptedData) error IsInterfaceNil() bool } + +type InterceptedDataVerifierFactory interface { + Create(topic string) (InterceptedDataVerifier, error) +} diff --git a/update/factory/exportHandlerFactory.go b/update/factory/exportHandlerFactory.go index f743c6f7bbe..0cda7a5d2e0 100644 --- a/update/factory/exportHandlerFactory.go +++ b/update/factory/exportHandlerFactory.go @@ -70,7 +70,7 @@ type ArgsExporter struct { TrieSyncerVersion int CheckNodesOnDisk bool NodeOperationMode common.NodeOperation - InterceptedDataCacheMap map[string]storage.Cacher + InterceptedDataVerifierFactory process.InterceptedDataVerifierFactory } type exportHandlerFactory struct { @@ -110,7 +110,7 @@ type exportHandlerFactory struct { trieSyncerVersion int checkNodesOnDisk bool nodeOperationMode common.NodeOperation - interceptedDataCacheMap map[string]storage.Cacher + interceptedDataVerifierFactory process.InterceptedDataVerifierFactory } // NewExportHandlerFactory creates an exporter factory @@ -269,7 +269,7 @@ func NewExportHandlerFactory(args ArgsExporter) (*exportHandlerFactory, error) { checkNodesOnDisk: args.CheckNodesOnDisk, statusCoreComponents: args.StatusCoreComponents, nodeOperationMode: args.NodeOperationMode, - interceptedDataCacheMap: args.InterceptedDataCacheMap, + interceptedDataVerifierFactory: args.InterceptedDataVerifierFactory, } return e, nil @@ -592,7 +592,7 @@ func (e *exportHandlerFactory) createInterceptors() error { FullArchiveInterceptorsContainer: e.fullArchiveInterceptorsContainer, AntifloodHandler: e.networkComponents.InputAntiFloodHandler(), NodeOperationMode: e.nodeOperationMode, - InterceptedDataCache: e.interceptedDataCacheMap, + InterceptedDataVerifierFactory: e.interceptedDataVerifierFactory, } fullSyncInterceptors, err := NewFullSyncInterceptorsContainerFactory(argsInterceptors) if err != nil { diff --git a/update/factory/fullSyncInterceptors.go b/update/factory/fullSyncInterceptors.go index 358056a7228..3fde2f96b2f 100644 --- a/update/factory/fullSyncInterceptors.go +++ b/update/factory/fullSyncInterceptors.go @@ -80,7 +80,7 @@ type ArgsNewFullSyncInterceptorsContainerFactory struct { FullArchiveInterceptorsContainer process.InterceptorsContainer AntifloodHandler process.P2PAntifloodHandler NodeOperationMode common.NodeOperation - InterceptedDataCache map[string]storage.Cacher + InterceptedDataVerifierFactory process.InterceptedDataVerifierFactory } // NewFullSyncInterceptorsContainerFactory is responsible for creating a new interceptors factory object From b622085ea6e5b6de26ce52b60b8239dd38ec9423 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Thu, 26 Sep 2024 10:48:56 +0300 Subject: [PATCH 26/37] added mock for interceptedDataVerifierFactory and injected into tests. --- factory/processing/processComponents_test.go | 9 ++++++++- .../bootstrapComponents_test.go | 8 +++++--- .../consensusComponents_test.go | 11 +++++++---- .../dataComponents/dataComponents_test.go | 6 +++--- .../heartbeatComponents_test.go | 11 +++++++---- .../processComponents_test.go | 11 +++++++---- .../stateComponents/stateComponents_test.go | 6 +++--- .../statusComponents/statusComponents_test.go | 11 +++++++---- .../metaInterceptorsContainerFactory_test.go | 16 ++++++++++++---- .../shardInterceptorsContainerFactory_test.go | 16 ++++++++++++---- .../interceptedDataVerifierFactoryStub.go | 19 +++++++++++++++++++ 11 files changed, 90 insertions(+), 34 deletions(-) create mode 100644 process/mock/interceptedDataVerifierFactoryStub.go diff --git a/factory/processing/processComponents_test.go b/factory/processing/processComponents_test.go index a1654ce3ba3..fecfa98165b 100644 --- a/factory/processing/processComponents_test.go +++ b/factory/processing/processComponents_test.go @@ -17,6 +17,8 @@ import ( "github.com/multiversx/mx-chain-core-go/hashing/blake2b" "github.com/multiversx/mx-chain-core-go/hashing/keccak" "github.com/multiversx/mx-chain-core-go/marshal" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/factory" disabledStatistics "github.com/multiversx/mx-chain-go/common/statistics/disabled" @@ -29,6 +31,7 @@ import ( testsMocks "github.com/multiversx/mx-chain-go/integrationTests/mock" "github.com/multiversx/mx-chain-go/p2p" "github.com/multiversx/mx-chain-go/process" + processMocks "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/sharding" "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator" "github.com/multiversx/mx-chain-go/state" @@ -55,7 +58,6 @@ import ( testState "github.com/multiversx/mx-chain-go/testscommon/state" "github.com/multiversx/mx-chain-go/testscommon/statusHandler" updateMocks "github.com/multiversx/mx-chain-go/update/mock" - "github.com/stretchr/testify/require" ) const ( @@ -265,6 +267,11 @@ func createMockProcessComponentsFactoryArgs() processComp.ProcessComponentsFacto } args.State = components.GetStateComponents(args.CoreData, args.StatusCoreComponents) + args.InterceptedDataVerifierFactory = &processMocks.InterceptedDataVerifierFactoryStub{ + CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { + return &processMocks.InterceptedDataVerifierStub{}, nil + }, + } return args } diff --git a/integrationTests/factory/bootstrapComponents/bootstrapComponents_test.go b/integrationTests/factory/bootstrapComponents/bootstrapComponents_test.go index 62cbd9434a8..25a19b3e15d 100644 --- a/integrationTests/factory/bootstrapComponents/bootstrapComponents_test.go +++ b/integrationTests/factory/bootstrapComponents/bootstrapComponents_test.go @@ -10,7 +10,9 @@ import ( "github.com/multiversx/mx-chain-go/integrationTests/factory" "github.com/multiversx/mx-chain-go/node" - "github.com/multiversx/mx-chain-go/storage" + //"github.com/multiversx/mx-chain-go/process" + //factory2 "github.com/multiversx/mx-chain-go/process/interceptors/factory" + "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/testscommon/goroutines" ) @@ -38,8 +40,8 @@ func TestBootstrapComponents_Create_Close_ShouldWork(t *testing.T) { require.Nil(t, err) managedNetworkComponents, err := nr.CreateManagedNetworkComponents(managedCoreComponents, managedStatusCoreComponents, managedCryptoComponents) require.Nil(t, err) - interceptedDataCacheMap := make(map[string]storage.Cacher) - managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents, interceptedDataCacheMap) + interceptedDataVerifierFactory := &mock.InterceptedDataVerifierFactoryStub{} + managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents, interceptedDataVerifierFactory) require.Nil(t, err) require.NotNil(t, managedBootstrapComponents) diff --git a/integrationTests/factory/consensusComponents/consensusComponents_test.go b/integrationTests/factory/consensusComponents/consensusComponents_test.go index 926f1deac78..7fe28403b81 100644 --- a/integrationTests/factory/consensusComponents/consensusComponents_test.go +++ b/integrationTests/factory/consensusComponents/consensusComponents_test.go @@ -13,7 +13,8 @@ import ( bootstrapComp "github.com/multiversx/mx-chain-go/factory/bootstrap" "github.com/multiversx/mx-chain-go/integrationTests/factory" "github.com/multiversx/mx-chain-go/node" - "github.com/multiversx/mx-chain-go/storage" + "github.com/multiversx/mx-chain-go/process" + "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/testscommon/goroutines" ) @@ -41,8 +42,10 @@ func TestConsensusComponents_Close_ShouldWork(t *testing.T) { require.Nil(t, err) managedNetworkComponents, err := nr.CreateManagedNetworkComponents(managedCoreComponents, managedStatusCoreComponents, managedCryptoComponents) require.Nil(t, err) - interceptedDataCacheMap := make(map[string]storage.Cacher) - managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents, interceptedDataCacheMap) + interceptedDataVerifierFactory := &mock.InterceptedDataVerifierFactoryStub{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { + return &mock.InterceptedDataVerifierStub{}, nil + }} + managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents, interceptedDataVerifierFactory) require.Nil(t, err) managedDataComponents, err := nr.CreateManagedDataComponents(managedStatusCoreComponents, managedCoreComponents, managedBootstrapComponents, managedCryptoComponents) require.Nil(t, err) @@ -106,7 +109,7 @@ func TestConsensusComponents_Close_ShouldWork(t *testing.T) { managedStatusCoreComponents, gasScheduleNotifier, nodesCoordinator, - interceptedDataCacheMap, + interceptedDataVerifierFactory, ) require.Nil(t, err) time.Sleep(2 * time.Second) diff --git a/integrationTests/factory/dataComponents/dataComponents_test.go b/integrationTests/factory/dataComponents/dataComponents_test.go index ad9459fe3b4..1641bce6b9c 100644 --- a/integrationTests/factory/dataComponents/dataComponents_test.go +++ b/integrationTests/factory/dataComponents/dataComponents_test.go @@ -10,7 +10,7 @@ import ( "github.com/multiversx/mx-chain-go/integrationTests/factory" "github.com/multiversx/mx-chain-go/node" - "github.com/multiversx/mx-chain-go/storage" + "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/testscommon/goroutines" ) @@ -38,8 +38,8 @@ func TestDataComponents_Create_Close_ShouldWork(t *testing.T) { require.Nil(t, err) managedNetworkComponents, err := nr.CreateManagedNetworkComponents(managedCoreComponents, managedStatusCoreComponents, managedCryptoComponents) require.Nil(t, err) - interceptedDataCacheMap := make(map[string]storage.Cacher) - managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents, interceptedDataCacheMap) + interceptedDataVerifierFactory := &mock.InterceptedDataVerifierFactoryStub{} + managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents, interceptedDataVerifierFactory) require.Nil(t, err) managedDataComponents, err := nr.CreateManagedDataComponents(managedStatusCoreComponents, managedCoreComponents, managedBootstrapComponents, managedCryptoComponents) require.Nil(t, err) diff --git a/integrationTests/factory/heartbeatComponents/heartbeatComponents_test.go b/integrationTests/factory/heartbeatComponents/heartbeatComponents_test.go index 8b7086017d7..384b39e7d37 100644 --- a/integrationTests/factory/heartbeatComponents/heartbeatComponents_test.go +++ b/integrationTests/factory/heartbeatComponents/heartbeatComponents_test.go @@ -13,7 +13,8 @@ import ( bootstrapComp "github.com/multiversx/mx-chain-go/factory/bootstrap" "github.com/multiversx/mx-chain-go/integrationTests/factory" "github.com/multiversx/mx-chain-go/node" - "github.com/multiversx/mx-chain-go/storage" + "github.com/multiversx/mx-chain-go/process" + "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/testscommon/goroutines" ) @@ -41,8 +42,10 @@ func TestHeartbeatComponents_Close_ShouldWork(t *testing.T) { require.Nil(t, err) managedNetworkComponents, err := nr.CreateManagedNetworkComponents(managedCoreComponents, managedStatusCoreComponents, managedCryptoComponents) require.Nil(t, err) - interceptedDataCacheMap := make(map[string]storage.Cacher) - managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents, interceptedDataCacheMap) + interceptedDataVerifierFactory := &mock.InterceptedDataVerifierFactoryStub{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { + return &mock.InterceptedDataVerifierStub{}, nil + }} + managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents, interceptedDataVerifierFactory) require.Nil(t, err) managedDataComponents, err := nr.CreateManagedDataComponents(managedStatusCoreComponents, managedCoreComponents, managedBootstrapComponents, managedCryptoComponents) require.Nil(t, err) @@ -106,7 +109,7 @@ func TestHeartbeatComponents_Close_ShouldWork(t *testing.T) { managedStatusCoreComponents, gasScheduleNotifier, nodesCoordinator, - interceptedDataCacheMap, + interceptedDataVerifierFactory, ) require.Nil(t, err) time.Sleep(2 * time.Second) diff --git a/integrationTests/factory/processComponents/processComponents_test.go b/integrationTests/factory/processComponents/processComponents_test.go index 42a59b274b7..ac715e3b367 100644 --- a/integrationTests/factory/processComponents/processComponents_test.go +++ b/integrationTests/factory/processComponents/processComponents_test.go @@ -13,7 +13,8 @@ import ( bootstrapComp "github.com/multiversx/mx-chain-go/factory/bootstrap" "github.com/multiversx/mx-chain-go/integrationTests/factory" "github.com/multiversx/mx-chain-go/node" - "github.com/multiversx/mx-chain-go/storage" + "github.com/multiversx/mx-chain-go/process" + "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/testscommon/goroutines" ) @@ -42,8 +43,10 @@ func TestProcessComponents_Close_ShouldWork(t *testing.T) { require.Nil(t, err) managedNetworkComponents, err := nr.CreateManagedNetworkComponents(managedCoreComponents, managedStatusCoreComponents, managedCryptoComponents) require.Nil(t, err) - interceptedDataCacheMap := make(map[string]storage.Cacher) - managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents, interceptedDataCacheMap) + interceptedDataVerifierFactory := &mock.InterceptedDataVerifierFactoryStub{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { + return &mock.InterceptedDataVerifierStub{}, nil + }} + managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents, interceptedDataVerifierFactory) require.Nil(t, err) managedDataComponents, err := nr.CreateManagedDataComponents(managedStatusCoreComponents, managedCoreComponents, managedBootstrapComponents, managedCryptoComponents) require.Nil(t, err) @@ -105,7 +108,7 @@ func TestProcessComponents_Close_ShouldWork(t *testing.T) { managedStatusCoreComponents, gasScheduleNotifier, nodesCoordinator, - interceptedDataCacheMap, + interceptedDataVerifierFactory, ) require.Nil(t, err) require.NotNil(t, managedProcessComponents) diff --git a/integrationTests/factory/stateComponents/stateComponents_test.go b/integrationTests/factory/stateComponents/stateComponents_test.go index ff011c7a803..8028eba038d 100644 --- a/integrationTests/factory/stateComponents/stateComponents_test.go +++ b/integrationTests/factory/stateComponents/stateComponents_test.go @@ -10,7 +10,7 @@ import ( "github.com/multiversx/mx-chain-go/integrationTests/factory" "github.com/multiversx/mx-chain-go/node" - "github.com/multiversx/mx-chain-go/storage" + "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/testscommon/goroutines" ) @@ -38,8 +38,8 @@ func TestStateComponents_Create_Close_ShouldWork(t *testing.T) { require.Nil(t, err) managedNetworkComponents, err := nr.CreateManagedNetworkComponents(managedCoreComponents, managedStatusCoreComponents, managedCryptoComponents) require.Nil(t, err) - interceptedDataCacheMap := make(map[string]storage.Cacher) - managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents, interceptedDataCacheMap) + interceptedDataVerifierFactory := &mock.InterceptedDataVerifierFactoryStub{} + managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents, interceptedDataVerifierFactory) require.Nil(t, err) managedDataComponents, err := nr.CreateManagedDataComponents(managedStatusCoreComponents, managedCoreComponents, managedBootstrapComponents, managedCryptoComponents) require.Nil(t, err) diff --git a/integrationTests/factory/statusComponents/statusComponents_test.go b/integrationTests/factory/statusComponents/statusComponents_test.go index 3c3ef519d1f..2def4237ee8 100644 --- a/integrationTests/factory/statusComponents/statusComponents_test.go +++ b/integrationTests/factory/statusComponents/statusComponents_test.go @@ -13,7 +13,8 @@ import ( bootstrapComp "github.com/multiversx/mx-chain-go/factory/bootstrap" "github.com/multiversx/mx-chain-go/integrationTests/factory" "github.com/multiversx/mx-chain-go/node" - "github.com/multiversx/mx-chain-go/storage" + "github.com/multiversx/mx-chain-go/process" + "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/testscommon/goroutines" ) @@ -42,8 +43,10 @@ func TestStatusComponents_Create_Close_ShouldWork(t *testing.T) { require.Nil(t, err) managedNetworkComponents, err := nr.CreateManagedNetworkComponents(managedCoreComponents, managedStatusCoreComponents, managedCryptoComponents) require.Nil(t, err) - interceptedDataCacheMap := make(map[string]storage.Cacher) - managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents, interceptedDataCacheMap) + interceptedDataVerifierFactory := &mock.InterceptedDataVerifierFactoryStub{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { + return &mock.InterceptedDataVerifierStub{}, nil + }} + managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents, interceptedDataVerifierFactory) require.Nil(t, err) managedDataComponents, err := nr.CreateManagedDataComponents(managedStatusCoreComponents, managedCoreComponents, managedBootstrapComponents, managedCryptoComponents) require.Nil(t, err) @@ -107,7 +110,7 @@ func TestStatusComponents_Create_Close_ShouldWork(t *testing.T) { managedStatusCoreComponents, gasScheduleNotifier, nodesCoordinator, - interceptedDataCacheMap, + interceptedDataVerifierFactory, ) require.Nil(t, err) time.Sleep(2 * time.Second) diff --git a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go index 8f8741efe75..43a38079f45 100644 --- a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go +++ b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go @@ -546,7 +546,9 @@ func testCreateMetaTopicShouldFail(matchStrToErrOnCreate string, matchStrToErrOn } else { args.MainMessenger = createMetaStubTopicHandler(matchStrToErrOnCreate, matchStrToErrOnRegister) } - args.ProcessedMessagesCacheMap = make(map[string]storage.Cacher) + args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryStub{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { + return &mock.InterceptedDataVerifierStub{}, nil + }} icf, _ := interceptorscontainer.NewMetaInterceptorsContainerFactory(args) mainContainer, fullArchiveConatiner, err := icf.Create() @@ -562,7 +564,9 @@ func TestMetaInterceptorsContainerFactory_CreateShouldWork(t *testing.T) { coreComp, cryptoComp := createMockComponentHolders() args := getArgumentsMeta(coreComp, cryptoComp) - args.ProcessedMessagesCacheMap = make(map[string]storage.Cacher) + args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryStub{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { + return &mock.InterceptedDataVerifierStub{}, nil + }} icf, _ := interceptorscontainer.NewMetaInterceptorsContainerFactory(args) mainContainer, fullArchiveContainer, err := icf.Create() @@ -595,7 +599,9 @@ func TestMetaInterceptorsContainerFactory_With4ShardsShouldWork(t *testing.T) { args := getArgumentsMeta(coreComp, cryptoComp) args.ShardCoordinator = shardCoordinator args.NodesCoordinator = nodesCoordinator - args.ProcessedMessagesCacheMap = make(map[string]storage.Cacher) + args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryStub{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { + return &mock.InterceptedDataVerifierStub{}, nil + }} icf, err := interceptorscontainer.NewMetaInterceptorsContainerFactory(args) require.Nil(t, err) @@ -647,7 +653,9 @@ func TestMetaInterceptorsContainerFactory_With4ShardsShouldWork(t *testing.T) { args.NodeOperationMode = common.FullArchiveMode args.ShardCoordinator = shardCoordinator args.NodesCoordinator = nodesCoordinator - args.ProcessedMessagesCacheMap = make(map[string]storage.Cacher) + args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryStub{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { + return &mock.InterceptedDataVerifierStub{}, nil + }} icf, err := interceptorscontainer.NewMetaInterceptorsContainerFactory(args) require.Nil(t, err) diff --git a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go index d9f16ea24a6..44fe9032cac 100644 --- a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go +++ b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go @@ -497,7 +497,9 @@ func testCreateShardTopicShouldFail(matchStrToErrOnCreate string, matchStrToErrO coreComp, cryptoComp := createMockComponentHolders() args := getArgumentsShard(coreComp, cryptoComp) - args.ProcessedMessagesCacheMap = make(map[string]storage.Cacher) + args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryStub{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { + return &mock.InterceptedDataVerifierStub{}, nil + }} if strings.Contains(t.Name(), "full_archive") { args.NodeOperationMode = common.FullArchiveMode args.FullArchiveMessenger = createShardStubTopicHandler(matchStrToErrOnCreate, matchStrToErrOnRegister) @@ -564,7 +566,9 @@ func TestShardInterceptorsContainerFactory_CreateShouldWork(t *testing.T) { }, } args.WhiteListerVerifiedTxs = &testscommon.WhiteListHandlerStub{} - args.ProcessedMessagesCacheMap = make(map[string]storage.Cacher) + args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryStub{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { + return &mock.InterceptedDataVerifierStub{}, nil + }} icf, _ := interceptorscontainer.NewShardInterceptorsContainerFactory(args) @@ -600,7 +604,9 @@ func TestShardInterceptorsContainerFactory_With4ShardsShouldWork(t *testing.T) { args.ShardCoordinator = shardCoordinator args.NodesCoordinator = nodesCoordinator args.PreferredPeersHolder = &p2pmocks.PeersHolderStub{} - args.ProcessedMessagesCacheMap = make(map[string]storage.Cacher) + args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryStub{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { + return &mock.InterceptedDataVerifierStub{}, nil + }} icf, _ := interceptorscontainer.NewShardInterceptorsContainerFactory(args) @@ -651,7 +657,9 @@ func TestShardInterceptorsContainerFactory_With4ShardsShouldWork(t *testing.T) { args.ShardCoordinator = shardCoordinator args.NodesCoordinator = nodesCoordinator args.PreferredPeersHolder = &p2pmocks.PeersHolderStub{} - args.ProcessedMessagesCacheMap = make(map[string]storage.Cacher) + args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryStub{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { + return &mock.InterceptedDataVerifierStub{}, nil + }} icf, _ := interceptorscontainer.NewShardInterceptorsContainerFactory(args) diff --git a/process/mock/interceptedDataVerifierFactoryStub.go b/process/mock/interceptedDataVerifierFactoryStub.go new file mode 100644 index 00000000000..711ddfd828a --- /dev/null +++ b/process/mock/interceptedDataVerifierFactoryStub.go @@ -0,0 +1,19 @@ +package mock + +import ( + "github.com/multiversx/mx-chain-go/process" +) + +// InterceptedDataVerifierFactoryStub - +type InterceptedDataVerifierFactoryStub struct { + CreateCalled func(topic string) (process.InterceptedDataVerifier, error) +} + +// Create - +func (idvfs *InterceptedDataVerifierFactoryStub) Create(topic string) (process.InterceptedDataVerifier, error) { + if idvfs.CreateCalled != nil { + return idvfs.CreateCalled(topic) + } + + return nil, nil +} From f0b34ef7fd61af431e9e8036c119223c9ee704c8 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Thu, 26 Sep 2024 11:42:18 +0300 Subject: [PATCH 27/37] more test fixes. --- epochStart/bootstrap/process.go | 29 ++++++++++--------- epochStart/bootstrap/process_test.go | 8 ++++- epochStart/bootstrap/storageProcess_test.go | 5 +++- .../bootstrap/syncEpochStartMeta_test.go | 10 +++++-- .../processing/blockProcessorCreator_test.go | 5 ++-- testscommon/components/components.go | 12 ++++++-- 6 files changed, 45 insertions(+), 24 deletions(-) diff --git a/epochStart/bootstrap/process.go b/epochStart/bootstrap/process.go index 62ab816ce33..c1c4eb8d4df 100644 --- a/epochStart/bootstrap/process.go +++ b/epochStart/bootstrap/process.go @@ -581,20 +581,21 @@ func (e *epochStartBootstrap) prepareComponentsToSyncFromNetwork() error { func (e *epochStartBootstrap) createSyncers() error { var err error args := factoryInterceptors.ArgsEpochStartInterceptorContainer{ - CoreComponents: e.coreComponentsHolder, - CryptoComponents: e.cryptoComponentsHolder, - Config: e.generalConfig, - ShardCoordinator: e.shardCoordinator, - MainMessenger: e.mainMessenger, - FullArchiveMessenger: e.fullArchiveMessenger, - DataPool: e.dataPool, - WhiteListHandler: e.whiteListHandler, - WhiteListerVerifiedTxs: e.whiteListerVerifiedTxs, - ArgumentsParser: e.argumentsParser, - HeaderIntegrityVerifier: e.headerIntegrityVerifier, - RequestHandler: e.requestHandler, - SignaturesHandler: e.mainMessenger, - NodeOperationMode: e.nodeOperationMode, + CoreComponents: e.coreComponentsHolder, + CryptoComponents: e.cryptoComponentsHolder, + Config: e.generalConfig, + ShardCoordinator: e.shardCoordinator, + MainMessenger: e.mainMessenger, + FullArchiveMessenger: e.fullArchiveMessenger, + DataPool: e.dataPool, + WhiteListHandler: e.whiteListHandler, + WhiteListerVerifiedTxs: e.whiteListerVerifiedTxs, + ArgumentsParser: e.argumentsParser, + HeaderIntegrityVerifier: e.headerIntegrityVerifier, + RequestHandler: e.requestHandler, + SignaturesHandler: e.mainMessenger, + NodeOperationMode: e.nodeOperationMode, + InterceptedDataVerifierFactory: e.interceptedDataVerifierFactory, } e.mainInterceptorContainer, e.fullArchiveInterceptorContainer, err = factoryInterceptors.NewEpochStartInterceptorsContainer(args) diff --git a/epochStart/bootstrap/process_test.go b/epochStart/bootstrap/process_test.go index 8807ee77814..3f142fe8459 100644 --- a/epochStart/bootstrap/process_test.go +++ b/epochStart/bootstrap/process_test.go @@ -32,6 +32,7 @@ import ( "github.com/multiversx/mx-chain-go/epochStart/bootstrap/types" "github.com/multiversx/mx-chain-go/epochStart/mock" "github.com/multiversx/mx-chain-go/process" + processMock "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/sharding" "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator" "github.com/multiversx/mx-chain-go/state" @@ -253,7 +254,9 @@ func createMockEpochStartBootstrapArgs( }, TrieSyncStatisticsProvider: &testscommon.SizeSyncStatisticsHandlerStub{}, StateStatsHandler: disabledStatistics.NewStateStatistics(), - InterceptedDataCache: make(map[string]storage.Cacher), + InterceptedDataVerifierFactory: &processMock.InterceptedDataVerifierFactoryStub{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { + return &processMock.InterceptedDataVerifierStub{}, nil + }}, } } @@ -995,6 +998,9 @@ func TestCreateSyncers(t *testing.T) { epochStartProvider.whiteListerVerifiedTxs = &testscommon.WhiteListHandlerStub{} epochStartProvider.requestHandler = &testscommon.RequestHandlerStub{} epochStartProvider.storageService = &storageMocks.ChainStorerStub{} + epochStartProvider.interceptedDataVerifierFactory = &processMock.InterceptedDataVerifierFactoryStub{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { + return &processMock.InterceptedDataVerifierStub{}, nil + }} err := epochStartProvider.createSyncers() assert.Nil(t, err) diff --git a/epochStart/bootstrap/storageProcess_test.go b/epochStart/bootstrap/storageProcess_test.go index 755c6155421..81f06ee801a 100644 --- a/epochStart/bootstrap/storageProcess_test.go +++ b/epochStart/bootstrap/storageProcess_test.go @@ -18,6 +18,7 @@ import ( "github.com/multiversx/mx-chain-go/epochStart" "github.com/multiversx/mx-chain-go/epochStart/mock" "github.com/multiversx/mx-chain-go/process" + processMock "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator" "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/testscommon" @@ -128,7 +129,9 @@ func TestStorageEpochStartBootstrap_BootstrapMetablockNotFound(t *testing.T) { } args.GeneralConfig = testscommon.GetGeneralConfig() args.GeneralConfig.EpochStartConfig.RoundsPerEpoch = roundsPerEpoch - args.InterceptedDataCache = make(map[string]storage.Cacher) + args.InterceptedDataVerifierFactory = &processMock.InterceptedDataVerifierFactoryStub{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { + return &processMock.InterceptedDataVerifierStub{}, nil + }} sesb, _ := NewStorageEpochStartBootstrap(args) params, err := sesb.Bootstrap() diff --git a/epochStart/bootstrap/syncEpochStartMeta_test.go b/epochStart/bootstrap/syncEpochStartMeta_test.go index a99722d9422..49a7263d0cc 100644 --- a/epochStart/bootstrap/syncEpochStartMeta_test.go +++ b/epochStart/bootstrap/syncEpochStartMeta_test.go @@ -16,7 +16,8 @@ import ( "github.com/multiversx/mx-chain-go/epochStart" "github.com/multiversx/mx-chain-go/epochStart/mock" "github.com/multiversx/mx-chain-go/p2p" - "github.com/multiversx/mx-chain-go/storage" + "github.com/multiversx/mx-chain-go/process" + processMock "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/testscommon" "github.com/multiversx/mx-chain-go/testscommon/cryptoMocks" "github.com/multiversx/mx-chain-go/testscommon/economicsmocks" @@ -73,7 +74,8 @@ func TestEpochStartMetaSyncer_SyncEpochStartMetaRegisterMessengerProcessorFailsS }, } args.Messenger = messenger - ess, _ := NewEpochStartMetaSyncer(args) + ess, err := NewEpochStartMetaSyncer(args) + require.NoError(t, err) mb, err := ess.SyncEpochStartMeta(time.Second) require.Equal(t, expectedErr, err) @@ -163,6 +165,8 @@ func getEpochStartSyncerArgs() ArgsNewEpochStartMetaSyncer { }, HeaderIntegrityVerifier: &mock.HeaderIntegrityVerifierStub{}, MetaBlockProcessor: &mock.EpochStartMetaBlockProcessorStub{}, - InterceptedDataCache: make(map[string]storage.Cacher), + InterceptedDataVerifierFactory: &processMock.InterceptedDataVerifierFactoryStub{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { + return &processMock.InterceptedDataVerifierStub{}, nil + }}, } } diff --git a/factory/processing/blockProcessorCreator_test.go b/factory/processing/blockProcessorCreator_test.go index 099fec4a82d..8b01c44c8f8 100644 --- a/factory/processing/blockProcessorCreator_test.go +++ b/factory/processing/blockProcessorCreator_test.go @@ -8,6 +8,9 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" + vmcommon "github.com/multiversx/mx-chain-vm-common-go" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/dataRetriever" dataComp "github.com/multiversx/mx-chain-go/factory/data" @@ -26,8 +29,6 @@ import ( storageManager "github.com/multiversx/mx-chain-go/testscommon/storage" trieMock "github.com/multiversx/mx-chain-go/testscommon/trie" "github.com/multiversx/mx-chain-go/trie" - vmcommon "github.com/multiversx/mx-chain-vm-common-go" - "github.com/stretchr/testify/require" ) func Test_newBlockProcessorCreatorForShard(t *testing.T) { diff --git a/testscommon/components/components.go b/testscommon/components/components.go index 6d33ad04fa0..146c9f1f7dc 100644 --- a/testscommon/components/components.go +++ b/testscommon/components/components.go @@ -8,6 +8,10 @@ import ( "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/endProcess" "github.com/multiversx/mx-chain-core-go/data/outport" + logger "github.com/multiversx/mx-chain-logger-go" + wasmConfig "github.com/multiversx/mx-chain-vm-go/config" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-go/common" commonFactory "github.com/multiversx/mx-chain-go/common/factory" "github.com/multiversx/mx-chain-go/config" @@ -31,6 +35,8 @@ import ( "github.com/multiversx/mx-chain-go/p2p" p2pConfig "github.com/multiversx/mx-chain-go/p2p/config" p2pFactory "github.com/multiversx/mx-chain-go/p2p/factory" + "github.com/multiversx/mx-chain-go/process" + processMock "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/sharding" "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator" "github.com/multiversx/mx-chain-go/state" @@ -41,9 +47,6 @@ import ( statusHandlerMock "github.com/multiversx/mx-chain-go/testscommon/statusHandler" "github.com/multiversx/mx-chain-go/testscommon/storage" "github.com/multiversx/mx-chain-go/trie" - logger "github.com/multiversx/mx-chain-logger-go" - wasmConfig "github.com/multiversx/mx-chain-vm-go/config" - "github.com/stretchr/testify/require" ) var log = logger.GetOrCreate("componentsMock") @@ -607,6 +610,9 @@ func GetProcessArgs( }, }, }, + InterceptedDataVerifierFactory: &processMock.InterceptedDataVerifierFactoryStub{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { + return &processMock.InterceptedDataVerifierStub{}, nil + }}, } } From b05c65083a9786611c813bade886c5f6e3780fae Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Thu, 26 Sep 2024 12:25:11 +0300 Subject: [PATCH 28/37] increase wait time for CI run. --- process/interceptors/interceptedDataVerifier_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/process/interceptors/interceptedDataVerifier_test.go b/process/interceptors/interceptedDataVerifier_test.go index 93a15204e51..af0401f4b9f 100644 --- a/process/interceptors/interceptedDataVerifier_test.go +++ b/process/interceptors/interceptedDataVerifier_test.go @@ -98,7 +98,7 @@ func TestInterceptedDataVerifier_CheckValidityShouldNotWork(t *testing.T) { // It is still invalid because it has the same hash. require.Equal(t, ErrInvalidInterceptedData, err) - <-time.After(defaultSpan + 1*time.Millisecond) + <-time.After(defaultSpan + 100*time.Millisecond) err = verifier.Verify(interceptedData) require.Nil(t, err) From 5dc6bba52cd72c2411e01f1ed47355ccb9da9148 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Thu, 26 Sep 2024 12:55:55 +0300 Subject: [PATCH 29/37] bring back rw mutex on interceptedDataVerifier. --- process/interceptors/interceptedDataVerifier.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/process/interceptors/interceptedDataVerifier.go b/process/interceptors/interceptedDataVerifier.go index 00ea4dbadb6..878c8d5e931 100644 --- a/process/interceptors/interceptedDataVerifier.go +++ b/process/interceptors/interceptedDataVerifier.go @@ -3,6 +3,8 @@ package interceptors import ( "errors" + "github.com/multiversx/mx-chain-core-go/core/sync" + "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/storage" ) @@ -19,16 +21,16 @@ var ( ) type interceptedDataVerifier struct { - //km sync.KeyRWMutexHandler + km sync.KeyRWMutexHandler cache storage.Cacher } // NewInterceptedDataVerifier creates a new instance of intercepted data verifier func NewInterceptedDataVerifier(cache storage.Cacher) *interceptedDataVerifier { - //keyRWMutex := sync.NewKeyRWMutex() + keyRWMutex := sync.NewKeyRWMutex() return &interceptedDataVerifier{ - //km: keyRWMutex, + km: keyRWMutex, cache: cache, } } @@ -65,10 +67,10 @@ func (idv *interceptedDataVerifier) IsInterfaceNil() bool { } func (idv *interceptedDataVerifier) checkValidity(interceptedData process.InterceptedData) error { - //hash := string(interceptedData.Hash()) + hash := string(interceptedData.Hash()) - //idv.km.Lock(hash) - //defer idv.km.Unlock(hash) + idv.km.Lock(hash) + defer idv.km.Unlock(hash) return interceptedData.CheckValidity() } From d2140273b7f7566f626fa6f99c14035bbd5103e5 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Mon, 30 Sep 2024 17:23:28 +0300 Subject: [PATCH 30/37] fixes after review. --- .../bootstrapComponents_test.go | 2 -- .../baseInterceptorsContainerFactory.go | 14 --------- .../factory/interceptedDataVerifierFactory.go | 5 ++++ .../interceptors/interceptedDataVerifier.go | 30 ++++++++++--------- .../interceptedDataVerifier_test.go | 12 +++++++- process/interface.go | 1 + .../interceptedDataVerifierFactoryStub.go | 5 ++++ 7 files changed, 38 insertions(+), 31 deletions(-) diff --git a/integrationTests/factory/bootstrapComponents/bootstrapComponents_test.go b/integrationTests/factory/bootstrapComponents/bootstrapComponents_test.go index 25a19b3e15d..ce8b2455234 100644 --- a/integrationTests/factory/bootstrapComponents/bootstrapComponents_test.go +++ b/integrationTests/factory/bootstrapComponents/bootstrapComponents_test.go @@ -10,8 +10,6 @@ import ( "github.com/multiversx/mx-chain-go/integrationTests/factory" "github.com/multiversx/mx-chain-go/node" - //"github.com/multiversx/mx-chain-go/process" - //factory2 "github.com/multiversx/mx-chain-go/process/interceptors/factory" "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/testscommon/goroutines" ) diff --git a/process/factory/interceptorscontainer/baseInterceptorsContainerFactory.go b/process/factory/interceptorscontainer/baseInterceptorsContainerFactory.go index 518ca765633..b05e496475f 100644 --- a/process/factory/interceptorscontainer/baseInterceptorsContainerFactory.go +++ b/process/factory/interceptorscontainer/baseInterceptorsContainerFactory.go @@ -958,17 +958,3 @@ func (bicf *baseInterceptorsContainerFactory) addInterceptorsToContainers(keys [ return bicf.fullArchiveContainer.AddMultiple(keys, interceptors) } - -//func (bicf *baseInterceptorsContainerFactory) createCacheForInterceptor(topic string) (process.InterceptedDataVerifier, error) { -// internalCache, err := cache.NewTimeCacher(cache.ArgTimeCacher{ -// DefaultSpan: cacheDefaultSpan, -// CacheExpiry: cacheDefaultExpiry, -// }) -// if err != nil { -// return nil, err -// } -// -// bicf.processedMessagesCacheMap[topic] = internalCache -// verifier := interceptors.NewInterceptedDataVerifier(internalCache) -// return verifier, nil -//} diff --git a/process/interceptors/factory/interceptedDataVerifierFactory.go b/process/interceptors/factory/interceptedDataVerifierFactory.go index 96d5a8bb2c3..544b82e257b 100644 --- a/process/interceptors/factory/interceptedDataVerifierFactory.go +++ b/process/interceptors/factory/interceptedDataVerifierFactory.go @@ -46,3 +46,8 @@ func (idvf *InterceptedDataVerifierFactory) Create(topic string) (process.Interc verifier := interceptors.NewInterceptedDataVerifier(internalCache) return verifier, nil } + +// IsInterfaceNil returns true if there is no value under the interface +func (idvf *InterceptedDataVerifierFactory) IsInterfaceNil() bool { + return idvf == nil +} diff --git a/process/interceptors/interceptedDataVerifier.go b/process/interceptors/interceptedDataVerifier.go index 878c8d5e931..753f7097b80 100644 --- a/process/interceptors/interceptedDataVerifier.go +++ b/process/interceptors/interceptedDataVerifier.go @@ -9,11 +9,13 @@ import ( "github.com/multiversx/mx-chain-go/storage" ) -type interceptedDataStatus int +type interceptedDataStatus int8 const ( ValidInterceptedData interceptedDataStatus = iota InvalidInterceptedData + + interceptedDataStatusBytesSize = 8 ) var ( @@ -39,11 +41,17 @@ func NewInterceptedDataVerifier(cache storage.Cacher) *interceptedDataVerifier { // It will retrieve the status in the cache if it exists, otherwise it will validate it and store the status of the // validation in the cache. Note that the entries are stored for a set period of time func (idv *interceptedDataVerifier) Verify(interceptedData process.InterceptedData) error { + hash := string(interceptedData.Hash()) + if len(interceptedData.Hash()) == 0 { return interceptedData.CheckValidity() } - if val, ok := idv.cache.Get(interceptedData.Hash()); ok { + idv.km.RLock(hash) + val, ok := idv.cache.Get(interceptedData.Hash()) + idv.km.RUnlock(hash) + + if ok { if val == ValidInterceptedData { return nil } @@ -51,13 +59,16 @@ func (idv *interceptedDataVerifier) Verify(interceptedData process.InterceptedDa return ErrInvalidInterceptedData } - err := idv.checkValidity(interceptedData) + err := interceptedData.CheckValidity() if err != nil { - idv.cache.Put(interceptedData.Hash(), InvalidInterceptedData, 8) + idv.km.Lock(hash) + idv.cache.Put(interceptedData.Hash(), InvalidInterceptedData, interceptedDataStatusBytesSize) + idv.km.Unlock(hash) + return ErrInvalidInterceptedData } - idv.cache.Put(interceptedData.Hash(), ValidInterceptedData, 100) + idv.cache.Put(interceptedData.Hash(), ValidInterceptedData, interceptedDataStatusBytesSize) return nil } @@ -65,12 +76,3 @@ func (idv *interceptedDataVerifier) Verify(interceptedData process.InterceptedDa func (idv *interceptedDataVerifier) IsInterfaceNil() bool { return idv == nil } - -func (idv *interceptedDataVerifier) checkValidity(interceptedData process.InterceptedData) error { - hash := string(interceptedData.Hash()) - - idv.km.Lock(hash) - defer idv.km.Unlock(hash) - - return interceptedData.CheckValidity() -} diff --git a/process/interceptors/interceptedDataVerifier_test.go b/process/interceptors/interceptedDataVerifier_test.go index af0401f4b9f..630c7604cb5 100644 --- a/process/interceptors/interceptedDataVerifier_test.go +++ b/process/interceptors/interceptedDataVerifier_test.go @@ -28,8 +28,11 @@ func defaultInterceptedDataVerifier(span time.Duration) process.InterceptedDataV func TestInterceptedDataVerifier_CheckValidityShouldWork(t *testing.T) { t.Parallel() + checkValidityCounter := atomic.Counter{} + interceptedData := &testscommon.InterceptedDataStub{ CheckValidityCalled: func() error { + checkValidityCounter.Add(1) return nil }, IsForCurrentShardCalled: func() bool { @@ -60,13 +63,17 @@ func TestInterceptedDataVerifier_CheckValidityShouldWork(t *testing.T) { wg.Wait() require.Equal(t, int64(0), errCount.Get()) + require.Equal(t, int64(1), checkValidityCounter.Get()) } func TestInterceptedDataVerifier_CheckValidityShouldNotWork(t *testing.T) { t.Parallel() + checkValidityCounter := atomic.Counter{} + interceptedData := &testscommon.InterceptedDataStub{ CheckValidityCalled: func() error { + checkValidityCounter.Add(1) return nil }, IsForCurrentShardCalled: func() bool { @@ -93,13 +100,16 @@ func TestInterceptedDataVerifier_CheckValidityShouldNotWork(t *testing.T) { err := verifier.Verify(interceptedDataWithErr) require.Equal(t, ErrInvalidInterceptedData, err) + require.Equal(t, int64(0), checkValidityCounter.Get()) err = verifier.Verify(interceptedData) // It is still invalid because it has the same hash. require.Equal(t, ErrInvalidInterceptedData, err) + require.Equal(t, int64(0), checkValidityCounter.Get()) <-time.After(defaultSpan + 100*time.Millisecond) err = verifier.Verify(interceptedData) - require.Nil(t, err) + require.NoError(t, err) + require.Equal(t, int64(1), checkValidityCounter.Get()) } diff --git a/process/interface.go b/process/interface.go index e12443fcb48..53c9ac48645 100644 --- a/process/interface.go +++ b/process/interface.go @@ -1409,4 +1409,5 @@ type InterceptedDataVerifier interface { type InterceptedDataVerifierFactory interface { Create(topic string) (InterceptedDataVerifier, error) + IsInterfaceNil() bool } diff --git a/process/mock/interceptedDataVerifierFactoryStub.go b/process/mock/interceptedDataVerifierFactoryStub.go index 711ddfd828a..6fdd9874903 100644 --- a/process/mock/interceptedDataVerifierFactoryStub.go +++ b/process/mock/interceptedDataVerifierFactoryStub.go @@ -17,3 +17,8 @@ func (idvfs *InterceptedDataVerifierFactoryStub) Create(topic string) (process.I return nil, nil } + +// IsInterfaceNil - +func (idvfs *InterceptedDataVerifierFactoryStub) IsInterfaceNil() bool { + return idvfs == nil +} From 899b04dd4c53cfbbf59e1d54c889a92080af4f05 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Tue, 1 Oct 2024 11:55:33 +0300 Subject: [PATCH 31/37] split test and improved lock mechanism. --- .../interceptors/interceptedDataVerifier.go | 10 +- .../interceptedDataVerifier_test.go | 124 ++++++++++++++---- 2 files changed, 102 insertions(+), 32 deletions(-) diff --git a/process/interceptors/interceptedDataVerifier.go b/process/interceptors/interceptedDataVerifier.go index 753f7097b80..9eef5dbeed8 100644 --- a/process/interceptors/interceptedDataVerifier.go +++ b/process/interceptors/interceptedDataVerifier.go @@ -47,11 +47,10 @@ func (idv *interceptedDataVerifier) Verify(interceptedData process.InterceptedDa return interceptedData.CheckValidity() } - idv.km.RLock(hash) - val, ok := idv.cache.Get(interceptedData.Hash()) - idv.km.RUnlock(hash) + idv.km.Lock(hash) + defer idv.km.Unlock(hash) - if ok { + if val, ok := idv.cache.Get(interceptedData.Hash()); ok { if val == ValidInterceptedData { return nil } @@ -61,10 +60,7 @@ func (idv *interceptedDataVerifier) Verify(interceptedData process.InterceptedDa err := interceptedData.CheckValidity() if err != nil { - idv.km.Lock(hash) idv.cache.Put(interceptedData.Hash(), InvalidInterceptedData, interceptedDataStatusBytesSize) - idv.km.Unlock(hash) - return ErrInvalidInterceptedData } diff --git a/process/interceptors/interceptedDataVerifier_test.go b/process/interceptors/interceptedDataVerifier_test.go index 630c7604cb5..90f4555315a 100644 --- a/process/interceptors/interceptedDataVerifier_test.go +++ b/process/interceptors/interceptedDataVerifier_test.go @@ -1,7 +1,6 @@ package interceptors import ( - "errors" "sync" "testing" "time" @@ -46,7 +45,7 @@ func TestInterceptedDataVerifier_CheckValidityShouldWork(t *testing.T) { verifier := defaultInterceptedDataVerifier(defaultSpan) err := verifier.Verify(interceptedData) - require.Nil(t, err) + require.NoError(t, err) errCount := atomic.Counter{} wg := sync.WaitGroup{} @@ -74,19 +73,7 @@ func TestInterceptedDataVerifier_CheckValidityShouldNotWork(t *testing.T) { interceptedData := &testscommon.InterceptedDataStub{ CheckValidityCalled: func() error { checkValidityCounter.Add(1) - return nil - }, - IsForCurrentShardCalled: func() bool { - return false - }, - HashCalled: func() []byte { - return []byte("hash") - }, - } - - interceptedDataWithErr := &testscommon.InterceptedDataStub{ - CheckValidityCalled: func() error { - return errors.New("error") + return ErrInvalidInterceptedData }, IsForCurrentShardCalled: func() bool { return false @@ -98,18 +85,105 @@ func TestInterceptedDataVerifier_CheckValidityShouldNotWork(t *testing.T) { verifier := defaultInterceptedDataVerifier(defaultSpan) - err := verifier.Verify(interceptedDataWithErr) - require.Equal(t, ErrInvalidInterceptedData, err) - require.Equal(t, int64(0), checkValidityCounter.Get()) - - err = verifier.Verify(interceptedData) - // It is still invalid because it has the same hash. + err := verifier.Verify(interceptedData) require.Equal(t, ErrInvalidInterceptedData, err) - require.Equal(t, int64(0), checkValidityCounter.Get()) - <-time.After(defaultSpan + 100*time.Millisecond) + errCount := atomic.Counter{} + wg := sync.WaitGroup{} + for i := 0; i < 3; i++ { + wg.Add(1) + go func() { + defer wg.Done() + err := verifier.Verify(interceptedData) + if err != nil { + errCount.Add(1) + } + }() + } + wg.Wait() - err = verifier.Verify(interceptedData) - require.NoError(t, err) + require.Equal(t, int64(3), errCount.Get()) require.Equal(t, int64(1), checkValidityCounter.Get()) } + +func TestInterceptedDataVerifier_CheckExpiryTime(t *testing.T) { + t.Parallel() + + t.Run("expiry on valid data", func(t *testing.T) { + expiryTestDuration := defaultSpan * 2 + + checkValidityCounter := atomic.Counter{} + + interceptedData := &testscommon.InterceptedDataStub{ + CheckValidityCalled: func() error { + checkValidityCounter.Add(1) + return nil + }, + IsForCurrentShardCalled: func() bool { + return false + }, + HashCalled: func() []byte { + return []byte("hash") + }, + } + + verifier := defaultInterceptedDataVerifier(expiryTestDuration) + + // First retrieval, check validity is reached. + err := verifier.Verify(interceptedData) + require.NoError(t, err) + require.Equal(t, int64(1), checkValidityCounter.Get()) + + // Second retrieval should be from the cache. + err = verifier.Verify(interceptedData) + require.NoError(t, err) + require.Equal(t, int64(1), checkValidityCounter.Get()) + + // Wait for the cache expiry + <-time.After(expiryTestDuration + 100*time.Millisecond) + + // Third retrieval should reach validity check again. + err = verifier.Verify(interceptedData) + require.NoError(t, err) + require.Equal(t, int64(2), checkValidityCounter.Get()) + }) + + t.Run("expiry on invalid data", func(t *testing.T) { + expiryTestDuration := defaultSpan * 2 + + checkValidityCounter := atomic.Counter{} + + interceptedData := &testscommon.InterceptedDataStub{ + CheckValidityCalled: func() error { + checkValidityCounter.Add(1) + return ErrInvalidInterceptedData + }, + IsForCurrentShardCalled: func() bool { + return false + }, + HashCalled: func() []byte { + return []byte("hash") + }, + } + + verifier := defaultInterceptedDataVerifier(expiryTestDuration) + + // First retrieval, check validity is reached. + err := verifier.Verify(interceptedData) + require.Equal(t, ErrInvalidInterceptedData, err) + require.Equal(t, int64(1), checkValidityCounter.Get()) + + // Second retrieval should be from the cache. + err = verifier.Verify(interceptedData) + require.Equal(t, ErrInvalidInterceptedData, err) + require.Equal(t, int64(1), checkValidityCounter.Get()) + + // Wait for the cache expiry + <-time.After(expiryTestDuration + 100*time.Millisecond) + + // Third retrieval should reach validity check again. + err = verifier.Verify(interceptedData) + require.Equal(t, ErrInvalidInterceptedData, err) + require.Equal(t, int64(2), checkValidityCounter.Get()) + }) +} From 959e0375de32618ae528e51e3a2611abb036b556 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean <36670455+cristure@users.noreply.github.com> Date: Wed, 2 Oct 2024 17:21:44 +0300 Subject: [PATCH 32/37] Update node/nodeRunner.go Co-authored-by: Sorin Stanculeanu <34831323+sstanculeanu@users.noreply.github.com> --- node/nodeRunner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/nodeRunner.go b/node/nodeRunner.go index 478e0ab4899..f9b9ae500af 100644 --- a/node/nodeRunner.go +++ b/node/nodeRunner.go @@ -324,7 +324,7 @@ func (nr *nodeRunner) executeOneComponentCreationCycle( log.Debug("creating bootstrap components") interceptedDataVerifierFactory := factory.NewInterceptedDataVerifierFactory(factory.InterceptedDataVerifierFactoryArgs{ - CacheSpan: time.Duration(nr.configs.GeneralConfig.InterceptedDataVerifier.CacheExpiryInSec), + CacheSpan: time.Duration(nr.configs.GeneralConfig.InterceptedDataVerifier.CacheSpanInSec), CacheExpiry: time.Duration(nr.configs.GeneralConfig.InterceptedDataVerifier.CacheExpiryInSec), }) managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents, interceptedDataVerifierFactory) From 7762f62f13cc83a34a7c7b8a6c6e508f54c96b1c Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Wed, 2 Oct 2024 17:26:19 +0300 Subject: [PATCH 33/37] addressed some comments. --- epochStart/bootstrap/syncEpochStartMeta.go | 3 ++ epochStart/errors.go | 3 ++ .../bootstrapComponents_test.go | 4 +-- .../consensusComponents_test.go | 8 +---- .../dataComponents/dataComponents_test.go | 5 ++- .../heartbeatComponents_test.go | 8 +---- .../processComponents_test.go | 8 +---- .../stateComponents/stateComponents_test.go | 4 +-- .../statusComponents/statusComponents_test.go | 8 +---- node/nodeRunner.go | 33 +++++++++---------- process/errors.go | 3 ++ .../metaInterceptorsContainerFactory.go | 3 ++ .../shardInterceptorsContainerFactory.go | 3 ++ 13 files changed, 38 insertions(+), 55 deletions(-) diff --git a/epochStart/bootstrap/syncEpochStartMeta.go b/epochStart/bootstrap/syncEpochStartMeta.go index 8b059909fb1..b550a25911a 100644 --- a/epochStart/bootstrap/syncEpochStartMeta.go +++ b/epochStart/bootstrap/syncEpochStartMeta.go @@ -65,6 +65,9 @@ func NewEpochStartMetaSyncer(args ArgsNewEpochStartMetaSyncer) (*epochStartMetaS if check.IfNil(args.MetaBlockProcessor) { return nil, epochStart.ErrNilMetablockProcessor } + if check.IfNil(args.InterceptedDataVerifierFactory) { + return nil, epochStart.ErrNilInterceptedDataVerifierFactory + } e := &epochStartMetaSyncer{ requestHandler: args.RequestHandler, diff --git a/epochStart/errors.go b/epochStart/errors.go index ca115e939f4..e022064c472 100644 --- a/epochStart/errors.go +++ b/epochStart/errors.go @@ -239,6 +239,9 @@ var ErrNilEpochNotifier = errors.New("nil EpochNotifier") // ErrNilMetablockProcessor signals that a nil metablock processor was provided var ErrNilMetablockProcessor = errors.New("nil metablock processor") +// ErrNilInterceptedDataVerifierFactory signals that a nil intercepted data verifier factory was provided +var ErrNilInterceptedDataVerifierFactory = errors.New("nil intercepted data verifier factory") + // ErrCouldNotInitDelegationSystemSC signals that delegation system sc init failed var ErrCouldNotInitDelegationSystemSC = errors.New("could not init delegation system sc") diff --git a/integrationTests/factory/bootstrapComponents/bootstrapComponents_test.go b/integrationTests/factory/bootstrapComponents/bootstrapComponents_test.go index ce8b2455234..2e9cb01e72a 100644 --- a/integrationTests/factory/bootstrapComponents/bootstrapComponents_test.go +++ b/integrationTests/factory/bootstrapComponents/bootstrapComponents_test.go @@ -10,7 +10,6 @@ import ( "github.com/multiversx/mx-chain-go/integrationTests/factory" "github.com/multiversx/mx-chain-go/node" - "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/testscommon/goroutines" ) @@ -38,8 +37,7 @@ func TestBootstrapComponents_Create_Close_ShouldWork(t *testing.T) { require.Nil(t, err) managedNetworkComponents, err := nr.CreateManagedNetworkComponents(managedCoreComponents, managedStatusCoreComponents, managedCryptoComponents) require.Nil(t, err) - interceptedDataVerifierFactory := &mock.InterceptedDataVerifierFactoryStub{} - managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents, interceptedDataVerifierFactory) + managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents) require.Nil(t, err) require.NotNil(t, managedBootstrapComponents) diff --git a/integrationTests/factory/consensusComponents/consensusComponents_test.go b/integrationTests/factory/consensusComponents/consensusComponents_test.go index 7fe28403b81..d4b120a9636 100644 --- a/integrationTests/factory/consensusComponents/consensusComponents_test.go +++ b/integrationTests/factory/consensusComponents/consensusComponents_test.go @@ -13,8 +13,6 @@ import ( bootstrapComp "github.com/multiversx/mx-chain-go/factory/bootstrap" "github.com/multiversx/mx-chain-go/integrationTests/factory" "github.com/multiversx/mx-chain-go/node" - "github.com/multiversx/mx-chain-go/process" - "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/testscommon/goroutines" ) @@ -42,10 +40,7 @@ func TestConsensusComponents_Close_ShouldWork(t *testing.T) { require.Nil(t, err) managedNetworkComponents, err := nr.CreateManagedNetworkComponents(managedCoreComponents, managedStatusCoreComponents, managedCryptoComponents) require.Nil(t, err) - interceptedDataVerifierFactory := &mock.InterceptedDataVerifierFactoryStub{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { - return &mock.InterceptedDataVerifierStub{}, nil - }} - managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents, interceptedDataVerifierFactory) + managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents) require.Nil(t, err) managedDataComponents, err := nr.CreateManagedDataComponents(managedStatusCoreComponents, managedCoreComponents, managedBootstrapComponents, managedCryptoComponents) require.Nil(t, err) @@ -109,7 +104,6 @@ func TestConsensusComponents_Close_ShouldWork(t *testing.T) { managedStatusCoreComponents, gasScheduleNotifier, nodesCoordinator, - interceptedDataVerifierFactory, ) require.Nil(t, err) time.Sleep(2 * time.Second) diff --git a/integrationTests/factory/dataComponents/dataComponents_test.go b/integrationTests/factory/dataComponents/dataComponents_test.go index 1641bce6b9c..d26cf7aa01f 100644 --- a/integrationTests/factory/dataComponents/dataComponents_test.go +++ b/integrationTests/factory/dataComponents/dataComponents_test.go @@ -10,7 +10,6 @@ import ( "github.com/multiversx/mx-chain-go/integrationTests/factory" "github.com/multiversx/mx-chain-go/node" - "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/testscommon/goroutines" ) @@ -38,8 +37,8 @@ func TestDataComponents_Create_Close_ShouldWork(t *testing.T) { require.Nil(t, err) managedNetworkComponents, err := nr.CreateManagedNetworkComponents(managedCoreComponents, managedStatusCoreComponents, managedCryptoComponents) require.Nil(t, err) - interceptedDataVerifierFactory := &mock.InterceptedDataVerifierFactoryStub{} - managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents, interceptedDataVerifierFactory) + + managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents) require.Nil(t, err) managedDataComponents, err := nr.CreateManagedDataComponents(managedStatusCoreComponents, managedCoreComponents, managedBootstrapComponents, managedCryptoComponents) require.Nil(t, err) diff --git a/integrationTests/factory/heartbeatComponents/heartbeatComponents_test.go b/integrationTests/factory/heartbeatComponents/heartbeatComponents_test.go index 384b39e7d37..889c4ff38f8 100644 --- a/integrationTests/factory/heartbeatComponents/heartbeatComponents_test.go +++ b/integrationTests/factory/heartbeatComponents/heartbeatComponents_test.go @@ -13,8 +13,6 @@ import ( bootstrapComp "github.com/multiversx/mx-chain-go/factory/bootstrap" "github.com/multiversx/mx-chain-go/integrationTests/factory" "github.com/multiversx/mx-chain-go/node" - "github.com/multiversx/mx-chain-go/process" - "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/testscommon/goroutines" ) @@ -42,10 +40,7 @@ func TestHeartbeatComponents_Close_ShouldWork(t *testing.T) { require.Nil(t, err) managedNetworkComponents, err := nr.CreateManagedNetworkComponents(managedCoreComponents, managedStatusCoreComponents, managedCryptoComponents) require.Nil(t, err) - interceptedDataVerifierFactory := &mock.InterceptedDataVerifierFactoryStub{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { - return &mock.InterceptedDataVerifierStub{}, nil - }} - managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents, interceptedDataVerifierFactory) + managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents) require.Nil(t, err) managedDataComponents, err := nr.CreateManagedDataComponents(managedStatusCoreComponents, managedCoreComponents, managedBootstrapComponents, managedCryptoComponents) require.Nil(t, err) @@ -109,7 +104,6 @@ func TestHeartbeatComponents_Close_ShouldWork(t *testing.T) { managedStatusCoreComponents, gasScheduleNotifier, nodesCoordinator, - interceptedDataVerifierFactory, ) require.Nil(t, err) time.Sleep(2 * time.Second) diff --git a/integrationTests/factory/processComponents/processComponents_test.go b/integrationTests/factory/processComponents/processComponents_test.go index ac715e3b367..110a8869878 100644 --- a/integrationTests/factory/processComponents/processComponents_test.go +++ b/integrationTests/factory/processComponents/processComponents_test.go @@ -13,8 +13,6 @@ import ( bootstrapComp "github.com/multiversx/mx-chain-go/factory/bootstrap" "github.com/multiversx/mx-chain-go/integrationTests/factory" "github.com/multiversx/mx-chain-go/node" - "github.com/multiversx/mx-chain-go/process" - "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/testscommon/goroutines" ) @@ -43,10 +41,7 @@ func TestProcessComponents_Close_ShouldWork(t *testing.T) { require.Nil(t, err) managedNetworkComponents, err := nr.CreateManagedNetworkComponents(managedCoreComponents, managedStatusCoreComponents, managedCryptoComponents) require.Nil(t, err) - interceptedDataVerifierFactory := &mock.InterceptedDataVerifierFactoryStub{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { - return &mock.InterceptedDataVerifierStub{}, nil - }} - managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents, interceptedDataVerifierFactory) + managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents) require.Nil(t, err) managedDataComponents, err := nr.CreateManagedDataComponents(managedStatusCoreComponents, managedCoreComponents, managedBootstrapComponents, managedCryptoComponents) require.Nil(t, err) @@ -108,7 +103,6 @@ func TestProcessComponents_Close_ShouldWork(t *testing.T) { managedStatusCoreComponents, gasScheduleNotifier, nodesCoordinator, - interceptedDataVerifierFactory, ) require.Nil(t, err) require.NotNil(t, managedProcessComponents) diff --git a/integrationTests/factory/stateComponents/stateComponents_test.go b/integrationTests/factory/stateComponents/stateComponents_test.go index 8028eba038d..ba93bdf8263 100644 --- a/integrationTests/factory/stateComponents/stateComponents_test.go +++ b/integrationTests/factory/stateComponents/stateComponents_test.go @@ -10,7 +10,6 @@ import ( "github.com/multiversx/mx-chain-go/integrationTests/factory" "github.com/multiversx/mx-chain-go/node" - "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/testscommon/goroutines" ) @@ -38,8 +37,7 @@ func TestStateComponents_Create_Close_ShouldWork(t *testing.T) { require.Nil(t, err) managedNetworkComponents, err := nr.CreateManagedNetworkComponents(managedCoreComponents, managedStatusCoreComponents, managedCryptoComponents) require.Nil(t, err) - interceptedDataVerifierFactory := &mock.InterceptedDataVerifierFactoryStub{} - managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents, interceptedDataVerifierFactory) + managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents) require.Nil(t, err) managedDataComponents, err := nr.CreateManagedDataComponents(managedStatusCoreComponents, managedCoreComponents, managedBootstrapComponents, managedCryptoComponents) require.Nil(t, err) diff --git a/integrationTests/factory/statusComponents/statusComponents_test.go b/integrationTests/factory/statusComponents/statusComponents_test.go index 2def4237ee8..38527da6a41 100644 --- a/integrationTests/factory/statusComponents/statusComponents_test.go +++ b/integrationTests/factory/statusComponents/statusComponents_test.go @@ -13,8 +13,6 @@ import ( bootstrapComp "github.com/multiversx/mx-chain-go/factory/bootstrap" "github.com/multiversx/mx-chain-go/integrationTests/factory" "github.com/multiversx/mx-chain-go/node" - "github.com/multiversx/mx-chain-go/process" - "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/testscommon/goroutines" ) @@ -43,10 +41,7 @@ func TestStatusComponents_Create_Close_ShouldWork(t *testing.T) { require.Nil(t, err) managedNetworkComponents, err := nr.CreateManagedNetworkComponents(managedCoreComponents, managedStatusCoreComponents, managedCryptoComponents) require.Nil(t, err) - interceptedDataVerifierFactory := &mock.InterceptedDataVerifierFactoryStub{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { - return &mock.InterceptedDataVerifierStub{}, nil - }} - managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents, interceptedDataVerifierFactory) + managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents) require.Nil(t, err) managedDataComponents, err := nr.CreateManagedDataComponents(managedStatusCoreComponents, managedCoreComponents, managedBootstrapComponents, managedCryptoComponents) require.Nil(t, err) @@ -110,7 +105,6 @@ func TestStatusComponents_Create_Close_ShouldWork(t *testing.T) { managedStatusCoreComponents, gasScheduleNotifier, nodesCoordinator, - interceptedDataVerifierFactory, ) require.Nil(t, err) time.Sleep(2 * time.Second) diff --git a/node/nodeRunner.go b/node/nodeRunner.go index f9b9ae500af..6a2490cd683 100644 --- a/node/nodeRunner.go +++ b/node/nodeRunner.go @@ -323,11 +323,7 @@ func (nr *nodeRunner) executeOneComponentCreationCycle( } log.Debug("creating bootstrap components") - interceptedDataVerifierFactory := factory.NewInterceptedDataVerifierFactory(factory.InterceptedDataVerifierFactoryArgs{ - CacheSpan: time.Duration(nr.configs.GeneralConfig.InterceptedDataVerifier.CacheSpanInSec), - CacheExpiry: time.Duration(nr.configs.GeneralConfig.InterceptedDataVerifier.CacheExpiryInSec), - }) - managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents, interceptedDataVerifierFactory) + managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents) if err != nil { return true, err } @@ -438,7 +434,6 @@ func (nr *nodeRunner) executeOneComponentCreationCycle( managedStatusCoreComponents, gasScheduleNotifier, nodesCoordinatorInstance, - interceptedDataVerifierFactory, ) if err != nil { return true, err @@ -1164,7 +1159,6 @@ func (nr *nodeRunner) CreateManagedProcessComponents( statusCoreComponents mainFactory.StatusCoreComponentsHolder, gasScheduleNotifier core.GasScheduleNotifier, nodesCoordinator nodesCoordinator.NodesCoordinator, - interceptedDataVerifierFactory process.InterceptedDataVerifierFactory, ) (mainFactory.ProcessComponentsHandler, error) { configs := nr.configs configurationPaths := nr.configs.ConfigurationPathsHolder @@ -1244,6 +1238,11 @@ func (nr *nodeRunner) CreateManagedProcessComponents( txExecutionOrderHandler := ordering.NewOrderedCollection() + interceptedDataVerifierFactory := factory.NewInterceptedDataVerifierFactory(factory.InterceptedDataVerifierFactoryArgs{ + CacheSpan: time.Duration(nr.configs.GeneralConfig.InterceptedDataVerifier.CacheExpiryInSec), + CacheExpiry: time.Duration(nr.configs.GeneralConfig.InterceptedDataVerifier.CacheExpiryInSec), + }) + processArgs := processComp.ProcessComponentsFactoryArgs{ Config: *configs.GeneralConfig, EpochConfig: *configs.EpochConfig, @@ -1386,20 +1385,18 @@ func (nr *nodeRunner) CreateManagedBootstrapComponents( coreComponents mainFactory.CoreComponentsHolder, cryptoComponents mainFactory.CryptoComponentsHolder, networkComponents mainFactory.NetworkComponentsHolder, - interceptedDataVerifierFactory process.InterceptedDataVerifierFactory, ) (mainFactory.BootstrapComponentsHandler, error) { bootstrapComponentsFactoryArgs := bootstrapComp.BootstrapComponentsFactoryArgs{ - Config: *nr.configs.GeneralConfig, - PrefConfig: *nr.configs.PreferencesConfig, - ImportDbConfig: *nr.configs.ImportDbConfig, - FlagsConfig: *nr.configs.FlagsConfig, - WorkingDir: nr.configs.FlagsConfig.DbDir, - CoreComponents: coreComponents, - CryptoComponents: cryptoComponents, - NetworkComponents: networkComponents, - StatusCoreComponents: statusCoreComponents, - InterceptedDataVerifierFactory: interceptedDataVerifierFactory, + Config: *nr.configs.GeneralConfig, + PrefConfig: *nr.configs.PreferencesConfig, + ImportDbConfig: *nr.configs.ImportDbConfig, + FlagsConfig: *nr.configs.FlagsConfig, + WorkingDir: nr.configs.FlagsConfig.DbDir, + CoreComponents: coreComponents, + CryptoComponents: cryptoComponents, + NetworkComponents: networkComponents, + StatusCoreComponents: statusCoreComponents, } bootstrapComponentsFactory, err := bootstrapComp.NewBootstrapComponentsFactory(bootstrapComponentsFactoryArgs) diff --git a/process/errors.go b/process/errors.go index a126b0f7513..87a1987ee30 100644 --- a/process/errors.go +++ b/process/errors.go @@ -1098,6 +1098,9 @@ var ErrInvalidExpiryTimespan = errors.New("invalid expiry timespan") // ErrNilPeerSignatureHandler signals that a nil peer signature handler was provided var ErrNilPeerSignatureHandler = errors.New("nil peer signature handler") +// ErrNilInterceptedDataVerifierFactory signals that a nil intercepted data verifier factory was provided +var ErrNilInterceptedDataVerifierFactory = errors.New("nil intercepted data verifier factory") + // ErrNilPeerAuthenticationCacher signals that a nil peer authentication cacher was provided var ErrNilPeerAuthenticationCacher = errors.New("nil peer authentication cacher") diff --git a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go index 6d92b8a34c4..1630ca56f48 100644 --- a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go +++ b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go @@ -81,6 +81,9 @@ func NewMetaInterceptorsContainerFactory( if check.IfNil(args.PeerSignatureHandler) { return nil, process.ErrNilPeerSignatureHandler } + if check.IfNil(args.InterceptedDataVerifierFactory) { + return nil, process.ErrNilInterceptedDataVerifierFactory + } if args.HeartbeatExpiryTimespanInSec < minTimespanDurationInSec { return nil, process.ErrInvalidExpiryTimespan } diff --git a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go index bdc72590993..d42961eff20 100644 --- a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go +++ b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go @@ -81,6 +81,9 @@ func NewShardInterceptorsContainerFactory( if check.IfNil(args.PeerSignatureHandler) { return nil, process.ErrNilPeerSignatureHandler } + if check.IfNil(args.InterceptedDataVerifierFactory) { + return nil, process.ErrNilInterceptedDataVerifierFactory + } if args.HeartbeatExpiryTimespanInSec < minTimespanDurationInSec { return nil, process.ErrInvalidExpiryTimespan } From 23dbb5a536b9cd7074832509d285bbe61bcce241 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Thu, 3 Oct 2024 10:47:51 +0300 Subject: [PATCH 34/37] more fixes. --- epochStart/bootstrap/process_test.go | 12 ++- epochStart/bootstrap/storageProcess_test.go | 4 +- .../bootstrap/syncEpochStartMeta_test.go | 9 +-- factory/processing/processComponents_test.go | 4 +- process/errors.go | 6 ++ .../metaInterceptorsContainerFactory_test.go | 16 ++-- .../shardInterceptorsContainerFactory_test.go | 16 ++-- .../factory/interceptedDataVerifierFactory.go | 28 ++++--- .../interceptedDataVerifierFactory_test.go | 44 ++++++++++ .../interceptors/interceptedDataVerifier.go | 34 ++++---- .../interceptedDataVerifier_test.go | 70 +++++++++++++--- .../interceptors/multiDataInterceptor_test.go | 17 +++- .../singleDataInterceptor_test.go | 15 +++- process/interface.go | 2 + ... => interceptedDataVerifierFactoryMock.go} | 10 +-- process/mock/interceptedDataVerifierMock.go | 8 +- testscommon/components/components.go | 5 +- update/factory/fullSyncInterceptors.go | 80 ++++++++----------- 18 files changed, 241 insertions(+), 139 deletions(-) create mode 100644 process/interceptors/factory/interceptedDataVerifierFactory_test.go rename process/mock/{interceptedDataVerifierFactoryStub.go => interceptedDataVerifierFactoryMock.go} (56%) diff --git a/epochStart/bootstrap/process_test.go b/epochStart/bootstrap/process_test.go index 3f142fe8459..dfc2c42411a 100644 --- a/epochStart/bootstrap/process_test.go +++ b/epochStart/bootstrap/process_test.go @@ -252,11 +252,9 @@ func createMockEpochStartBootstrapArgs( FlagsConfig: config.ContextFlagsConfig{ ForceStartFromNetwork: false, }, - TrieSyncStatisticsProvider: &testscommon.SizeSyncStatisticsHandlerStub{}, - StateStatsHandler: disabledStatistics.NewStateStatistics(), - InterceptedDataVerifierFactory: &processMock.InterceptedDataVerifierFactoryStub{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { - return &processMock.InterceptedDataVerifierStub{}, nil - }}, + TrieSyncStatisticsProvider: &testscommon.SizeSyncStatisticsHandlerStub{}, + StateStatsHandler: disabledStatistics.NewStateStatistics(), + InterceptedDataVerifierFactory: &processMock.InterceptedDataVerifierFactoryMock{}, } } @@ -998,8 +996,8 @@ func TestCreateSyncers(t *testing.T) { epochStartProvider.whiteListerVerifiedTxs = &testscommon.WhiteListHandlerStub{} epochStartProvider.requestHandler = &testscommon.RequestHandlerStub{} epochStartProvider.storageService = &storageMocks.ChainStorerStub{} - epochStartProvider.interceptedDataVerifierFactory = &processMock.InterceptedDataVerifierFactoryStub{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { - return &processMock.InterceptedDataVerifierStub{}, nil + epochStartProvider.interceptedDataVerifierFactory = &processMock.InterceptedDataVerifierFactoryMock{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { + return &processMock.InterceptedDataVerifierMock{}, nil }} err := epochStartProvider.createSyncers() diff --git a/epochStart/bootstrap/storageProcess_test.go b/epochStart/bootstrap/storageProcess_test.go index 81f06ee801a..16a3b506cb4 100644 --- a/epochStart/bootstrap/storageProcess_test.go +++ b/epochStart/bootstrap/storageProcess_test.go @@ -129,8 +129,8 @@ func TestStorageEpochStartBootstrap_BootstrapMetablockNotFound(t *testing.T) { } args.GeneralConfig = testscommon.GetGeneralConfig() args.GeneralConfig.EpochStartConfig.RoundsPerEpoch = roundsPerEpoch - args.InterceptedDataVerifierFactory = &processMock.InterceptedDataVerifierFactoryStub{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { - return &processMock.InterceptedDataVerifierStub{}, nil + args.InterceptedDataVerifierFactory = &processMock.InterceptedDataVerifierFactoryMock{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { + return &processMock.InterceptedDataVerifierMock{}, nil }} sesb, _ := NewStorageEpochStartBootstrap(args) diff --git a/epochStart/bootstrap/syncEpochStartMeta_test.go b/epochStart/bootstrap/syncEpochStartMeta_test.go index 49a7263d0cc..4cf8babeb2d 100644 --- a/epochStart/bootstrap/syncEpochStartMeta_test.go +++ b/epochStart/bootstrap/syncEpochStartMeta_test.go @@ -16,7 +16,6 @@ import ( "github.com/multiversx/mx-chain-go/epochStart" "github.com/multiversx/mx-chain-go/epochStart/mock" "github.com/multiversx/mx-chain-go/p2p" - "github.com/multiversx/mx-chain-go/process" processMock "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/testscommon" "github.com/multiversx/mx-chain-go/testscommon/cryptoMocks" @@ -163,10 +162,8 @@ func getEpochStartSyncerArgs() ArgsNewEpochStartMetaSyncer { MinNumConnectedPeersToStart: 2, MinNumOfPeersToConsiderBlockValid: 2, }, - HeaderIntegrityVerifier: &mock.HeaderIntegrityVerifierStub{}, - MetaBlockProcessor: &mock.EpochStartMetaBlockProcessorStub{}, - InterceptedDataVerifierFactory: &processMock.InterceptedDataVerifierFactoryStub{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { - return &processMock.InterceptedDataVerifierStub{}, nil - }}, + HeaderIntegrityVerifier: &mock.HeaderIntegrityVerifierStub{}, + MetaBlockProcessor: &mock.EpochStartMetaBlockProcessorStub{}, + InterceptedDataVerifierFactory: &processMock.InterceptedDataVerifierFactoryMock{}, } } diff --git a/factory/processing/processComponents_test.go b/factory/processing/processComponents_test.go index fecfa98165b..606b8470edb 100644 --- a/factory/processing/processComponents_test.go +++ b/factory/processing/processComponents_test.go @@ -267,9 +267,9 @@ func createMockProcessComponentsFactoryArgs() processComp.ProcessComponentsFacto } args.State = components.GetStateComponents(args.CoreData, args.StatusCoreComponents) - args.InterceptedDataVerifierFactory = &processMocks.InterceptedDataVerifierFactoryStub{ + args.InterceptedDataVerifierFactory = &processMocks.InterceptedDataVerifierFactoryMock{ CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { - return &processMocks.InterceptedDataVerifierStub{}, nil + return &processMocks.InterceptedDataVerifierMock{}, nil }, } diff --git a/process/errors.go b/process/errors.go index 87a1987ee30..d6d38c75180 100644 --- a/process/errors.go +++ b/process/errors.go @@ -1257,5 +1257,11 @@ var ErrNilEquivalentProofsPool = errors.New("nil equivalent proofs pool") // ErrNilHeaderProof signals that a nil header proof has been provided var ErrNilHeaderProof = errors.New("nil header proof") +// ErrNilInterceptedDataCache signals that a nil cacher was provided for intercepted data verifier +var ErrNilInterceptedDataCache = errors.New("nil cache for intercepted data") + // ErrFlagNotActive signals that a flag is not active var ErrFlagNotActive = errors.New("flag not active") + +// ErrInvalidInterceptedData signals that an invalid data has been intercepted +var ErrInvalidInterceptedData = errors.New("invalid intercepted data") diff --git a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go index 43a38079f45..cb09ed78ee2 100644 --- a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go +++ b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go @@ -546,8 +546,8 @@ func testCreateMetaTopicShouldFail(matchStrToErrOnCreate string, matchStrToErrOn } else { args.MainMessenger = createMetaStubTopicHandler(matchStrToErrOnCreate, matchStrToErrOnRegister) } - args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryStub{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { - return &mock.InterceptedDataVerifierStub{}, nil + args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryMock{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { + return &mock.InterceptedDataVerifierMock{}, nil }} icf, _ := interceptorscontainer.NewMetaInterceptorsContainerFactory(args) @@ -564,8 +564,8 @@ func TestMetaInterceptorsContainerFactory_CreateShouldWork(t *testing.T) { coreComp, cryptoComp := createMockComponentHolders() args := getArgumentsMeta(coreComp, cryptoComp) - args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryStub{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { - return &mock.InterceptedDataVerifierStub{}, nil + args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryMock{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { + return &mock.InterceptedDataVerifierMock{}, nil }} icf, _ := interceptorscontainer.NewMetaInterceptorsContainerFactory(args) @@ -599,8 +599,8 @@ func TestMetaInterceptorsContainerFactory_With4ShardsShouldWork(t *testing.T) { args := getArgumentsMeta(coreComp, cryptoComp) args.ShardCoordinator = shardCoordinator args.NodesCoordinator = nodesCoordinator - args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryStub{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { - return &mock.InterceptedDataVerifierStub{}, nil + args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryMock{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { + return &mock.InterceptedDataVerifierMock{}, nil }} icf, err := interceptorscontainer.NewMetaInterceptorsContainerFactory(args) @@ -653,8 +653,8 @@ func TestMetaInterceptorsContainerFactory_With4ShardsShouldWork(t *testing.T) { args.NodeOperationMode = common.FullArchiveMode args.ShardCoordinator = shardCoordinator args.NodesCoordinator = nodesCoordinator - args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryStub{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { - return &mock.InterceptedDataVerifierStub{}, nil + args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryMock{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { + return &mock.InterceptedDataVerifierMock{}, nil }} icf, err := interceptorscontainer.NewMetaInterceptorsContainerFactory(args) diff --git a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go index 44fe9032cac..a59b4b965a8 100644 --- a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go +++ b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go @@ -497,8 +497,8 @@ func testCreateShardTopicShouldFail(matchStrToErrOnCreate string, matchStrToErrO coreComp, cryptoComp := createMockComponentHolders() args := getArgumentsShard(coreComp, cryptoComp) - args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryStub{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { - return &mock.InterceptedDataVerifierStub{}, nil + args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryMock{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { + return &mock.InterceptedDataVerifierMock{}, nil }} if strings.Contains(t.Name(), "full_archive") { args.NodeOperationMode = common.FullArchiveMode @@ -566,8 +566,8 @@ func TestShardInterceptorsContainerFactory_CreateShouldWork(t *testing.T) { }, } args.WhiteListerVerifiedTxs = &testscommon.WhiteListHandlerStub{} - args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryStub{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { - return &mock.InterceptedDataVerifierStub{}, nil + args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryMock{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { + return &mock.InterceptedDataVerifierMock{}, nil }} icf, _ := interceptorscontainer.NewShardInterceptorsContainerFactory(args) @@ -604,8 +604,8 @@ func TestShardInterceptorsContainerFactory_With4ShardsShouldWork(t *testing.T) { args.ShardCoordinator = shardCoordinator args.NodesCoordinator = nodesCoordinator args.PreferredPeersHolder = &p2pmocks.PeersHolderStub{} - args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryStub{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { - return &mock.InterceptedDataVerifierStub{}, nil + args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryMock{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { + return &mock.InterceptedDataVerifierMock{}, nil }} icf, _ := interceptorscontainer.NewShardInterceptorsContainerFactory(args) @@ -657,8 +657,8 @@ func TestShardInterceptorsContainerFactory_With4ShardsShouldWork(t *testing.T) { args.ShardCoordinator = shardCoordinator args.NodesCoordinator = nodesCoordinator args.PreferredPeersHolder = &p2pmocks.PeersHolderStub{} - args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryStub{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { - return &mock.InterceptedDataVerifierStub{}, nil + args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryMock{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { + return &mock.InterceptedDataVerifierMock{}, nil }} icf, _ := interceptorscontainer.NewShardInterceptorsContainerFactory(args) diff --git a/process/interceptors/factory/interceptedDataVerifierFactory.go b/process/interceptors/factory/interceptedDataVerifierFactory.go index 544b82e257b..db50dfebd92 100644 --- a/process/interceptors/factory/interceptedDataVerifierFactory.go +++ b/process/interceptors/factory/interceptedDataVerifierFactory.go @@ -1,6 +1,7 @@ package factory import ( + "sync" "time" "github.com/multiversx/mx-chain-go/process" @@ -9,31 +10,34 @@ import ( "github.com/multiversx/mx-chain-go/storage/cache" ) -// InterceptedDataVerifierFactoryArgs holds the required arguments for InterceptedDataVerifierFactory +// InterceptedDataVerifierFactoryArgs holds the required arguments for interceptedDataVerifierFactory type InterceptedDataVerifierFactoryArgs struct { CacheSpan time.Duration CacheExpiry time.Duration } -// InterceptedDataVerifierFactory encapsulates the required arguments to create InterceptedDataVerifier +// interceptedDataVerifierFactory encapsulates the required arguments to create InterceptedDataVerifier // Furthermore it will hold all such instances in an internal map. -type InterceptedDataVerifierFactory struct { - cacheSpan time.Duration - cacheExpiry time.Duration +type interceptedDataVerifierFactory struct { + cacheSpan time.Duration + cacheExpiry time.Duration + interceptedDataVerifierMap map[string]storage.Cacher + mutex sync.Mutex } // NewInterceptedDataVerifierFactory will create a factory instance that will create instance of InterceptedDataVerifiers -func NewInterceptedDataVerifierFactory(args InterceptedDataVerifierFactoryArgs) *InterceptedDataVerifierFactory { - return &InterceptedDataVerifierFactory{ +func NewInterceptedDataVerifierFactory(args InterceptedDataVerifierFactoryArgs) *interceptedDataVerifierFactory { + return &interceptedDataVerifierFactory{ cacheSpan: args.CacheSpan, cacheExpiry: args.CacheExpiry, interceptedDataVerifierMap: make(map[string]storage.Cacher), + mutex: sync.Mutex{}, } } // Create will return an instance of InterceptedDataVerifier -func (idvf *InterceptedDataVerifierFactory) Create(topic string) (process.InterceptedDataVerifier, error) { +func (idvf *interceptedDataVerifierFactory) Create(topic string) (process.InterceptedDataVerifier, error) { internalCache, err := cache.NewTimeCacher(cache.ArgTimeCacher{ DefaultSpan: idvf.cacheSpan, CacheExpiry: idvf.cacheExpiry, @@ -42,12 +46,14 @@ func (idvf *InterceptedDataVerifierFactory) Create(topic string) (process.Interc return nil, err } + idvf.mutex.Lock() idvf.interceptedDataVerifierMap[topic] = internalCache - verifier := interceptors.NewInterceptedDataVerifier(internalCache) - return verifier, nil + idvf.mutex.Unlock() + + return interceptors.NewInterceptedDataVerifier(internalCache) } // IsInterfaceNil returns true if there is no value under the interface -func (idvf *InterceptedDataVerifierFactory) IsInterfaceNil() bool { +func (idvf *interceptedDataVerifierFactory) IsInterfaceNil() bool { return idvf == nil } diff --git a/process/interceptors/factory/interceptedDataVerifierFactory_test.go b/process/interceptors/factory/interceptedDataVerifierFactory_test.go new file mode 100644 index 00000000000..45f42ec05fd --- /dev/null +++ b/process/interceptors/factory/interceptedDataVerifierFactory_test.go @@ -0,0 +1,44 @@ +package factory + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func createMockArgInterceptedDataVerifierFactory() InterceptedDataVerifierFactoryArgs { + return InterceptedDataVerifierFactoryArgs{ + CacheSpan: time.Second, + CacheExpiry: time.Second, + } +} + +func TestInterceptedDataVerifierFactory_IsInterfaceNil(t *testing.T) { + t.Parallel() + + var factory *interceptedDataVerifierFactory + require.True(t, factory.IsInterfaceNil()) + + factory = NewInterceptedDataVerifierFactory(createMockArgInterceptedDataVerifierFactory()) + require.False(t, factory.IsInterfaceNil()) +} + +func TestNewInterceptedDataVerifierFactory(t *testing.T) { + t.Parallel() + + factory := NewInterceptedDataVerifierFactory(createMockArgInterceptedDataVerifierFactory()) + require.NotNil(t, factory) +} + +func TestInterceptedDataVerifierFactory_Create(t *testing.T) { + t.Parallel() + + factory := NewInterceptedDataVerifierFactory(createMockArgInterceptedDataVerifierFactory()) + require.NotNil(t, factory) + + interceptedDataVerifier, err := factory.Create("mockTopic") + require.NoError(t, err) + + require.False(t, interceptedDataVerifier.IsInterfaceNil()) +} diff --git a/process/interceptors/interceptedDataVerifier.go b/process/interceptors/interceptedDataVerifier.go index 9eef5dbeed8..0accf41d3fc 100644 --- a/process/interceptors/interceptedDataVerifier.go +++ b/process/interceptors/interceptedDataVerifier.go @@ -1,8 +1,7 @@ package interceptors import ( - "errors" - + "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/core/sync" "github.com/multiversx/mx-chain-go/process" @@ -12,59 +11,56 @@ import ( type interceptedDataStatus int8 const ( - ValidInterceptedData interceptedDataStatus = iota - InvalidInterceptedData + validInterceptedData interceptedDataStatus = iota + invalidInterceptedData interceptedDataStatusBytesSize = 8 ) -var ( - ErrInvalidInterceptedData = errors.New("invalid intercepted data") -) - type interceptedDataVerifier struct { km sync.KeyRWMutexHandler cache storage.Cacher } // NewInterceptedDataVerifier creates a new instance of intercepted data verifier -func NewInterceptedDataVerifier(cache storage.Cacher) *interceptedDataVerifier { - keyRWMutex := sync.NewKeyRWMutex() +func NewInterceptedDataVerifier(cache storage.Cacher) (*interceptedDataVerifier, error) { + if check.IfNil(cache) { + return nil, process.ErrNilInterceptedDataCache + } return &interceptedDataVerifier{ - km: keyRWMutex, + km: sync.NewKeyRWMutex(), cache: cache, - } + }, nil } // Verify will check if the intercepted data has been validated before and put in the time cache. // It will retrieve the status in the cache if it exists, otherwise it will validate it and store the status of the // validation in the cache. Note that the entries are stored for a set period of time func (idv *interceptedDataVerifier) Verify(interceptedData process.InterceptedData) error { - hash := string(interceptedData.Hash()) - if len(interceptedData.Hash()) == 0 { return interceptedData.CheckValidity() } + hash := string(interceptedData.Hash()) idv.km.Lock(hash) defer idv.km.Unlock(hash) if val, ok := idv.cache.Get(interceptedData.Hash()); ok { - if val == ValidInterceptedData { + if val == validInterceptedData { return nil } - return ErrInvalidInterceptedData + return process.ErrInvalidInterceptedData } err := interceptedData.CheckValidity() if err != nil { - idv.cache.Put(interceptedData.Hash(), InvalidInterceptedData, interceptedDataStatusBytesSize) - return ErrInvalidInterceptedData + idv.cache.Put(interceptedData.Hash(), invalidInterceptedData, interceptedDataStatusBytesSize) + return process.ErrInvalidInterceptedData } - idv.cache.Put(interceptedData.Hash(), ValidInterceptedData, interceptedDataStatusBytesSize) + idv.cache.Put(interceptedData.Hash(), validInterceptedData, interceptedDataStatusBytesSize) return nil } diff --git a/process/interceptors/interceptedDataVerifier_test.go b/process/interceptors/interceptedDataVerifier_test.go index 90f4555315a..8913f5828d8 100644 --- a/process/interceptors/interceptedDataVerifier_test.go +++ b/process/interceptors/interceptedDataVerifier_test.go @@ -9,19 +9,67 @@ import ( "github.com/stretchr/testify/require" "github.com/multiversx/mx-chain-go/process" + "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/storage/cache" "github.com/multiversx/mx-chain-go/testscommon" ) const defaultSpan = 1 * time.Second -func defaultInterceptedDataVerifier(span time.Duration) process.InterceptedDataVerifier { +func defaultInterceptedDataVerifier(span time.Duration) *interceptedDataVerifier { c, _ := cache.NewTimeCacher(cache.ArgTimeCacher{ DefaultSpan: span, CacheExpiry: span, }) - return NewInterceptedDataVerifier(c) + verifier, _ := NewInterceptedDataVerifier(c) + return verifier +} + +func TestNewInterceptedDataVerifier(t *testing.T) { + t.Parallel() + + var c storage.Cacher + verifier, err := NewInterceptedDataVerifier(c) + require.Equal(t, process.ErrNilInterceptedDataCache, err) + require.Nil(t, verifier) +} + +func TestInterceptedDataVerifier_IsInterfaceNil(t *testing.T) { + t.Parallel() + + var verifier *interceptedDataVerifier + require.True(t, verifier.IsInterfaceNil()) + + verifier = defaultInterceptedDataVerifier(defaultSpan) + require.False(t, verifier.IsInterfaceNil()) +} + +func TestInterceptedDataVerifier_EmptyHash(t *testing.T) { + t.Parallel() + + var checkValidityCounter int + verifier := defaultInterceptedDataVerifier(defaultSpan) + interceptedData := &testscommon.InterceptedDataStub{ + CheckValidityCalled: func() error { + checkValidityCounter++ + return nil + }, + IsForCurrentShardCalled: func() bool { + return false + }, + HashCalled: func() []byte { + return nil + }, + } + + err := verifier.Verify(interceptedData) + require.NoError(t, err) + require.Equal(t, 1, checkValidityCounter) + + err = verifier.Verify(interceptedData) + require.NoError(t, err) + require.Equal(t, 2, checkValidityCounter) } func TestInterceptedDataVerifier_CheckValidityShouldWork(t *testing.T) { @@ -49,7 +97,7 @@ func TestInterceptedDataVerifier_CheckValidityShouldWork(t *testing.T) { errCount := atomic.Counter{} wg := sync.WaitGroup{} - for i := 0; i < 3; i++ { + for i := 0; i < 100; i++ { wg.Add(1) go func() { defer wg.Done() @@ -73,7 +121,7 @@ func TestInterceptedDataVerifier_CheckValidityShouldNotWork(t *testing.T) { interceptedData := &testscommon.InterceptedDataStub{ CheckValidityCalled: func() error { checkValidityCounter.Add(1) - return ErrInvalidInterceptedData + return process.ErrInvalidInterceptedData }, IsForCurrentShardCalled: func() bool { return false @@ -86,11 +134,11 @@ func TestInterceptedDataVerifier_CheckValidityShouldNotWork(t *testing.T) { verifier := defaultInterceptedDataVerifier(defaultSpan) err := verifier.Verify(interceptedData) - require.Equal(t, ErrInvalidInterceptedData, err) + require.Equal(t, process.ErrInvalidInterceptedData, err) errCount := atomic.Counter{} wg := sync.WaitGroup{} - for i := 0; i < 3; i++ { + for i := 0; i < 100; i++ { wg.Add(1) go func() { defer wg.Done() @@ -102,7 +150,7 @@ func TestInterceptedDataVerifier_CheckValidityShouldNotWork(t *testing.T) { } wg.Wait() - require.Equal(t, int64(3), errCount.Get()) + require.Equal(t, int64(100), errCount.Get()) require.Equal(t, int64(1), checkValidityCounter.Get()) } @@ -156,7 +204,7 @@ func TestInterceptedDataVerifier_CheckExpiryTime(t *testing.T) { interceptedData := &testscommon.InterceptedDataStub{ CheckValidityCalled: func() error { checkValidityCounter.Add(1) - return ErrInvalidInterceptedData + return process.ErrInvalidInterceptedData }, IsForCurrentShardCalled: func() bool { return false @@ -170,12 +218,12 @@ func TestInterceptedDataVerifier_CheckExpiryTime(t *testing.T) { // First retrieval, check validity is reached. err := verifier.Verify(interceptedData) - require.Equal(t, ErrInvalidInterceptedData, err) + require.Equal(t, process.ErrInvalidInterceptedData, err) require.Equal(t, int64(1), checkValidityCounter.Get()) // Second retrieval should be from the cache. err = verifier.Verify(interceptedData) - require.Equal(t, ErrInvalidInterceptedData, err) + require.Equal(t, process.ErrInvalidInterceptedData, err) require.Equal(t, int64(1), checkValidityCounter.Get()) // Wait for the cache expiry @@ -183,7 +231,7 @@ func TestInterceptedDataVerifier_CheckExpiryTime(t *testing.T) { // Third retrieval should reach validity check again. err = verifier.Verify(interceptedData) - require.Equal(t, ErrInvalidInterceptedData, err) + require.Equal(t, process.ErrInvalidInterceptedData, err) require.Equal(t, int64(2), checkValidityCounter.Get()) }) } diff --git a/process/interceptors/multiDataInterceptor_test.go b/process/interceptors/multiDataInterceptor_test.go index 13b9dadae38..ede867dba07 100644 --- a/process/interceptors/multiDataInterceptor_test.go +++ b/process/interceptors/multiDataInterceptor_test.go @@ -33,7 +33,7 @@ func createMockArgMultiDataInterceptor() interceptors.ArgMultiDataInterceptor { WhiteListRequest: &testscommon.WhiteListHandlerStub{}, PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, CurrentPeerId: "pid", - InterceptedDataVerifier: &mock.InterceptedDataVerifierStub{}, + InterceptedDataVerifier: &mock.InterceptedDataVerifierMock{}, } } @@ -70,6 +70,17 @@ func TestNewMultiDataInterceptor_NilInterceptedDataFactoryShouldErr(t *testing.T assert.Equal(t, process.ErrNilInterceptedDataFactory, err) } +func TestNewMultiDataInterceptor_NilInterceptedDataVerifierShouldErr(t *testing.T) { + t.Parallel() + + arg := createMockArgMultiDataInterceptor() + arg.InterceptedDataVerifier = nil + mdi, err := interceptors.NewMultiDataInterceptor(arg) + + assert.True(t, check.IfNil(mdi)) + assert.Equal(t, process.ErrNilInterceptedDataVerifier, err) +} + func TestNewMultiDataInterceptor_NilInterceptedDataProcessorShouldErr(t *testing.T) { t.Parallel() @@ -357,7 +368,7 @@ func testProcessReceiveMessageMultiData(t *testing.T, isForCurrentShard bool, ex } arg.Processor = createMockInterceptorStub(&checkCalledNum, &processCalledNum) arg.Throttler = throttler - arg.InterceptedDataVerifier = &mock.InterceptedDataVerifierStub{ + arg.InterceptedDataVerifier = &mock.InterceptedDataVerifierMock{ VerifyCalled: func(interceptedData process.InterceptedData) error { return interceptedData.CheckValidity() }, @@ -614,7 +625,7 @@ func processReceivedMessageMultiDataInvalidVersion(t *testing.T, expectedErr err return true }, } - arg.InterceptedDataVerifier = &mock.InterceptedDataVerifierStub{ + arg.InterceptedDataVerifier = &mock.InterceptedDataVerifierMock{ VerifyCalled: func(interceptedData process.InterceptedData) error { return interceptedData.CheckValidity() }, diff --git a/process/interceptors/singleDataInterceptor_test.go b/process/interceptors/singleDataInterceptor_test.go index 408d6d52078..9b1fad0a840 100644 --- a/process/interceptors/singleDataInterceptor_test.go +++ b/process/interceptors/singleDataInterceptor_test.go @@ -59,8 +59,8 @@ func createMockThrottler() *mock.InterceptorThrottlerStub { } } -func createMockInterceptedDataVerifier() *mock.InterceptedDataVerifierStub { - return &mock.InterceptedDataVerifierStub{ +func createMockInterceptedDataVerifier() *mock.InterceptedDataVerifierMock { + return &mock.InterceptedDataVerifierMock{ VerifyCalled: func(interceptedData process.InterceptedData) error { return interceptedData.CheckValidity() }, @@ -155,6 +155,17 @@ func TestNewSingleDataInterceptor_EmptyPeerIDShouldErr(t *testing.T) { assert.Equal(t, process.ErrEmptyPeerID, err) } +func TestNewSingleDataInterceptor_NilInterceptedDataVerifierShouldErr(t *testing.T) { + t.Parallel() + + arg := createMockArgMultiDataInterceptor() + arg.InterceptedDataVerifier = nil + mdi, err := interceptors.NewMultiDataInterceptor(arg) + + assert.True(t, check.IfNil(mdi)) + assert.Equal(t, process.ErrNilInterceptedDataVerifier, err) +} + func TestNewSingleDataInterceptor(t *testing.T) { t.Parallel() diff --git a/process/interface.go b/process/interface.go index 53c9ac48645..117c8376f0c 100644 --- a/process/interface.go +++ b/process/interface.go @@ -1402,11 +1402,13 @@ type SentSignaturesTracker interface { IsInterfaceNil() bool } +// InterceptedDataVerifier defines a component able to verify intercepted data validity type InterceptedDataVerifier interface { Verify(interceptedData InterceptedData) error IsInterfaceNil() bool } +// InterceptedDataVerifierFactory defines a component that is able to create intercepted data verifiers type InterceptedDataVerifierFactory interface { Create(topic string) (InterceptedDataVerifier, error) IsInterfaceNil() bool diff --git a/process/mock/interceptedDataVerifierFactoryStub.go b/process/mock/interceptedDataVerifierFactoryMock.go similarity index 56% rename from process/mock/interceptedDataVerifierFactoryStub.go rename to process/mock/interceptedDataVerifierFactoryMock.go index 6fdd9874903..2b17d849563 100644 --- a/process/mock/interceptedDataVerifierFactoryStub.go +++ b/process/mock/interceptedDataVerifierFactoryMock.go @@ -4,21 +4,21 @@ import ( "github.com/multiversx/mx-chain-go/process" ) -// InterceptedDataVerifierFactoryStub - -type InterceptedDataVerifierFactoryStub struct { +// InterceptedDataVerifierFactoryMock - +type InterceptedDataVerifierFactoryMock struct { CreateCalled func(topic string) (process.InterceptedDataVerifier, error) } // Create - -func (idvfs *InterceptedDataVerifierFactoryStub) Create(topic string) (process.InterceptedDataVerifier, error) { +func (idvfs *InterceptedDataVerifierFactoryMock) Create(topic string) (process.InterceptedDataVerifier, error) { if idvfs.CreateCalled != nil { return idvfs.CreateCalled(topic) } - return nil, nil + return &InterceptedDataVerifierMock{}, nil } // IsInterfaceNil - -func (idvfs *InterceptedDataVerifierFactoryStub) IsInterfaceNil() bool { +func (idvfs *InterceptedDataVerifierFactoryMock) IsInterfaceNil() bool { return idvfs == nil } diff --git a/process/mock/interceptedDataVerifierMock.go b/process/mock/interceptedDataVerifierMock.go index f6329cc408c..6668a6ea625 100644 --- a/process/mock/interceptedDataVerifierMock.go +++ b/process/mock/interceptedDataVerifierMock.go @@ -4,13 +4,13 @@ import ( "github.com/multiversx/mx-chain-go/process" ) -// InterceptedDataVerifierStub - -type InterceptedDataVerifierStub struct { +// InterceptedDataVerifierMock - +type InterceptedDataVerifierMock struct { VerifyCalled func(interceptedData process.InterceptedData) error } // Verify - -func (idv *InterceptedDataVerifierStub) Verify(interceptedData process.InterceptedData) error { +func (idv *InterceptedDataVerifierMock) Verify(interceptedData process.InterceptedData) error { if idv.VerifyCalled != nil { return idv.VerifyCalled(interceptedData) } @@ -19,6 +19,6 @@ func (idv *InterceptedDataVerifierStub) Verify(interceptedData process.Intercept } // IsInterfaceNil - -func (idv *InterceptedDataVerifierStub) IsInterfaceNil() bool { +func (idv *InterceptedDataVerifierMock) IsInterfaceNil() bool { return idv == nil } diff --git a/testscommon/components/components.go b/testscommon/components/components.go index 146c9f1f7dc..daab8391b39 100644 --- a/testscommon/components/components.go +++ b/testscommon/components/components.go @@ -35,7 +35,6 @@ import ( "github.com/multiversx/mx-chain-go/p2p" p2pConfig "github.com/multiversx/mx-chain-go/p2p/config" p2pFactory "github.com/multiversx/mx-chain-go/p2p/factory" - "github.com/multiversx/mx-chain-go/process" processMock "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/sharding" "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator" @@ -610,9 +609,7 @@ func GetProcessArgs( }, }, }, - InterceptedDataVerifierFactory: &processMock.InterceptedDataVerifierFactoryStub{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { - return &processMock.InterceptedDataVerifierStub{}, nil - }}, + InterceptedDataVerifierFactory: &processMock.InterceptedDataVerifierFactoryMock{}, } } diff --git a/update/factory/fullSyncInterceptors.go b/update/factory/fullSyncInterceptors.go index 3fde2f96b2f..037155226c9 100644 --- a/update/factory/fullSyncInterceptors.go +++ b/update/factory/fullSyncInterceptors.go @@ -2,7 +2,6 @@ package factory import ( "fmt" - "time" "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" @@ -21,8 +20,6 @@ import ( "github.com/multiversx/mx-chain-go/sharding" "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator" "github.com/multiversx/mx-chain-go/state" - "github.com/multiversx/mx-chain-go/storage" - "github.com/multiversx/mx-chain-go/storage/cache" "github.com/multiversx/mx-chain-go/update" "github.com/multiversx/mx-chain-go/update/disabled" ) @@ -33,26 +30,26 @@ const numGoRoutines = 2000 // fullSyncInterceptorsContainerFactory will handle the creation the interceptors container for shards type fullSyncInterceptorsContainerFactory struct { - mainContainer process.InterceptorsContainer - fullArchiveContainer process.InterceptorsContainer - shardCoordinator sharding.Coordinator - accounts state.AccountsAdapter - store dataRetriever.StorageService - dataPool dataRetriever.PoolsHolder - mainMessenger process.TopicHandler - fullArchiveMessenger process.TopicHandler - nodesCoordinator nodesCoordinator.NodesCoordinator - blockBlackList process.TimeCacher - argInterceptorFactory *interceptorFactory.ArgInterceptedDataFactory - globalThrottler process.InterceptorThrottler - maxTxNonceDeltaAllowed int - addressPubkeyConv core.PubkeyConverter - whiteListHandler update.WhiteListHandler - whiteListerVerifiedTxs update.WhiteListHandler - antifloodHandler process.P2PAntifloodHandler - preferredPeersHolder update.PreferredPeersHolderHandler - nodeOperationMode common.NodeOperation - interceptedDataCache map[string]storage.Cacher + mainContainer process.InterceptorsContainer + fullArchiveContainer process.InterceptorsContainer + shardCoordinator sharding.Coordinator + accounts state.AccountsAdapter + store dataRetriever.StorageService + dataPool dataRetriever.PoolsHolder + mainMessenger process.TopicHandler + fullArchiveMessenger process.TopicHandler + nodesCoordinator nodesCoordinator.NodesCoordinator + blockBlackList process.TimeCacher + argInterceptorFactory *interceptorFactory.ArgInterceptedDataFactory + globalThrottler process.InterceptorThrottler + maxTxNonceDeltaAllowed int + addressPubkeyConv core.PubkeyConverter + whiteListHandler update.WhiteListHandler + whiteListerVerifiedTxs update.WhiteListHandler + antifloodHandler process.P2PAntifloodHandler + preferredPeersHolder update.PreferredPeersHolderHandler + nodeOperationMode common.NodeOperation + interceptedDataVerifierFactory process.InterceptedDataVerifierFactory } // ArgsNewFullSyncInterceptorsContainerFactory holds the arguments needed for fullSyncInterceptorsContainerFactory @@ -138,6 +135,9 @@ func NewFullSyncInterceptorsContainerFactory( if check.IfNil(args.AntifloodHandler) { return nil, process.ErrNilAntifloodHandler } + if check.IfNil(args.InterceptedDataVerifierFactory) { + return nil, process.ErrNilInterceptedDataVerifierFactory + } argInterceptorFactory := &interceptorFactory.ArgInterceptedDataFactory{ CoreComponents: args.CoreComponents, @@ -170,8 +170,9 @@ func NewFullSyncInterceptorsContainerFactory( whiteListerVerifiedTxs: args.WhiteListerVerifiedTxs, antifloodHandler: args.AntifloodHandler, //TODO: inject the real peers holder once we have the peers mapping before epoch bootstrap finishes - preferredPeersHolder: disabled.NewPreferredPeersHolder(), - nodeOperationMode: args.NodeOperationMode, + preferredPeersHolder: disabled.NewPreferredPeersHolder(), + nodeOperationMode: args.NodeOperationMode, + interceptedDataVerifierFactory: args.InterceptedDataVerifierFactory, } icf.globalThrottler, err = throttler.NewNumGoRoutinesThrottler(numGoRoutines) @@ -355,7 +356,7 @@ func (ficf *fullSyncInterceptorsContainerFactory) createOneShardHeaderIntercepto return nil, err } - interceptedDataVerifier, err := ficf.createCacheForInterceptor(topic) + interceptedDataVerifier, err := ficf.interceptedDataVerifierFactory.Create(topic) if err != nil { return nil, err } @@ -563,7 +564,7 @@ func (ficf *fullSyncInterceptorsContainerFactory) createOneTxInterceptor(topic s return nil, err } - interceptedDataVerifier, err := ficf.createCacheForInterceptor(topic) + interceptedDataVerifier, err := ficf.interceptedDataVerifierFactory.Create(topic) if err != nil { return nil, err } @@ -604,7 +605,7 @@ func (ficf *fullSyncInterceptorsContainerFactory) createOneUnsignedTxInterceptor return nil, err } - interceptedDataVerifier, err := ficf.createCacheForInterceptor(topic) + interceptedDataVerifier, err := ficf.interceptedDataVerifierFactory.Create(topic) if err != nil { return nil, err } @@ -645,7 +646,7 @@ func (ficf *fullSyncInterceptorsContainerFactory) createOneRewardTxInterceptor(t return nil, err } - interceptedDataVerifier, err := ficf.createCacheForInterceptor(topic) + interceptedDataVerifier, err := ficf.interceptedDataVerifierFactory.Create(topic) if err != nil { return nil, err } @@ -724,7 +725,7 @@ func (ficf *fullSyncInterceptorsContainerFactory) createOneMiniBlocksInterceptor return nil, err } - interceptedDataVerifier, err := ficf.createCacheForInterceptor(topic) + interceptedDataVerifier, err := ficf.interceptedDataVerifierFactory.Create(topic) if err != nil { return nil, err } @@ -769,7 +770,7 @@ func (ficf *fullSyncInterceptorsContainerFactory) generateMetachainHeaderInterce return err } - interceptedDataVerifier, err := ficf.createCacheForInterceptor(identifierHdr) + interceptedDataVerifier, err := ficf.interceptedDataVerifierFactory.Create(identifierHdr) if err != nil { return err } @@ -811,7 +812,7 @@ func (ficf *fullSyncInterceptorsContainerFactory) createOneTrieNodesInterceptor( return nil, err } - interceptedDataVerifier, err := ficf.createCacheForInterceptor(topic) + interceptedDataVerifier, err := ficf.interceptedDataVerifierFactory.Create(topic) if err != nil { return nil, err } @@ -859,7 +860,6 @@ func (ficf *fullSyncInterceptorsContainerFactory) generateRewardTxInterceptors() if err != nil { return err } - keys[int(idx)] = identifierScr interceptorSlice[int(idx)] = interceptor } @@ -880,20 +880,6 @@ func (ficf *fullSyncInterceptorsContainerFactory) addInterceptorsToContainers(ke return ficf.fullArchiveContainer.AddMultiple(keys, interceptors) } -func (ficf *fullSyncInterceptorsContainerFactory) createCacheForInterceptor(topic string) (process.InterceptedDataVerifier, error) { - internalCache, err := cache.NewTimeCacher(cache.ArgTimeCacher{ - DefaultSpan: 30 * time.Second, - CacheExpiry: 30 * time.Second, - }) - if err != nil { - return nil, err - } - - ficf.interceptedDataCache[topic] = internalCache - verifier := interceptors.NewInterceptedDataVerifier(internalCache) - return verifier, nil -} - // IsInterfaceNil returns true if there is no value under the interface func (ficf *fullSyncInterceptorsContainerFactory) IsInterfaceNil() bool { return ficf == nil From b9eb2a5a872612eea2158385fad86586304e7bda Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Thu, 3 Oct 2024 11:07:33 +0300 Subject: [PATCH 35/37] fix unit tests. --- .../metaInterceptorsContainerFactory_test.go | 59 ++++++++++--------- .../shardInterceptorsContainerFactory_test.go | 59 ++++++++++--------- 2 files changed, 60 insertions(+), 58 deletions(-) diff --git a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go index cb09ed78ee2..ce8961eacca 100644 --- a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go +++ b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go @@ -698,34 +698,35 @@ func getArgumentsMeta( cryptoComp *mock.CryptoComponentsMock, ) interceptorscontainer.CommonInterceptorsContainerFactoryArgs { return interceptorscontainer.CommonInterceptorsContainerFactoryArgs{ - CoreComponents: coreComp, - CryptoComponents: cryptoComp, - Accounts: &stateMock.AccountsStub{}, - ShardCoordinator: mock.NewOneShardCoordinatorMock(), - NodesCoordinator: shardingMocks.NewNodesCoordinatorMock(), - MainMessenger: &mock.TopicHandlerStub{}, - FullArchiveMessenger: &mock.TopicHandlerStub{}, - Store: createMetaStore(), - DataPool: createMetaDataPools(), - MaxTxNonceDeltaAllowed: maxTxNonceDeltaAllowed, - TxFeeHandler: &economicsmocks.EconomicsHandlerStub{}, - BlockBlackList: &testscommon.TimeCacheStub{}, - HeaderSigVerifier: &consensus.HeaderSigVerifierMock{}, - HeaderIntegrityVerifier: &mock.HeaderIntegrityVerifierStub{}, - ValidityAttester: &mock.ValidityAttesterStub{}, - EpochStartTrigger: &mock.EpochStartTriggerStub{}, - WhiteListHandler: &testscommon.WhiteListHandlerStub{}, - WhiteListerVerifiedTxs: &testscommon.WhiteListHandlerStub{}, - AntifloodHandler: &mock.P2PAntifloodHandlerStub{}, - ArgumentsParser: &mock.ArgumentParserMock{}, - PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, - RequestHandler: &testscommon.RequestHandlerStub{}, - PeerSignatureHandler: &mock.PeerSignatureHandlerStub{}, - SignaturesHandler: &mock.SignaturesHandlerStub{}, - HeartbeatExpiryTimespanInSec: 30, - MainPeerShardMapper: &p2pmocks.NetworkShardingCollectorStub{}, - FullArchivePeerShardMapper: &p2pmocks.NetworkShardingCollectorStub{}, - HardforkTrigger: &testscommon.HardforkTriggerStub{}, - NodeOperationMode: common.NormalOperation, + CoreComponents: coreComp, + CryptoComponents: cryptoComp, + Accounts: &stateMock.AccountsStub{}, + ShardCoordinator: mock.NewOneShardCoordinatorMock(), + NodesCoordinator: shardingMocks.NewNodesCoordinatorMock(), + MainMessenger: &mock.TopicHandlerStub{}, + FullArchiveMessenger: &mock.TopicHandlerStub{}, + Store: createMetaStore(), + DataPool: createMetaDataPools(), + MaxTxNonceDeltaAllowed: maxTxNonceDeltaAllowed, + TxFeeHandler: &economicsmocks.EconomicsHandlerStub{}, + BlockBlackList: &testscommon.TimeCacheStub{}, + HeaderSigVerifier: &consensus.HeaderSigVerifierMock{}, + HeaderIntegrityVerifier: &mock.HeaderIntegrityVerifierStub{}, + ValidityAttester: &mock.ValidityAttesterStub{}, + EpochStartTrigger: &mock.EpochStartTriggerStub{}, + WhiteListHandler: &testscommon.WhiteListHandlerStub{}, + WhiteListerVerifiedTxs: &testscommon.WhiteListHandlerStub{}, + AntifloodHandler: &mock.P2PAntifloodHandlerStub{}, + ArgumentsParser: &mock.ArgumentParserMock{}, + PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, + RequestHandler: &testscommon.RequestHandlerStub{}, + PeerSignatureHandler: &mock.PeerSignatureHandlerStub{}, + SignaturesHandler: &mock.SignaturesHandlerStub{}, + HeartbeatExpiryTimespanInSec: 30, + MainPeerShardMapper: &p2pmocks.NetworkShardingCollectorStub{}, + FullArchivePeerShardMapper: &p2pmocks.NetworkShardingCollectorStub{}, + HardforkTrigger: &testscommon.HardforkTriggerStub{}, + NodeOperationMode: common.NormalOperation, + InterceptedDataVerifierFactory: &mock.InterceptedDataVerifierFactoryMock{}, } } diff --git a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go index a59b4b965a8..897dafa5b0a 100644 --- a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go +++ b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go @@ -724,34 +724,35 @@ func getArgumentsShard( cryptoComp *mock.CryptoComponentsMock, ) interceptorscontainer.CommonInterceptorsContainerFactoryArgs { return interceptorscontainer.CommonInterceptorsContainerFactoryArgs{ - CoreComponents: coreComp, - CryptoComponents: cryptoComp, - Accounts: &stateMock.AccountsStub{}, - ShardCoordinator: mock.NewOneShardCoordinatorMock(), - NodesCoordinator: shardingMocks.NewNodesCoordinatorMock(), - MainMessenger: &mock.TopicHandlerStub{}, - FullArchiveMessenger: &mock.TopicHandlerStub{}, - Store: createShardStore(), - DataPool: createShardDataPools(), - MaxTxNonceDeltaAllowed: maxTxNonceDeltaAllowed, - TxFeeHandler: &economicsmocks.EconomicsHandlerStub{}, - BlockBlackList: &testscommon.TimeCacheStub{}, - HeaderSigVerifier: &consensus.HeaderSigVerifierMock{}, - HeaderIntegrityVerifier: &mock.HeaderIntegrityVerifierStub{}, - SizeCheckDelta: 0, - ValidityAttester: &mock.ValidityAttesterStub{}, - EpochStartTrigger: &mock.EpochStartTriggerStub{}, - AntifloodHandler: &mock.P2PAntifloodHandlerStub{}, - WhiteListHandler: &testscommon.WhiteListHandlerStub{}, - WhiteListerVerifiedTxs: &testscommon.WhiteListHandlerStub{}, - ArgumentsParser: &mock.ArgumentParserMock{}, - PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, - RequestHandler: &testscommon.RequestHandlerStub{}, - PeerSignatureHandler: &mock.PeerSignatureHandlerStub{}, - SignaturesHandler: &mock.SignaturesHandlerStub{}, - HeartbeatExpiryTimespanInSec: 30, - MainPeerShardMapper: &p2pmocks.NetworkShardingCollectorStub{}, - FullArchivePeerShardMapper: &p2pmocks.NetworkShardingCollectorStub{}, - HardforkTrigger: &testscommon.HardforkTriggerStub{}, + CoreComponents: coreComp, + CryptoComponents: cryptoComp, + Accounts: &stateMock.AccountsStub{}, + ShardCoordinator: mock.NewOneShardCoordinatorMock(), + NodesCoordinator: shardingMocks.NewNodesCoordinatorMock(), + MainMessenger: &mock.TopicHandlerStub{}, + FullArchiveMessenger: &mock.TopicHandlerStub{}, + Store: createShardStore(), + DataPool: createShardDataPools(), + MaxTxNonceDeltaAllowed: maxTxNonceDeltaAllowed, + TxFeeHandler: &economicsmocks.EconomicsHandlerStub{}, + BlockBlackList: &testscommon.TimeCacheStub{}, + HeaderSigVerifier: &consensus.HeaderSigVerifierMock{}, + HeaderIntegrityVerifier: &mock.HeaderIntegrityVerifierStub{}, + SizeCheckDelta: 0, + ValidityAttester: &mock.ValidityAttesterStub{}, + EpochStartTrigger: &mock.EpochStartTriggerStub{}, + AntifloodHandler: &mock.P2PAntifloodHandlerStub{}, + WhiteListHandler: &testscommon.WhiteListHandlerStub{}, + WhiteListerVerifiedTxs: &testscommon.WhiteListHandlerStub{}, + ArgumentsParser: &mock.ArgumentParserMock{}, + PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, + RequestHandler: &testscommon.RequestHandlerStub{}, + PeerSignatureHandler: &mock.PeerSignatureHandlerStub{}, + SignaturesHandler: &mock.SignaturesHandlerStub{}, + HeartbeatExpiryTimespanInSec: 30, + MainPeerShardMapper: &p2pmocks.NetworkShardingCollectorStub{}, + FullArchivePeerShardMapper: &p2pmocks.NetworkShardingCollectorStub{}, + HardforkTrigger: &testscommon.HardforkTriggerStub{}, + InterceptedDataVerifierFactory: &mock.InterceptedDataVerifierFactoryMock{}, } } From 8451f624fc5c74e482d3f682cb66a57e9fb3c480 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Thu, 3 Oct 2024 17:00:21 +0300 Subject: [PATCH 36/37] added close to intercepted data cacher and a few unit tests on nil checkers. --- epochStart/bootstrap/process_test.go | 4 +- epochStart/bootstrap/storageProcess_test.go | 4 +- .../bootstrap/syncEpochStartMeta_test.go | 6 ++ factory/processing/processComponents.go | 15 ++++- factory/processing/processComponents_test.go | 6 -- node/nodeRunner.go | 61 ++++++++----------- .../metaInterceptorsContainerFactory_test.go | 28 +++++---- .../shardInterceptorsContainerFactory_test.go | 28 +++++---- .../factory/interceptedDataVerifierFactory.go | 12 ++++ process/interface.go | 1 + .../interceptedDataVerifierFactoryMock.go | 5 ++ testscommon/components/components.go | 2 - testscommon/generalConfig.go | 4 ++ 13 files changed, 101 insertions(+), 75 deletions(-) diff --git a/epochStart/bootstrap/process_test.go b/epochStart/bootstrap/process_test.go index dfc2c42411a..67d6a9d1295 100644 --- a/epochStart/bootstrap/process_test.go +++ b/epochStart/bootstrap/process_test.go @@ -996,9 +996,7 @@ func TestCreateSyncers(t *testing.T) { epochStartProvider.whiteListerVerifiedTxs = &testscommon.WhiteListHandlerStub{} epochStartProvider.requestHandler = &testscommon.RequestHandlerStub{} epochStartProvider.storageService = &storageMocks.ChainStorerStub{} - epochStartProvider.interceptedDataVerifierFactory = &processMock.InterceptedDataVerifierFactoryMock{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { - return &processMock.InterceptedDataVerifierMock{}, nil - }} + epochStartProvider.interceptedDataVerifierFactory = &processMock.InterceptedDataVerifierFactoryMock{} err := epochStartProvider.createSyncers() assert.Nil(t, err) diff --git a/epochStart/bootstrap/storageProcess_test.go b/epochStart/bootstrap/storageProcess_test.go index 16a3b506cb4..64708040acd 100644 --- a/epochStart/bootstrap/storageProcess_test.go +++ b/epochStart/bootstrap/storageProcess_test.go @@ -129,9 +129,7 @@ func TestStorageEpochStartBootstrap_BootstrapMetablockNotFound(t *testing.T) { } args.GeneralConfig = testscommon.GetGeneralConfig() args.GeneralConfig.EpochStartConfig.RoundsPerEpoch = roundsPerEpoch - args.InterceptedDataVerifierFactory = &processMock.InterceptedDataVerifierFactoryMock{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { - return &processMock.InterceptedDataVerifierMock{}, nil - }} + args.InterceptedDataVerifierFactory = &processMock.InterceptedDataVerifierFactoryMock{} sesb, _ := NewStorageEpochStartBootstrap(args) params, err := sesb.Bootstrap() diff --git a/epochStart/bootstrap/syncEpochStartMeta_test.go b/epochStart/bootstrap/syncEpochStartMeta_test.go index 4cf8babeb2d..ac05d2ba977 100644 --- a/epochStart/bootstrap/syncEpochStartMeta_test.go +++ b/epochStart/bootstrap/syncEpochStartMeta_test.go @@ -50,6 +50,12 @@ func TestNewEpochStartMetaSyncer_NilsShouldError(t *testing.T) { ess, err = NewEpochStartMetaSyncer(args) assert.True(t, check.IfNil(ess)) assert.Equal(t, epochStart.ErrNilMetablockProcessor, err) + + args = getEpochStartSyncerArgs() + args.InterceptedDataVerifierFactory = nil + ess, err = NewEpochStartMetaSyncer(args) + assert.True(t, check.IfNil(ess)) + assert.Equal(t, epochStart.ErrNilInterceptedDataVerifierFactory, err) } func TestNewEpochStartMetaSyncer_ShouldWork(t *testing.T) { diff --git a/factory/processing/processComponents.go b/factory/processing/processComponents.go index fbab1357680..ce7da0e7006 100644 --- a/factory/processing/processComponents.go +++ b/factory/processing/processComponents.go @@ -57,6 +57,7 @@ import ( "github.com/multiversx/mx-chain-go/process/factory/interceptorscontainer" "github.com/multiversx/mx-chain-go/process/headerCheck" "github.com/multiversx/mx-chain-go/process/heartbeat/validator" + interceptorFactory "github.com/multiversx/mx-chain-go/process/interceptors/factory" "github.com/multiversx/mx-chain-go/process/peer" "github.com/multiversx/mx-chain-go/process/receipts" "github.com/multiversx/mx-chain-go/process/smartContract" @@ -133,6 +134,7 @@ type processComponents struct { receiptsRepository mainFactory.ReceiptsRepository sentSignaturesTracker process.SentSignaturesTracker epochSystemSCProcessor process.EpochStartSystemSCProcessor + interceptedDataVerifierFactory process.InterceptedDataVerifierFactory } // ProcessComponentsFactoryArgs holds the arguments needed to create a process components factory @@ -168,8 +170,6 @@ type ProcessComponentsFactoryArgs struct { GenesisNonce uint64 GenesisRound uint64 - - InterceptedDataVerifierFactory process.InterceptedDataVerifierFactory } type processComponentsFactory struct { @@ -221,6 +221,11 @@ func NewProcessComponentsFactory(args ProcessComponentsFactoryArgs) (*processCom return nil, err } + interceptedDataVerifierFactory := interceptorFactory.NewInterceptedDataVerifierFactory(interceptorFactory.InterceptedDataVerifierFactoryArgs{ + CacheSpan: time.Duration(args.Config.InterceptedDataVerifier.CacheSpanInSec) * time.Second, + CacheExpiry: time.Duration(args.Config.InterceptedDataVerifier.CacheExpiryInSec) * time.Second, + }) + return &processComponentsFactory{ config: args.Config, epochConfig: args.EpochConfig, @@ -252,7 +257,7 @@ func NewProcessComponentsFactory(args ProcessComponentsFactoryArgs) (*processCom genesisNonce: args.GenesisNonce, genesisRound: args.GenesisRound, roundConfig: args.RoundConfig, - interceptedDataVerifierFactory: args.InterceptedDataVerifierFactory, + interceptedDataVerifierFactory: interceptedDataVerifierFactory, }, nil } @@ -769,6 +774,7 @@ func (pcf *processComponentsFactory) Create() (*processComponents, error) { accountsParser: pcf.accountsParser, receiptsRepository: receiptsRepository, sentSignaturesTracker: sentSignaturesTracker, + interceptedDataVerifierFactory: pcf.interceptedDataVerifierFactory, }, nil } @@ -2055,6 +2061,9 @@ func (pc *processComponents) Close() error { if !check.IfNil(pc.txsSender) { log.LogIfError(pc.txsSender.Close()) } + if !check.IfNil(pc.interceptedDataVerifierFactory) { + log.LogIfError(pc.interceptedDataVerifierFactory.Close()) + } return nil } diff --git a/factory/processing/processComponents_test.go b/factory/processing/processComponents_test.go index 606b8470edb..6ddf5ea2d8b 100644 --- a/factory/processing/processComponents_test.go +++ b/factory/processing/processComponents_test.go @@ -31,7 +31,6 @@ import ( testsMocks "github.com/multiversx/mx-chain-go/integrationTests/mock" "github.com/multiversx/mx-chain-go/p2p" "github.com/multiversx/mx-chain-go/process" - processMocks "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/sharding" "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator" "github.com/multiversx/mx-chain-go/state" @@ -267,11 +266,6 @@ func createMockProcessComponentsFactoryArgs() processComp.ProcessComponentsFacto } args.State = components.GetStateComponents(args.CoreData, args.StatusCoreComponents) - args.InterceptedDataVerifierFactory = &processMocks.InterceptedDataVerifierFactoryMock{ - CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { - return &processMocks.InterceptedDataVerifierMock{}, nil - }, - } return args } diff --git a/node/nodeRunner.go b/node/nodeRunner.go index 6a2490cd683..f6fa53a660e 100644 --- a/node/nodeRunner.go +++ b/node/nodeRunner.go @@ -56,7 +56,6 @@ import ( "github.com/multiversx/mx-chain-go/outport" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/interceptors" - "github.com/multiversx/mx-chain-go/process/interceptors/factory" "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator" "github.com/multiversx/mx-chain-go/state/syncer" "github.com/multiversx/mx-chain-go/storage/cache" @@ -1238,40 +1237,34 @@ func (nr *nodeRunner) CreateManagedProcessComponents( txExecutionOrderHandler := ordering.NewOrderedCollection() - interceptedDataVerifierFactory := factory.NewInterceptedDataVerifierFactory(factory.InterceptedDataVerifierFactoryArgs{ - CacheSpan: time.Duration(nr.configs.GeneralConfig.InterceptedDataVerifier.CacheExpiryInSec), - CacheExpiry: time.Duration(nr.configs.GeneralConfig.InterceptedDataVerifier.CacheExpiryInSec), - }) - processArgs := processComp.ProcessComponentsFactoryArgs{ - Config: *configs.GeneralConfig, - EpochConfig: *configs.EpochConfig, - RoundConfig: *configs.RoundConfig, - PrefConfigs: *configs.PreferencesConfig, - ImportDBConfig: *configs.ImportDbConfig, - EconomicsConfig: *configs.EconomicsConfig, - AccountsParser: accountsParser, - SmartContractParser: smartContractParser, - GasSchedule: gasScheduleNotifier, - NodesCoordinator: nodesCoordinator, - Data: dataComponents, - CoreData: coreComponents, - Crypto: cryptoComponents, - State: stateComponents, - Network: networkComponents, - BootstrapComponents: bootstrapComponents, - StatusComponents: statusComponents, - StatusCoreComponents: statusCoreComponents, - RequestedItemsHandler: requestedItemsHandler, - WhiteListHandler: whiteListRequest, - WhiteListerVerifiedTxs: whiteListerVerifiedTxs, - MaxRating: configs.RatingsConfig.General.MaxRating, - SystemSCConfig: configs.SystemSCConfig, - ImportStartHandler: importStartHandler, - HistoryRepo: historyRepository, - FlagsConfig: *configs.FlagsConfig, - TxExecutionOrderHandler: txExecutionOrderHandler, - InterceptedDataVerifierFactory: interceptedDataVerifierFactory, + Config: *configs.GeneralConfig, + EpochConfig: *configs.EpochConfig, + RoundConfig: *configs.RoundConfig, + PrefConfigs: *configs.PreferencesConfig, + ImportDBConfig: *configs.ImportDbConfig, + EconomicsConfig: *configs.EconomicsConfig, + AccountsParser: accountsParser, + SmartContractParser: smartContractParser, + GasSchedule: gasScheduleNotifier, + NodesCoordinator: nodesCoordinator, + Data: dataComponents, + CoreData: coreComponents, + Crypto: cryptoComponents, + State: stateComponents, + Network: networkComponents, + BootstrapComponents: bootstrapComponents, + StatusComponents: statusComponents, + StatusCoreComponents: statusCoreComponents, + RequestedItemsHandler: requestedItemsHandler, + WhiteListHandler: whiteListRequest, + WhiteListerVerifiedTxs: whiteListerVerifiedTxs, + MaxRating: configs.RatingsConfig.General.MaxRating, + SystemSCConfig: configs.SystemSCConfig, + ImportStartHandler: importStartHandler, + HistoryRepo: historyRepository, + FlagsConfig: *configs.FlagsConfig, + TxExecutionOrderHandler: txExecutionOrderHandler, } processComponentsFactory, err := processComp.NewProcessComponentsFactory(processArgs) if err != nil { diff --git a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go index ce8961eacca..927742f00fa 100644 --- a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go +++ b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go @@ -400,6 +400,18 @@ func TestNewMetaInterceptorsContainerFactory_NilPeerSignatureHandler(t *testing. assert.Equal(t, process.ErrNilPeerSignatureHandler, err) } +func TestNewMetaInterceptorsContainerFactory_NilInterceptedDataVerifierFactory(t *testing.T) { + t.Parallel() + + coreComp, cryptoComp := createMockComponentHolders() + args := getArgumentsShard(coreComp, cryptoComp) + args.InterceptedDataVerifierFactory = nil + icf, err := interceptorscontainer.NewMetaInterceptorsContainerFactory(args) + + assert.Nil(t, icf) + assert.Equal(t, process.ErrNilInterceptedDataVerifierFactory, err) +} + func TestNewMetaInterceptorsContainerFactory_InvalidExpiryTimespan(t *testing.T) { t.Parallel() @@ -546,9 +558,7 @@ func testCreateMetaTopicShouldFail(matchStrToErrOnCreate string, matchStrToErrOn } else { args.MainMessenger = createMetaStubTopicHandler(matchStrToErrOnCreate, matchStrToErrOnRegister) } - args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryMock{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { - return &mock.InterceptedDataVerifierMock{}, nil - }} + args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryMock{} icf, _ := interceptorscontainer.NewMetaInterceptorsContainerFactory(args) mainContainer, fullArchiveConatiner, err := icf.Create() @@ -564,9 +574,7 @@ func TestMetaInterceptorsContainerFactory_CreateShouldWork(t *testing.T) { coreComp, cryptoComp := createMockComponentHolders() args := getArgumentsMeta(coreComp, cryptoComp) - args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryMock{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { - return &mock.InterceptedDataVerifierMock{}, nil - }} + args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryMock{} icf, _ := interceptorscontainer.NewMetaInterceptorsContainerFactory(args) mainContainer, fullArchiveContainer, err := icf.Create() @@ -599,9 +607,7 @@ func TestMetaInterceptorsContainerFactory_With4ShardsShouldWork(t *testing.T) { args := getArgumentsMeta(coreComp, cryptoComp) args.ShardCoordinator = shardCoordinator args.NodesCoordinator = nodesCoordinator - args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryMock{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { - return &mock.InterceptedDataVerifierMock{}, nil - }} + args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryMock{} icf, err := interceptorscontainer.NewMetaInterceptorsContainerFactory(args) require.Nil(t, err) @@ -653,9 +659,7 @@ func TestMetaInterceptorsContainerFactory_With4ShardsShouldWork(t *testing.T) { args.NodeOperationMode = common.FullArchiveMode args.ShardCoordinator = shardCoordinator args.NodesCoordinator = nodesCoordinator - args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryMock{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { - return &mock.InterceptedDataVerifierMock{}, nil - }} + args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryMock{} icf, err := interceptorscontainer.NewMetaInterceptorsContainerFactory(args) require.Nil(t, err) diff --git a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go index 897dafa5b0a..40dc922f5b0 100644 --- a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go +++ b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go @@ -356,6 +356,18 @@ func TestNewShardInterceptorsContainerFactory_NilValidityAttesterShouldErr(t *te assert.Equal(t, process.ErrNilValidityAttester, err) } +func TestNewShardInterceptorsContainerFactory_NilInterceptedDataVerifierFactory(t *testing.T) { + t.Parallel() + + coreComp, cryptoComp := createMockComponentHolders() + args := getArgumentsShard(coreComp, cryptoComp) + args.InterceptedDataVerifierFactory = nil + icf, err := interceptorscontainer.NewShardInterceptorsContainerFactory(args) + + assert.Nil(t, icf) + assert.Equal(t, process.ErrNilInterceptedDataVerifierFactory, err) +} + func TestNewShardInterceptorsContainerFactory_InvalidChainIDShouldErr(t *testing.T) { t.Parallel() @@ -497,9 +509,7 @@ func testCreateShardTopicShouldFail(matchStrToErrOnCreate string, matchStrToErrO coreComp, cryptoComp := createMockComponentHolders() args := getArgumentsShard(coreComp, cryptoComp) - args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryMock{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { - return &mock.InterceptedDataVerifierMock{}, nil - }} + args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryMock{} if strings.Contains(t.Name(), "full_archive") { args.NodeOperationMode = common.FullArchiveMode args.FullArchiveMessenger = createShardStubTopicHandler(matchStrToErrOnCreate, matchStrToErrOnRegister) @@ -566,9 +576,7 @@ func TestShardInterceptorsContainerFactory_CreateShouldWork(t *testing.T) { }, } args.WhiteListerVerifiedTxs = &testscommon.WhiteListHandlerStub{} - args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryMock{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { - return &mock.InterceptedDataVerifierMock{}, nil - }} + args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryMock{} icf, _ := interceptorscontainer.NewShardInterceptorsContainerFactory(args) @@ -604,9 +612,7 @@ func TestShardInterceptorsContainerFactory_With4ShardsShouldWork(t *testing.T) { args.ShardCoordinator = shardCoordinator args.NodesCoordinator = nodesCoordinator args.PreferredPeersHolder = &p2pmocks.PeersHolderStub{} - args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryMock{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { - return &mock.InterceptedDataVerifierMock{}, nil - }} + args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryMock{} icf, _ := interceptorscontainer.NewShardInterceptorsContainerFactory(args) @@ -657,9 +663,7 @@ func TestShardInterceptorsContainerFactory_With4ShardsShouldWork(t *testing.T) { args.ShardCoordinator = shardCoordinator args.NodesCoordinator = nodesCoordinator args.PreferredPeersHolder = &p2pmocks.PeersHolderStub{} - args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryMock{CreateCalled: func(topic string) (process.InterceptedDataVerifier, error) { - return &mock.InterceptedDataVerifierMock{}, nil - }} + args.InterceptedDataVerifierFactory = &mock.InterceptedDataVerifierFactoryMock{} icf, _ := interceptorscontainer.NewShardInterceptorsContainerFactory(args) diff --git a/process/interceptors/factory/interceptedDataVerifierFactory.go b/process/interceptors/factory/interceptedDataVerifierFactory.go index db50dfebd92..4253c9bf20c 100644 --- a/process/interceptors/factory/interceptedDataVerifierFactory.go +++ b/process/interceptors/factory/interceptedDataVerifierFactory.go @@ -1,6 +1,7 @@ package factory import ( + "fmt" "sync" "time" @@ -53,6 +54,17 @@ func (idvf *interceptedDataVerifierFactory) Create(topic string) (process.Interc return interceptors.NewInterceptedDataVerifier(internalCache) } +func (idvf *interceptedDataVerifierFactory) Close() error { + for topic, cacher := range idvf.interceptedDataVerifierMap { + err := cacher.Close() + if err != nil { + return fmt.Errorf("failed to close cacher on topic %q: %w", topic, err) + } + } + + return nil +} + // IsInterfaceNil returns true if there is no value under the interface func (idvf *interceptedDataVerifierFactory) IsInterfaceNil() bool { return idvf == nil diff --git a/process/interface.go b/process/interface.go index 117c8376f0c..99693655a83 100644 --- a/process/interface.go +++ b/process/interface.go @@ -1411,5 +1411,6 @@ type InterceptedDataVerifier interface { // InterceptedDataVerifierFactory defines a component that is able to create intercepted data verifiers type InterceptedDataVerifierFactory interface { Create(topic string) (InterceptedDataVerifier, error) + Close() error IsInterfaceNil() bool } diff --git a/process/mock/interceptedDataVerifierFactoryMock.go b/process/mock/interceptedDataVerifierFactoryMock.go index 2b17d849563..245be014b15 100644 --- a/process/mock/interceptedDataVerifierFactoryMock.go +++ b/process/mock/interceptedDataVerifierFactoryMock.go @@ -18,6 +18,11 @@ func (idvfs *InterceptedDataVerifierFactoryMock) Create(topic string) (process.I return &InterceptedDataVerifierMock{}, nil } +// Close - +func (idvfs *InterceptedDataVerifierFactoryMock) Close() error { + return nil +} + // IsInterfaceNil - func (idvfs *InterceptedDataVerifierFactoryMock) IsInterfaceNil() bool { return idvfs == nil diff --git a/testscommon/components/components.go b/testscommon/components/components.go index daab8391b39..6e630b9050d 100644 --- a/testscommon/components/components.go +++ b/testscommon/components/components.go @@ -35,7 +35,6 @@ import ( "github.com/multiversx/mx-chain-go/p2p" p2pConfig "github.com/multiversx/mx-chain-go/p2p/config" p2pFactory "github.com/multiversx/mx-chain-go/p2p/factory" - processMock "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/sharding" "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator" "github.com/multiversx/mx-chain-go/state" @@ -609,7 +608,6 @@ func GetProcessArgs( }, }, }, - InterceptedDataVerifierFactory: &processMock.InterceptedDataVerifierFactoryMock{}, } } diff --git a/testscommon/generalConfig.go b/testscommon/generalConfig.go index 515c64518b4..f5777cfae6b 100644 --- a/testscommon/generalConfig.go +++ b/testscommon/generalConfig.go @@ -441,6 +441,10 @@ func GetGeneralConfig() config.Config { ResourceStats: config.ResourceStatsConfig{ RefreshIntervalInSec: 1, }, + InterceptedDataVerifier: config.InterceptedDataVerifierConfig{ + CacheSpanInSec: 1, + CacheExpiryInSec: 1, + }, } } From 8dd18b880e952541935a9e758cd9edf9e895d508 Mon Sep 17 00:00:00 2001 From: Alexander Cristurean Date: Thu, 3 Oct 2024 17:23:58 +0300 Subject: [PATCH 37/37] cosmetic changes. --- process/interceptors/factory/interceptedDataVerifierFactory.go | 1 + 1 file changed, 1 insertion(+) diff --git a/process/interceptors/factory/interceptedDataVerifierFactory.go b/process/interceptors/factory/interceptedDataVerifierFactory.go index 4253c9bf20c..2775bbdc61a 100644 --- a/process/interceptors/factory/interceptedDataVerifierFactory.go +++ b/process/interceptors/factory/interceptedDataVerifierFactory.go @@ -54,6 +54,7 @@ func (idvf *interceptedDataVerifierFactory) Create(topic string) (process.Interc return interceptors.NewInterceptedDataVerifier(internalCache) } +// Close will close all the sweeping routines created by the cache. func (idvf *interceptedDataVerifierFactory) Close() error { for topic, cacher := range idvf.interceptedDataVerifierMap { err := cacher.Close()