Message validation optimization

cmd/node/config/config.toml

@@ -958,3 +958,7 @@
         # All validators will broadcast the message right away
         { EndIndex = 0, DelayInMilliseconds = 0 },
     ]
+
+[InterceptedDataVerifier]
+    CacheSpanInSec = 30
+    CacheExpiryInSec = 30

config/config.go

@@ -229,6 +229,8 @@ type Config struct {
 	PoolsCleanersConfig       PoolsCleanersConfig
 	Redundancy                RedundancyConfig
 	ConsensusGradualBroadcast ConsensusGradualBroadcastConfig
+
+	InterceptedDataVerifier InterceptedDataVerifierConfig
 }
 
 // PeersRatingConfig will hold settings related to peers rating
@@ -679,3 +681,9 @@ type IndexBroadcastDelay struct {
 type ConsensusGradualBroadcastConfig struct {
 	GradualIndexBroadcastDelay []IndexBroadcastDelay
 }
+
+// InterceptedDataVerifierConfig holds the configuration for the intercepted data verifier
+type InterceptedDataVerifierConfig struct {
+	CacheSpanInSec   uint64
+	CacheExpiryInSec uint64
+}

epochStart/bootstrap/factory/epochStartInterceptorsContainerFactory.go

@@ -6,6 +6,7 @@ import (
 	"github.com/multiversx/mx-chain-core-go/core"
 	"github.com/multiversx/mx-chain-core-go/core/check"
 	"github.com/multiversx/mx-chain-core-go/data/typeConverters"
+
 	"github.com/multiversx/mx-chain-go/common"
 	"github.com/multiversx/mx-chain-go/config"
 	"github.com/multiversx/mx-chain-go/dataRetriever"
@@ -25,23 +26,24 @@ const timeSpanForBadHeaders = time.Minute
 // ArgsEpochStartInterceptorContainer holds the arguments needed for creating a new epoch start interceptors
 // container factory
 type ArgsEpochStartInterceptorContainer struct {
-	CoreComponents          process.CoreComponentsHolder
-	CryptoComponents        process.CryptoComponentsHolder
-	Config                  config.Config
-	ShardCoordinator        sharding.Coordinator
-	MainMessenger           process.TopicHandler
-	FullArchiveMessenger    process.TopicHandler
-	DataPool                dataRetriever.PoolsHolder
-	WhiteListHandler        update.WhiteListHandler
-	WhiteListerVerifiedTxs  update.WhiteListHandler
-	AddressPubkeyConv       core.PubkeyConverter
-	NonceConverter          typeConverters.Uint64ByteSliceConverter
-	ChainID                 []byte
-	ArgumentsParser         process.ArgumentsParser
-	HeaderIntegrityVerifier process.HeaderIntegrityVerifier
-	RequestHandler          process.RequestHandler
-	SignaturesHandler       process.SignaturesHandler
-	NodeOperationMode       common.NodeOperation
+	CoreComponents                 process.CoreComponentsHolder
+	CryptoComponents               process.CryptoComponentsHolder
+	Config                         config.Config
+	ShardCoordinator               sharding.Coordinator
+	MainMessenger                  process.TopicHandler
+	FullArchiveMessenger           process.TopicHandler
+	DataPool                       dataRetriever.PoolsHolder
+	WhiteListHandler               update.WhiteListHandler
+	WhiteListerVerifiedTxs         update.WhiteListHandler
+	AddressPubkeyConv              core.PubkeyConverter
+	NonceConverter                 typeConverters.Uint64ByteSliceConverter
+	ChainID                        []byte
+	ArgumentsParser                process.ArgumentsParser
+	HeaderIntegrityVerifier        process.HeaderIntegrityVerifier
+	RequestHandler                 process.RequestHandler
+	SignaturesHandler              process.SignaturesHandler
+	NodeOperationMode              common.NodeOperation
+	InterceptedDataVerifierFactory process.InterceptedDataVerifierFactory
 }
 
 // NewEpochStartInterceptorsContainer will return a real interceptors container factory, but with many disabled components
@@ -78,36 +80,37 @@ func NewEpochStartInterceptorsContainer(args ArgsEpochStartInterceptorContainer)
 	hardforkTrigger := disabledFactory.HardforkTrigger()
 
 	containerFactoryArgs := interceptorscontainer.CommonInterceptorsContainerFactoryArgs{
-		CoreComponents:               args.CoreComponents,
-		CryptoComponents:             cryptoComponents,
-		Accounts:                     accountsAdapter,
-		ShardCoordinator:             args.ShardCoordinator,
-		NodesCoordinator:             nodesCoordinator,
-		MainMessenger:                args.MainMessenger,
-		FullArchiveMessenger:         args.FullArchiveMessenger,
-		Store:                        storer,
-		DataPool:                     args.DataPool,
-		MaxTxNonceDeltaAllowed:       common.MaxTxNonceDeltaAllowed,
-		TxFeeHandler:                 feeHandler,
-		BlockBlackList:               blackListHandler,
-		HeaderSigVerifier:            headerSigVerifier,
-		HeaderIntegrityVerifier:      args.HeaderIntegrityVerifier,
-		ValidityAttester:             validityAttester,
-		EpochStartTrigger:            epochStartTrigger,
-		WhiteListHandler:             args.WhiteListHandler,
-		WhiteListerVerifiedTxs:       args.WhiteListerVerifiedTxs,
-		AntifloodHandler:             antiFloodHandler,
-		ArgumentsParser:              args.ArgumentsParser,
-		PreferredPeersHolder:         disabled.NewPreferredPeersHolder(),
-		SizeCheckDelta:               uint32(sizeCheckDelta),
-		RequestHandler:               args.RequestHandler,
-		PeerSignatureHandler:         cryptoComponents.PeerSignatureHandler(),
-		SignaturesHandler:            args.SignaturesHandler,
-		HeartbeatExpiryTimespanInSec: args.Config.HeartbeatV2.HeartbeatExpiryTimespanInSec,
-		MainPeerShardMapper:          peerShardMapper,
-		FullArchivePeerShardMapper:   fullArchivePeerShardMapper,
-		HardforkTrigger:              hardforkTrigger,
-		NodeOperationMode:            args.NodeOperationMode,
+		CoreComponents:                 args.CoreComponents,
+		CryptoComponents:               cryptoComponents,
+		Accounts:                       accountsAdapter,
+		ShardCoordinator:               args.ShardCoordinator,
+		NodesCoordinator:               nodesCoordinator,
+		MainMessenger:                  args.MainMessenger,
+		FullArchiveMessenger:           args.FullArchiveMessenger,
+		Store:                          storer,
+		DataPool:                       args.DataPool,
+		MaxTxNonceDeltaAllowed:         common.MaxTxNonceDeltaAllowed,
+		TxFeeHandler:                   feeHandler,
+		BlockBlackList:                 blackListHandler,
+		HeaderSigVerifier:              headerSigVerifier,
+		HeaderIntegrityVerifier:        args.HeaderIntegrityVerifier,
+		ValidityAttester:               validityAttester,
+		EpochStartTrigger:              epochStartTrigger,
+		WhiteListHandler:               args.WhiteListHandler,
+		WhiteListerVerifiedTxs:         args.WhiteListerVerifiedTxs,
+		AntifloodHandler:               antiFloodHandler,
+		ArgumentsParser:                args.ArgumentsParser,
+		PreferredPeersHolder:           disabled.NewPreferredPeersHolder(),
+		SizeCheckDelta:                 uint32(sizeCheckDelta),
+		RequestHandler:                 args.RequestHandler,
+		PeerSignatureHandler:           cryptoComponents.PeerSignatureHandler(),
+		SignaturesHandler:              args.SignaturesHandler,
+		HeartbeatExpiryTimespanInSec:   args.Config.HeartbeatV2.HeartbeatExpiryTimespanInSec,
+		MainPeerShardMapper:            peerShardMapper,
+		FullArchivePeerShardMapper:     fullArchivePeerShardMapper,
+		HardforkTrigger:                hardforkTrigger,
+		NodeOperationMode:              args.NodeOperationMode,
+		InterceptedDataVerifierFactory: args.InterceptedDataVerifierFactory,
 	}
 
 	interceptorsContainerFactory, err := interceptorscontainer.NewMetaInterceptorsContainerFactory(containerFactoryArgs)

epochStart/bootstrap/process.go

@@ -14,6 +14,8 @@ import (
 	"github.com/multiversx/mx-chain-core-go/data"
 	"github.com/multiversx/mx-chain-core-go/data/block"
 	"github.com/multiversx/mx-chain-core-go/data/typeConverters/uint64ByteSlice"
+	logger "github.com/multiversx/mx-chain-logger-go"
+
 	"github.com/multiversx/mx-chain-go/common"
 	disabledCommon "github.com/multiversx/mx-chain-go/common/disabled"
 	"github.com/multiversx/mx-chain-go/common/ordering"
@@ -52,7 +54,6 @@ import (
 	"github.com/multiversx/mx-chain-go/trie/storageMarker"
 	"github.com/multiversx/mx-chain-go/update"
 	updateSync "github.com/multiversx/mx-chain-go/update/sync"
-	logger "github.com/multiversx/mx-chain-logger-go"
 )
 
 var log = logger.GetOrCreate("epochStart/bootstrap")
@@ -152,6 +153,8 @@ type epochStartBootstrap struct {
 	nodeType           core.NodeType
 	startEpoch         uint32
 	shuffledOut        bool
+
+	interceptedDataVerifierFactory process.InterceptedDataVerifierFactory
 }
 
 type baseDataInStorage struct {
@@ -190,6 +193,7 @@ type ArgsEpochStartBootstrap struct {
 	NodeProcessingMode              common.NodeProcessingMode
 	StateStatsHandler               common.StateStatisticsHandler
 	NodesCoordinatorRegistryFactory nodesCoordinator.NodesCoordinatorRegistryFactory
+	InterceptedDataVerifierFactory  process.InterceptedDataVerifierFactory
 }
 
 type dataToSync struct {
@@ -242,6 +246,7 @@ func NewEpochStartBootstrap(args ArgsEpochStartBootstrap) (*epochStartBootstrap,
 		stateStatsHandler:               args.StateStatsHandler,
 		startEpoch:                      args.GeneralConfig.EpochStartConfig.GenesisEpoch,
 		nodesCoordinatorRegistryFactory: args.NodesCoordinatorRegistryFactory,
+		interceptedDataVerifierFactory:  args.InterceptedDataVerifierFactory,
 	}
 
 	if epochStartProvider.prefsConfig.FullArchive {
@@ -553,16 +558,17 @@ func (e *epochStartBootstrap) prepareComponentsToSyncFromNetwork() error {
 	}
 
 	argsEpochStartSyncer := ArgsNewEpochStartMetaSyncer{
-		CoreComponentsHolder:    e.coreComponentsHolder,
-		CryptoComponentsHolder:  e.cryptoComponentsHolder,
-		RequestHandler:          e.requestHandler,
-		Messenger:               e.mainMessenger,
-		ShardCoordinator:        e.shardCoordinator,
-		EconomicsData:           e.economicsData,
-		WhitelistHandler:        e.whiteListHandler,
-		StartInEpochConfig:      epochStartConfig,
-		HeaderIntegrityVerifier: e.headerIntegrityVerifier,
-		MetaBlockProcessor:      metaBlockProcessor,
+		CoreComponentsHolder:           e.coreComponentsHolder,
+		CryptoComponentsHolder:         e.cryptoComponentsHolder,
+		RequestHandler:                 e.requestHandler,
+		Messenger:                      e.mainMessenger,
+		ShardCoordinator:               e.shardCoordinator,
+		EconomicsData:                  e.economicsData,
+		WhitelistHandler:               e.whiteListHandler,
+		StartInEpochConfig:             epochStartConfig,
+		HeaderIntegrityVerifier:        e.headerIntegrityVerifier,
+		MetaBlockProcessor:             metaBlockProcessor,
+		InterceptedDataVerifierFactory: e.interceptedDataVerifierFactory,
 	}
 	e.epochStartMetaBlockSyncer, err = NewEpochStartMetaSyncer(argsEpochStartSyncer)
 	if err != nil {
@@ -575,20 +581,21 @@ func (e *epochStartBootstrap) prepareComponentsToSyncFromNetwork() error {
 func (e *epochStartBootstrap) createSyncers() error {
 	var err error
 	args := factoryInterceptors.ArgsEpochStartInterceptorContainer{
-		CoreComponents:          e.coreComponentsHolder,
-		CryptoComponents:        e.cryptoComponentsHolder,
-		Config:                  e.generalConfig,
-		ShardCoordinator:        e.shardCoordinator,
-		MainMessenger:           e.mainMessenger,
-		FullArchiveMessenger:    e.fullArchiveMessenger,
-		DataPool:                e.dataPool,
-		WhiteListHandler:        e.whiteListHandler,
-		WhiteListerVerifiedTxs:  e.whiteListerVerifiedTxs,
-		ArgumentsParser:         e.argumentsParser,
-		HeaderIntegrityVerifier: e.headerIntegrityVerifier,
-		RequestHandler:          e.requestHandler,
-		SignaturesHandler:       e.mainMessenger,
-		NodeOperationMode:       e.nodeOperationMode,
+		CoreComponents:                 e.coreComponentsHolder,
+		CryptoComponents:               e.cryptoComponentsHolder,
+		Config:                         e.generalConfig,
+		ShardCoordinator:               e.shardCoordinator,
+		MainMessenger:                  e.mainMessenger,
+		FullArchiveMessenger:           e.fullArchiveMessenger,
+		DataPool:                       e.dataPool,
+		WhiteListHandler:               e.whiteListHandler,
+		WhiteListerVerifiedTxs:         e.whiteListerVerifiedTxs,
+		ArgumentsParser:                e.argumentsParser,
+		HeaderIntegrityVerifier:        e.headerIntegrityVerifier,
+		RequestHandler:                 e.requestHandler,
+		SignaturesHandler:              e.mainMessenger,
+		NodeOperationMode:              e.nodeOperationMode,
+		InterceptedDataVerifierFactory: e.interceptedDataVerifierFactory,
 	}
 
 	e.mainInterceptorContainer, e.fullArchiveInterceptorContainer, err = factoryInterceptors.NewEpochStartInterceptorsContainer(args)

epochStart/bootstrap/process_test.go

@@ -32,6 +32,7 @@ import (
 	"github.com/multiversx/mx-chain-go/epochStart/bootstrap/types"
 	"github.com/multiversx/mx-chain-go/epochStart/mock"
 	"github.com/multiversx/mx-chain-go/process"
+	processMock "github.com/multiversx/mx-chain-go/process/mock"
 	"github.com/multiversx/mx-chain-go/sharding"
 	"github.com/multiversx/mx-chain-go/sharding/nodesCoordinator"
 	"github.com/multiversx/mx-chain-go/state"
@@ -251,8 +252,9 @@ func createMockEpochStartBootstrapArgs(
 		FlagsConfig: config.ContextFlagsConfig{
 			ForceStartFromNetwork: false,
 		},
-		TrieSyncStatisticsProvider: &testscommon.SizeSyncStatisticsHandlerStub{},
-		StateStatsHandler:          disabledStatistics.NewStateStatistics(),
+		TrieSyncStatisticsProvider:     &testscommon.SizeSyncStatisticsHandlerStub{},
+		StateStatsHandler:              disabledStatistics.NewStateStatistics(),
+		InterceptedDataVerifierFactory: &processMock.InterceptedDataVerifierFactoryMock{},
 	}
 }
 
@@ -994,6 +996,7 @@ func TestCreateSyncers(t *testing.T) {
 	epochStartProvider.whiteListerVerifiedTxs = &testscommon.WhiteListHandlerStub{}
 	epochStartProvider.requestHandler = &testscommon.RequestHandlerStub{}
 	epochStartProvider.storageService = &storageMocks.ChainStorerStub{}
+	epochStartProvider.interceptedDataVerifierFactory = &processMock.InterceptedDataVerifierFactoryMock{}
 
 	err := epochStartProvider.createSyncers()
 	assert.Nil(t, err)

epochStart/bootstrap/storageProcess.go

@@ -11,6 +11,7 @@ import (
 	"github.com/multiversx/mx-chain-core-go/data"
 	"github.com/multiversx/mx-chain-core-go/data/block"
 	"github.com/multiversx/mx-chain-core-go/data/endProcess"
+
 	"github.com/multiversx/mx-chain-go/common"
 	"github.com/multiversx/mx-chain-go/config"
 	"github.com/multiversx/mx-chain-go/dataRetriever"
@@ -177,16 +178,17 @@ func (sesb *storageEpochStartBootstrap) prepareComponentsToSync() error {
 	}
 
 	argsEpochStartSyncer := ArgsNewEpochStartMetaSyncer{
-		CoreComponentsHolder:    sesb.coreComponentsHolder,
-		CryptoComponentsHolder:  sesb.cryptoComponentsHolder,
-		RequestHandler:          sesb.requestHandler,
-		Messenger:               sesb.mainMessenger,
-		ShardCoordinator:        sesb.shardCoordinator,
-		EconomicsData:           sesb.economicsData,
-		WhitelistHandler:        sesb.whiteListHandler,
-		StartInEpochConfig:      sesb.generalConfig.EpochStartConfig,
-		HeaderIntegrityVerifier: sesb.headerIntegrityVerifier,
-		MetaBlockProcessor:      metablockProcessor,
+		CoreComponentsHolder:           sesb.coreComponentsHolder,
+		CryptoComponentsHolder:         sesb.cryptoComponentsHolder,
+		RequestHandler:                 sesb.requestHandler,
+		Messenger:                      sesb.mainMessenger,
+		ShardCoordinator:               sesb.shardCoordinator,
+		EconomicsData:                  sesb.economicsData,
+		WhitelistHandler:               sesb.whiteListHandler,
+		StartInEpochConfig:             sesb.generalConfig.EpochStartConfig,
+		HeaderIntegrityVerifier:        sesb.headerIntegrityVerifier,
+		MetaBlockProcessor:             metablockProcessor,
+		InterceptedDataVerifierFactory: sesb.interceptedDataVerifierFactory,
 	}
 
 	sesb.epochStartMetaBlockSyncer, err = NewEpochStartMetaSyncer(argsEpochStartSyncer)

epochStart/bootstrap/storageProcess_test.go

@@ -11,19 +11,21 @@ import (
 	"github.com/multiversx/mx-chain-core-go/data"
 	"github.com/multiversx/mx-chain-core-go/data/block"
 	"github.com/multiversx/mx-chain-core-go/data/endProcess"
+	"github.com/stretchr/testify/assert"
+
 	"github.com/multiversx/mx-chain-go/config"
 	"github.com/multiversx/mx-chain-go/dataRetriever"
 	"github.com/multiversx/mx-chain-go/epochStart"
 	"github.com/multiversx/mx-chain-go/epochStart/mock"
 	"github.com/multiversx/mx-chain-go/process"
+	processMock "github.com/multiversx/mx-chain-go/process/mock"
 	"github.com/multiversx/mx-chain-go/sharding/nodesCoordinator"
 	"github.com/multiversx/mx-chain-go/storage"
 	"github.com/multiversx/mx-chain-go/testscommon"
 	epochStartMocks "github.com/multiversx/mx-chain-go/testscommon/bootstrapMocks/epochStart"
 	dataRetrieverMock "github.com/multiversx/mx-chain-go/testscommon/dataRetriever"
 	"github.com/multiversx/mx-chain-go/testscommon/economicsmocks"
 	"github.com/multiversx/mx-chain-go/testscommon/genesisMocks"
-	"github.com/stretchr/testify/assert"
 )
 
 func createMockStorageEpochStartBootstrapArgs(
@@ -127,6 +129,7 @@ func TestStorageEpochStartBootstrap_BootstrapMetablockNotFound(t *testing.T) {
 	}
 	args.GeneralConfig = testscommon.GetGeneralConfig()
 	args.GeneralConfig.EpochStartConfig.RoundsPerEpoch = roundsPerEpoch
+	args.InterceptedDataVerifierFactory = &processMock.InterceptedDataVerifierFactoryMock{}
 	sesb, _ := NewStorageEpochStartBootstrap(args)
 
 	params, err := sesb.Bootstrap()