Skip to content

Commit

Permalink
Merge pull request #6462 from multiversx/message_validation_optimization
Browse files Browse the repository at this point in the history 
Message validation optimization
  • Loading branch information
@AdoAdoAdo
AdoAdoAdo authored Oct 4, 2024
2 parents 5c24692 + 111fdb1 commit c49cde8
Show file tree
Hide file tree
Showing 55 changed files with 1,504 additions and 672 deletions.
4 changes: 4 additions & 0 deletions cmd/node/config/config.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -958,3 +958,7 @@
# All validators will broadcast the message right away
{ EndIndex = 0, DelayInMilliseconds = 0 },
]

[InterceptedDataVerifier]
CacheSpanInSec = 30
CacheExpiryInSec = 30
8 changes: 8 additions & 0 deletions config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,8 @@ type Config struct {
PoolsCleanersConfig PoolsCleanersConfig
Redundancy RedundancyConfig
ConsensusGradualBroadcast ConsensusGradualBroadcastConfig

InterceptedDataVerifier InterceptedDataVerifierConfig
}

// PeersRatingConfig will hold settings related to peers rating
Expand Down Expand Up @@ -679,3 +681,9 @@ type IndexBroadcastDelay struct {
type ConsensusGradualBroadcastConfig struct {
GradualIndexBroadcastDelay []IndexBroadcastDelay
}

// InterceptedDataVerifierConfig holds the configuration for the intercepted data verifier
type InterceptedDataVerifierConfig struct {
CacheSpanInSec uint64
CacheExpiryInSec uint64
}
97 changes: 50 additions & 47 deletions epochStart/bootstrap/factory/epochStartInterceptorsContainerFactory.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"github.com/multiversx/mx-chain-core-go/core"
"github.com/multiversx/mx-chain-core-go/core/check"
"github.com/multiversx/mx-chain-core-go/data/typeConverters"

"github.com/multiversx/mx-chain-go/common"
"github.com/multiversx/mx-chain-go/config"
"github.com/multiversx/mx-chain-go/dataRetriever"
Expand All @@ -25,23 +26,24 @@ const timeSpanForBadHeaders = time.Minute
// ArgsEpochStartInterceptorContainer holds the arguments needed for creating a new epoch start interceptors
// container factory
type ArgsEpochStartInterceptorContainer struct {
CoreComponents process.CoreComponentsHolder
CryptoComponents process.CryptoComponentsHolder
Config config.Config
ShardCoordinator sharding.Coordinator
MainMessenger process.TopicHandler
FullArchiveMessenger process.TopicHandler
DataPool dataRetriever.PoolsHolder
WhiteListHandler update.WhiteListHandler
WhiteListerVerifiedTxs update.WhiteListHandler
AddressPubkeyConv core.PubkeyConverter
NonceConverter typeConverters.Uint64ByteSliceConverter
ChainID []byte
ArgumentsParser process.ArgumentsParser
HeaderIntegrityVerifier process.HeaderIntegrityVerifier
RequestHandler process.RequestHandler
SignaturesHandler process.SignaturesHandler
NodeOperationMode common.NodeOperation
CoreComponents process.CoreComponentsHolder
CryptoComponents process.CryptoComponentsHolder
Config config.Config
ShardCoordinator sharding.Coordinator
MainMessenger process.TopicHandler
FullArchiveMessenger process.TopicHandler
DataPool dataRetriever.PoolsHolder
WhiteListHandler update.WhiteListHandler
WhiteListerVerifiedTxs update.WhiteListHandler
AddressPubkeyConv core.PubkeyConverter
NonceConverter typeConverters.Uint64ByteSliceConverter
ChainID []byte
ArgumentsParser process.ArgumentsParser
HeaderIntegrityVerifier process.HeaderIntegrityVerifier
RequestHandler process.RequestHandler
SignaturesHandler process.SignaturesHandler
NodeOperationMode common.NodeOperation
InterceptedDataVerifierFactory process.InterceptedDataVerifierFactory
}

// NewEpochStartInterceptorsContainer will return a real interceptors container factory, but with many disabled components
Expand Down Expand Up @@ -78,36 +80,37 @@ func NewEpochStartInterceptorsContainer(args ArgsEpochStartInterceptorContainer)
hardforkTrigger := disabledFactory.HardforkTrigger()

containerFactoryArgs := interceptorscontainer.CommonInterceptorsContainerFactoryArgs{
CoreComponents: args.CoreComponents,
CryptoComponents: cryptoComponents,
Accounts: accountsAdapter,
ShardCoordinator: args.ShardCoordinator,
NodesCoordinator: nodesCoordinator,
MainMessenger: args.MainMessenger,
FullArchiveMessenger: args.FullArchiveMessenger,
Store: storer,
DataPool: args.DataPool,
MaxTxNonceDeltaAllowed: common.MaxTxNonceDeltaAllowed,
TxFeeHandler: feeHandler,
BlockBlackList: blackListHandler,
HeaderSigVerifier: headerSigVerifier,
HeaderIntegrityVerifier: args.HeaderIntegrityVerifier,
ValidityAttester: validityAttester,
EpochStartTrigger: epochStartTrigger,
WhiteListHandler: args.WhiteListHandler,
WhiteListerVerifiedTxs: args.WhiteListerVerifiedTxs,
AntifloodHandler: antiFloodHandler,
ArgumentsParser: args.ArgumentsParser,
PreferredPeersHolder: disabled.NewPreferredPeersHolder(),
SizeCheckDelta: uint32(sizeCheckDelta),
RequestHandler: args.RequestHandler,
PeerSignatureHandler: cryptoComponents.PeerSignatureHandler(),
SignaturesHandler: args.SignaturesHandler,
HeartbeatExpiryTimespanInSec: args.Config.HeartbeatV2.HeartbeatExpiryTimespanInSec,
MainPeerShardMapper: peerShardMapper,
FullArchivePeerShardMapper: fullArchivePeerShardMapper,
HardforkTrigger: hardforkTrigger,
NodeOperationMode: args.NodeOperationMode,
CoreComponents: args.CoreComponents,
CryptoComponents: cryptoComponents,
Accounts: accountsAdapter,
ShardCoordinator: args.ShardCoordinator,
NodesCoordinator: nodesCoordinator,
MainMessenger: args.MainMessenger,
FullArchiveMessenger: args.FullArchiveMessenger,
Store: storer,
DataPool: args.DataPool,
MaxTxNonceDeltaAllowed: common.MaxTxNonceDeltaAllowed,
TxFeeHandler: feeHandler,
BlockBlackList: blackListHandler,
HeaderSigVerifier: headerSigVerifier,
HeaderIntegrityVerifier: args.HeaderIntegrityVerifier,
ValidityAttester: validityAttester,
EpochStartTrigger: epochStartTrigger,
WhiteListHandler: args.WhiteListHandler,
WhiteListerVerifiedTxs: args.WhiteListerVerifiedTxs,
AntifloodHandler: antiFloodHandler,
ArgumentsParser: args.ArgumentsParser,
PreferredPeersHolder: disabled.NewPreferredPeersHolder(),
SizeCheckDelta: uint32(sizeCheckDelta),
RequestHandler: args.RequestHandler,
PeerSignatureHandler: cryptoComponents.PeerSignatureHandler(),
SignaturesHandler: args.SignaturesHandler,
HeartbeatExpiryTimespanInSec: args.Config.HeartbeatV2.HeartbeatExpiryTimespanInSec,
MainPeerShardMapper: peerShardMapper,
FullArchivePeerShardMapper: fullArchivePeerShardMapper,
HardforkTrigger: hardforkTrigger,
NodeOperationMode: args.NodeOperationMode,
InterceptedDataVerifierFactory: args.InterceptedDataVerifierFactory,
}

interceptorsContainerFactory, err := interceptorscontainer.NewMetaInterceptorsContainerFactory(containerFactoryArgs)
Expand Down
57 changes: 32 additions & 25 deletions epochStart/bootstrap/process.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ import (
"github.com/multiversx/mx-chain-core-go/data"
"github.com/multiversx/mx-chain-core-go/data/block"
"github.com/multiversx/mx-chain-core-go/data/typeConverters/uint64ByteSlice"
logger "github.com/multiversx/mx-chain-logger-go"

"github.com/multiversx/mx-chain-go/common"
disabledCommon "github.com/multiversx/mx-chain-go/common/disabled"
"github.com/multiversx/mx-chain-go/common/ordering"
Expand Down Expand Up @@ -52,7 +54,6 @@ import (
"github.com/multiversx/mx-chain-go/trie/storageMarker"
"github.com/multiversx/mx-chain-go/update"
updateSync "github.com/multiversx/mx-chain-go/update/sync"
logger "github.com/multiversx/mx-chain-logger-go"
)

var log = logger.GetOrCreate("epochStart/bootstrap")
Expand Down Expand Up @@ -152,6 +153,8 @@ type epochStartBootstrap struct {
nodeType core.NodeType
startEpoch uint32
shuffledOut bool

interceptedDataVerifierFactory process.InterceptedDataVerifierFactory
}

type baseDataInStorage struct {
Expand Down Expand Up @@ -190,6 +193,7 @@ type ArgsEpochStartBootstrap struct {
NodeProcessingMode common.NodeProcessingMode
StateStatsHandler common.StateStatisticsHandler
NodesCoordinatorRegistryFactory nodesCoordinator.NodesCoordinatorRegistryFactory
InterceptedDataVerifierFactory process.InterceptedDataVerifierFactory
}

type dataToSync struct {
Expand Down Expand Up @@ -242,6 +246,7 @@ func NewEpochStartBootstrap(args ArgsEpochStartBootstrap) (*epochStartBootstrap,
stateStatsHandler: args.StateStatsHandler,
startEpoch: args.GeneralConfig.EpochStartConfig.GenesisEpoch,
nodesCoordinatorRegistryFactory: args.NodesCoordinatorRegistryFactory,
interceptedDataVerifierFactory: args.InterceptedDataVerifierFactory,
}

if epochStartProvider.prefsConfig.FullArchive {
Expand Down Expand Up @@ -553,16 +558,17 @@ func (e *epochStartBootstrap) prepareComponentsToSyncFromNetwork() error {
}

argsEpochStartSyncer := ArgsNewEpochStartMetaSyncer{
CoreComponentsHolder: e.coreComponentsHolder,
CryptoComponentsHolder: e.cryptoComponentsHolder,
RequestHandler: e.requestHandler,
Messenger: e.mainMessenger,
ShardCoordinator: e.shardCoordinator,
EconomicsData: e.economicsData,
WhitelistHandler: e.whiteListHandler,
StartInEpochConfig: epochStartConfig,
HeaderIntegrityVerifier: e.headerIntegrityVerifier,
MetaBlockProcessor: metaBlockProcessor,
CoreComponentsHolder: e.coreComponentsHolder,
CryptoComponentsHolder: e.cryptoComponentsHolder,
RequestHandler: e.requestHandler,
Messenger: e.mainMessenger,
ShardCoordinator: e.shardCoordinator,
EconomicsData: e.economicsData,
WhitelistHandler: e.whiteListHandler,
StartInEpochConfig: epochStartConfig,
HeaderIntegrityVerifier: e.headerIntegrityVerifier,
MetaBlockProcessor: metaBlockProcessor,
InterceptedDataVerifierFactory: e.interceptedDataVerifierFactory,
}
e.epochStartMetaBlockSyncer, err = NewEpochStartMetaSyncer(argsEpochStartSyncer)
if err != nil {
Expand All @@ -575,20 +581,21 @@ func (e *epochStartBootstrap) prepareComponentsToSyncFromNetwork() error {
func (e *epochStartBootstrap) createSyncers() error {
var err error
args := factoryInterceptors.ArgsEpochStartInterceptorContainer{
CoreComponents: e.coreComponentsHolder,
CryptoComponents: e.cryptoComponentsHolder,
Config: e.generalConfig,
ShardCoordinator: e.shardCoordinator,
MainMessenger: e.mainMessenger,
FullArchiveMessenger: e.fullArchiveMessenger,
DataPool: e.dataPool,
WhiteListHandler: e.whiteListHandler,
WhiteListerVerifiedTxs: e.whiteListerVerifiedTxs,
ArgumentsParser: e.argumentsParser,
HeaderIntegrityVerifier: e.headerIntegrityVerifier,
RequestHandler: e.requestHandler,
SignaturesHandler: e.mainMessenger,
NodeOperationMode: e.nodeOperationMode,
CoreComponents: e.coreComponentsHolder,
CryptoComponents: e.cryptoComponentsHolder,
Config: e.generalConfig,
ShardCoordinator: e.shardCoordinator,
MainMessenger: e.mainMessenger,
FullArchiveMessenger: e.fullArchiveMessenger,
DataPool: e.dataPool,
WhiteListHandler: e.whiteListHandler,
WhiteListerVerifiedTxs: e.whiteListerVerifiedTxs,
ArgumentsParser: e.argumentsParser,
HeaderIntegrityVerifier: e.headerIntegrityVerifier,
RequestHandler: e.requestHandler,
SignaturesHandler: e.mainMessenger,
NodeOperationMode: e.nodeOperationMode,
InterceptedDataVerifierFactory: e.interceptedDataVerifierFactory,
}

e.mainInterceptorContainer, e.fullArchiveInterceptorContainer, err = factoryInterceptors.NewEpochStartInterceptorsContainer(args)
Expand Down
7 changes: 5 additions & 2 deletions epochStart/bootstrap/process_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
"github.com/multiversx/mx-chain-go/epochStart/bootstrap/types"
"github.com/multiversx/mx-chain-go/epochStart/mock"
"github.com/multiversx/mx-chain-go/process"
processMock "github.com/multiversx/mx-chain-go/process/mock"
"github.com/multiversx/mx-chain-go/sharding"
"github.com/multiversx/mx-chain-go/sharding/nodesCoordinator"
"github.com/multiversx/mx-chain-go/state"
Expand Down Expand Up @@ -251,8 +252,9 @@ func createMockEpochStartBootstrapArgs(
FlagsConfig: config.ContextFlagsConfig{
ForceStartFromNetwork: false,
},
TrieSyncStatisticsProvider: &testscommon.SizeSyncStatisticsHandlerStub{},
StateStatsHandler: disabledStatistics.NewStateStatistics(),
TrieSyncStatisticsProvider: &testscommon.SizeSyncStatisticsHandlerStub{},
StateStatsHandler: disabledStatistics.NewStateStatistics(),
InterceptedDataVerifierFactory: &processMock.InterceptedDataVerifierFactoryMock{},
}
}

Expand Down Expand Up @@ -994,6 +996,7 @@ func TestCreateSyncers(t *testing.T) {
epochStartProvider.whiteListerVerifiedTxs = &testscommon.WhiteListHandlerStub{}
epochStartProvider.requestHandler = &testscommon.RequestHandlerStub{}
epochStartProvider.storageService = &storageMocks.ChainStorerStub{}
epochStartProvider.interceptedDataVerifierFactory = &processMock.InterceptedDataVerifierFactoryMock{}

err := epochStartProvider.createSyncers()
assert.Nil(t, err)
Expand Down
22 changes: 12 additions & 10 deletions epochStart/bootstrap/storageProcess.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/multiversx/mx-chain-core-go/data"
"github.com/multiversx/mx-chain-core-go/data/block"
"github.com/multiversx/mx-chain-core-go/data/endProcess"

"github.com/multiversx/mx-chain-go/common"
"github.com/multiversx/mx-chain-go/config"
"github.com/multiversx/mx-chain-go/dataRetriever"
Expand Down Expand Up @@ -177,16 +178,17 @@ func (sesb *storageEpochStartBootstrap) prepareComponentsToSync() error {
}

argsEpochStartSyncer := ArgsNewEpochStartMetaSyncer{
CoreComponentsHolder: sesb.coreComponentsHolder,
CryptoComponentsHolder: sesb.cryptoComponentsHolder,
RequestHandler: sesb.requestHandler,
Messenger: sesb.mainMessenger,
ShardCoordinator: sesb.shardCoordinator,
EconomicsData: sesb.economicsData,
WhitelistHandler: sesb.whiteListHandler,
StartInEpochConfig: sesb.generalConfig.EpochStartConfig,
HeaderIntegrityVerifier: sesb.headerIntegrityVerifier,
MetaBlockProcessor: metablockProcessor,
CoreComponentsHolder: sesb.coreComponentsHolder,
CryptoComponentsHolder: sesb.cryptoComponentsHolder,
RequestHandler: sesb.requestHandler,
Messenger: sesb.mainMessenger,
ShardCoordinator: sesb.shardCoordinator,
EconomicsData: sesb.economicsData,
WhitelistHandler: sesb.whiteListHandler,
StartInEpochConfig: sesb.generalConfig.EpochStartConfig,
HeaderIntegrityVerifier: sesb.headerIntegrityVerifier,
MetaBlockProcessor: metablockProcessor,
InterceptedDataVerifierFactory: sesb.interceptedDataVerifierFactory,
}

sesb.epochStartMetaBlockSyncer, err = NewEpochStartMetaSyncer(argsEpochStartSyncer)
Expand Down
5 changes: 4 additions & 1 deletion epochStart/bootstrap/storageProcess_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,19 +11,21 @@ import (
"github.com/multiversx/mx-chain-core-go/data"
"github.com/multiversx/mx-chain-core-go/data/block"
"github.com/multiversx/mx-chain-core-go/data/endProcess"
"github.com/stretchr/testify/assert"

"github.com/multiversx/mx-chain-go/config"
"github.com/multiversx/mx-chain-go/dataRetriever"
"github.com/multiversx/mx-chain-go/epochStart"
"github.com/multiversx/mx-chain-go/epochStart/mock"
"github.com/multiversx/mx-chain-go/process"
processMock "github.com/multiversx/mx-chain-go/process/mock"
"github.com/multiversx/mx-chain-go/sharding/nodesCoordinator"
"github.com/multiversx/mx-chain-go/storage"
"github.com/multiversx/mx-chain-go/testscommon"
epochStartMocks "github.com/multiversx/mx-chain-go/testscommon/bootstrapMocks/epochStart"
dataRetrieverMock "github.com/multiversx/mx-chain-go/testscommon/dataRetriever"
"github.com/multiversx/mx-chain-go/testscommon/economicsmocks"
"github.com/multiversx/mx-chain-go/testscommon/genesisMocks"
"github.com/stretchr/testify/assert"
)

func createMockStorageEpochStartBootstrapArgs(
Expand Down Expand Up @@ -127,6 +129,7 @@ func TestStorageEpochStartBootstrap_BootstrapMetablockNotFound(t *testing.T) {
}
args.GeneralConfig = testscommon.GetGeneralConfig()
args.GeneralConfig.EpochStartConfig.RoundsPerEpoch = roundsPerEpoch
args.InterceptedDataVerifierFactory = &processMock.InterceptedDataVerifierFactoryMock{}
sesb, _ := NewStorageEpochStartBootstrap(args)

params, err := sesb.Bootstrap()
Expand Down
Loading

0 comments on commit c49cde8

Please sign in to comment.