Merge pull request #5966 from multiversx/merge-rc-1.6.next1-rc-1.7.0-…

…2024.02.16 Merge rc 1.6.next1 rc 1.7.0 2024.02.16
multiversx · Feb 16, 2024 · a4206f6 · a4206f6
2 parents 2ffb571 + 923149b
commit a4206f6
Show file tree

Hide file tree

Showing 74 changed files with 1,411 additions and 885 deletions.
diff --git a/api/groups/nodeGroup.go b/api/groups/nodeGroup.go
@@ -28,6 +28,7 @@ const (
 	bootstrapStatusPath       = "/bootstrapstatus"
 	connectedPeersRatingsPath = "/connected-peers-ratings"
 	managedKeys               = "/managed-keys"
+	loadedKeys                = "/loaded-keys"
 	managedKeysCount          = "/managed-keys/count"
 	eligibleManagedKeys       = "/managed-keys/eligible"
 	waitingManagedKeys        = "/managed-keys/waiting"
@@ -44,6 +45,7 @@ type nodeFacadeHandler interface {
 	GetConnectedPeersRatingsOnMainNetwork() (string, error)
 	GetManagedKeysCount() int
 	GetManagedKeys() []string
+	GetLoadedKeys() []string
 	GetEligibleManagedKeys() ([]string, error)
 	GetWaitingManagedKeys() ([]string, error)
 	GetWaitingEpochsLeftForPublicKey(publicKey string) (uint32, error)
@@ -129,6 +131,11 @@ func NewNodeGroup(facade nodeFacadeHandler) (*nodeGroup, error) {
 			Method:  http.MethodGet,
 			Handler: ng.managedKeys,
 		},
+		{
+			Path:    loadedKeys,
+			Method:  http.MethodGet,
+			Handler: ng.loadedKeys,
+		},
 		{
 			Path:    eligibleManagedKeys,
 			Method:  http.MethodGet,
@@ -411,6 +418,19 @@ func (ng *nodeGroup) managedKeys(c *gin.Context) {
 	)
 }
 
+// loadedKeys returns all keys loaded by the current node
+func (ng *nodeGroup) loadedKeys(c *gin.Context) {
+	keys := ng.getFacade().GetLoadedKeys()
+	c.JSON(
+		http.StatusOK,
+		shared.GenericAPIResponse{
+			Data:  gin.H{"loadedKeys": keys},
+			Error: "",
+			Code:  shared.ReturnCodeSuccess,
+		},
+	)
+}
+
 // managedKeysEligible returns the node's eligible managed keys
 func (ng *nodeGroup) managedKeysEligible(c *gin.Context) {
 	keys, err := ng.getFacade().GetEligibleManagedKeys()

diff --git a/api/groups/nodeGroup_test.go b/api/groups/nodeGroup_test.go
@@ -81,6 +81,13 @@ type managedKeysResponse struct {
 	generalResponse
 }
 
+type loadedKeysResponse struct {
+	Data struct {
+		LoadedKeys []string `json:"loadedKeys"`
+	} `json:"data"`
+	generalResponse
+}
+
 type managedEligibleKeysResponse struct {
 	Data struct {
 		Keys []string `json:"eligibleKeys"`
@@ -764,6 +771,36 @@ func TestNodeGroup_ManagedKeys(t *testing.T) {
 	assert.Equal(t, providedKeys, response.Data.ManagedKeys)
 }
 
+func TestNodeGroup_LoadedKeys(t *testing.T) {
+	t.Parallel()
+
+	providedKeys := []string{
+		"pk1",
+		"pk2",
+	}
+	facade := mock.FacadeStub{
+		GetLoadedKeysCalled: func() []string {
+			return providedKeys
+		},
+	}
+
+	nodeGroup, err := groups.NewNodeGroup(&facade)
+	require.NoError(t, err)
+
+	ws := startWebServer(nodeGroup, "node", getNodeRoutesConfig())
+
+	req, _ := http.NewRequest("GET", "/node/loaded-keys", nil)
+	resp := httptest.NewRecorder()
+	ws.ServeHTTP(resp, req)
+
+	response := &loadedKeysResponse{}
+	loadResponse(resp.Body, response)
+
+	assert.Equal(t, http.StatusOK, resp.Code)
+	assert.Equal(t, "", response.Error)
+	assert.Equal(t, providedKeys, response.Data.LoadedKeys)
+}
+
 func TestNodeGroup_ManagedKeysEligible(t *testing.T) {
 	t.Parallel()
 
@@ -1046,6 +1083,7 @@ func getNodeRoutesConfig() config.ApiRoutesConfig {
 					{Name: "/connected-peers-ratings", Open: true},
 					{Name: "/managed-keys/count", Open: true},
 					{Name: "/managed-keys", Open: true},
+					{Name: "/loaded-keys", Open: true},
 					{Name: "/managed-keys/eligible", Open: true},
 					{Name: "/managed-keys/waiting", Open: true},
 					{Name: "/waiting-epochs-left/:key", Open: true},

diff --git a/api/mock/facadeStub.go b/api/mock/facadeStub.go
@@ -91,6 +91,7 @@ type FacadeStub struct {
 	IsDataTrieMigratedCalled                    func(address string, options api.AccountQueryOptions) (bool, error)
 	GetManagedKeysCountCalled                   func() int
 	GetManagedKeysCalled                        func() []string
+	GetLoadedKeysCalled                         func() []string
 	GetEligibleManagedKeysCalled                func() ([]string, error)
 	GetWaitingManagedKeysCalled                 func() ([]string, error)
 	GetWaitingEpochsLeftForPublicKeyCalled      func(publicKey string) (uint32, error)
@@ -596,6 +597,14 @@ func (f *FacadeStub) GetManagedKeys() []string {
 	return make([]string, 0)
 }
 
+// GetLoadedKeys -
+func (f *FacadeStub) GetLoadedKeys() []string {
+	if f.GetLoadedKeysCalled != nil {
+		return f.GetLoadedKeysCalled()
+	}
+	return make([]string, 0)
+}
+
 // GetEligibleManagedKeys -
 func (f *FacadeStub) GetEligibleManagedKeys() ([]string, error) {
 	if f.GetEligibleManagedKeysCalled != nil {

diff --git a/api/shared/interface.go b/api/shared/interface.go
@@ -130,6 +130,7 @@ type FacadeHandler interface {
 	IsDataTrieMigrated(address string, options api.AccountQueryOptions) (bool, error)
 	GetManagedKeysCount() int
 	GetManagedKeys() []string
+	GetLoadedKeys() []string
 	GetEligibleManagedKeys() ([]string, error)
 	GetWaitingManagedKeys() ([]string, error)
 	GetWaitingEpochsLeftForPublicKey(publicKey string) (uint32, error)

diff --git a/cmd/node/config/api.toml b/cmd/node/config/api.toml
@@ -43,6 +43,9 @@
         # /node/managed-keys will return the keys managed by the node
         { Name = "/managed-keys", Open = true },
 
+        # /node/loaded-keys will return the keys loaded by the node
+        { Name = "/loaded-keys", Open = true },
+
         # /node/managed-keys/count will return the number of keys managed by the node
         { Name = "/managed-keys/count", Open = true },
 

diff --git a/cmd/node/config/config.toml b/cmd/node/config/config.toml
@@ -492,6 +492,7 @@
 [Antiflood]
     Enabled = true
     NumConcurrentResolverJobs = 50
+    NumConcurrentResolvingTrieNodesJobs = 3
     [Antiflood.FastReacting]
         IntervalInSeconds = 1
         ReservedPercent   = 20.0

diff --git a/cmd/node/config/fullArchiveP2P.toml b/cmd/node/config/fullArchiveP2P.toml
@@ -23,10 +23,11 @@
         [Node.Transports.TCP]
             ListenAddress = "/ip4/0.0.0.0/tcp/%d" # TCP listen address
             PreventPortReuse = false
-        [Node.ResourceLimiter]
-            Type = "default autoscale" #available options "default autoscale", "infinite", "default with manual scale".
-            ManualSystemMemoryInMB = 0 # not taken into account if the type is not "default with manual scale"
-            ManualMaximumFD = 0 # not taken into account if the type is not "default with manual scale"
+
+    [Node.ResourceLimiter]
+        Type = "default autoscale" #available options "default autoscale", "infinite", "default with manual scale".
+        ManualSystemMemoryInMB = 0 # not taken into account if the type is not "default with manual scale"
+        ManualMaximumFD = 0 # not taken into account if the type is not "default with manual scale"
 
 # P2P peer discovery section
 

diff --git a/cmd/node/config/p2p.toml b/cmd/node/config/p2p.toml
@@ -23,10 +23,11 @@
         [Node.Transports.TCP]
             ListenAddress = "/ip4/0.0.0.0/tcp/%d" # TCP listen address
             PreventPortReuse = false
-        [Node.ResourceLimiter]
-            Type = "default autoscale" #available options "default autoscale", "infinite", "default with manual scale".
-            ManualSystemMemoryInMB = 0 # not taken into account if the type is not "default with manual scale"
-            ManualMaximumFD = 0 # not taken into account if the type is not "default with manual scale"
+
+    [Node.ResourceLimiter]
+        Type = "default autoscale" #available options "default autoscale", "infinite", "default with manual scale".
+        ManualSystemMemoryInMB = 0 # not taken into account if the type is not "default with manual scale"
+        ManualMaximumFD = 0 # not taken into account if the type is not "default with manual scale"
 
 # P2P peer discovery section
 

diff --git a/cmd/seednode/config/p2p.toml b/cmd/seednode/config/p2p.toml
@@ -22,10 +22,11 @@
         [Node.Transports.TCP]
             ListenAddress = "/ip4/0.0.0.0/tcp/%d" # TCP listen address
             PreventPortReuse = true # seeder nodes will need to enable this option
-        [Node.ResourceLimiter]
-            Type = "default with manual scale"
-            ManualSystemMemoryInMB = 65536 # pretend that the host running the seeder has more RAM so it can handle more connections
-            ManualMaximumFD = 1048576
+
+    [Node.ResourceLimiter]
+        Type = "default with manual scale"
+        ManualSystemMemoryInMB = 65536 # pretend that the host running the seeder has more RAM so it can handle more connections
+        ManualMaximumFD = 1048576
 
 # P2P peer discovery section
 

diff --git a/common/interface.go b/common/interface.go
@@ -314,6 +314,7 @@ type ManagedPeersHolder interface {
 	IncrementRoundsWithoutReceivedMessages(pkBytes []byte)
 	ResetRoundsWithoutReceivedMessages(pkBytes []byte, pid core.PeerID)
 	GetManagedKeysByCurrentNode() map[string]crypto.PrivateKey
+	GetLoadedKeysByCurrentNode() [][]byte
 	IsKeyManagedByCurrentNode(pkBytes []byte) bool
 	IsKeyRegistered(pkBytes []byte) bool
 	IsPidManagedByCurrentNode(pid core.PeerID) bool
@@ -343,6 +344,7 @@ type StateSyncNotifierSubscriber interface {
 type ManagedPeersMonitor interface {
 	GetManagedKeysCount() int
 	GetManagedKeys() [][]byte
+	GetLoadedKeys() [][]byte
 	GetEligibleManagedKeys() ([][]byte, error)
 	GetWaitingManagedKeys() ([][]byte, error)
 	IsInterfaceNil() bool

diff --git a/config/config.go b/config/config.go
@@ -366,15 +366,16 @@ type TxAccumulatorConfig struct {
 
 // AntifloodConfig will hold all p2p antiflood parameters
 type AntifloodConfig struct {
-	Enabled                   bool
-	NumConcurrentResolverJobs int32
-	OutOfSpecs                FloodPreventerConfig
-	FastReacting              FloodPreventerConfig
-	SlowReacting              FloodPreventerConfig
-	PeerMaxOutput             AntifloodLimitsConfig
-	Cache                     CacheConfig
-	Topic                     TopicAntifloodConfig
-	TxAccumulator             TxAccumulatorConfig
+	Enabled                             bool
+	NumConcurrentResolverJobs           int32
+	NumConcurrentResolvingTrieNodesJobs int32
+	OutOfSpecs                          FloodPreventerConfig
+	FastReacting                        FloodPreventerConfig
+	SlowReacting                        FloodPreventerConfig
+	PeerMaxOutput                       AntifloodLimitsConfig
+	Cache                               CacheConfig
+	Topic                               TopicAntifloodConfig
+	TxAccumulator                       TxAccumulatorConfig
 }
 
 // FloodPreventerConfig will hold all flood preventer parameters

diff --git a/config/tomlConfig_test.go b/config/tomlConfig_test.go
@@ -489,10 +489,11 @@ func TestP2pConfig(t *testing.T) {
         [Node.Transports.TCP]
             ListenAddress = "/ip4/0.0.0.0/tcp/%d"
             PreventPortReuse = true
-        [Node.ResourceLimiter]
-            Type = "default autoscale" #available options "default autoscale", "infinite", "default with manual scale".
-            ManualSystemMemoryInMB = 1 # not taken into account if the type is not "default with manual scale"
-            ManualMaximumFD = 2 # not taken into account if the type is not "default with manual scale"
+
+    [Node.ResourceLimiter]
+        Type = "default autoscale" #available options "default autoscale", "infinite", "default with manual scale".
+        ManualSystemMemoryInMB = 1 # not taken into account if the type is not "default with manual scale"
+        ManualMaximumFD = 2 # not taken into account if the type is not "default with manual scale"
 
 [KadDhtPeerDiscovery]
     Enabled = false

diff --git a/dataRetriever/factory/resolverscontainer/args.go b/dataRetriever/factory/resolverscontainer/args.go
@@ -11,21 +11,22 @@ import (
 
 // FactoryArgs will hold the arguments for ResolversContainerFactory for both shard and meta
 type FactoryArgs struct {
-	NumConcurrentResolvingJobs      int32
-	ShardCoordinator                sharding.Coordinator
-	MainMessenger                   p2p.Messenger
-	FullArchiveMessenger            p2p.Messenger
-	Store                           dataRetriever.StorageService
-	Marshalizer                     marshal.Marshalizer
-	DataPools                       dataRetriever.PoolsHolder
-	Uint64ByteSliceConverter        typeConverters.Uint64ByteSliceConverter
-	DataPacker                      dataRetriever.DataPacker
-	TriesContainer                  common.TriesHolder
-	InputAntifloodHandler           dataRetriever.P2PAntifloodHandler
-	OutputAntifloodHandler          dataRetriever.P2PAntifloodHandler
-	MainPreferredPeersHolder        p2p.PreferredPeersHolderHandler
-	FullArchivePreferredPeersHolder p2p.PreferredPeersHolderHandler
-	SizeCheckDelta                  uint32
-	IsFullHistoryNode               bool
-	PayloadValidator                dataRetriever.PeerAuthenticationPayloadValidator
+	NumConcurrentResolvingJobs          int32
+	NumConcurrentResolvingTrieNodesJobs int32
+	ShardCoordinator                    sharding.Coordinator
+	MainMessenger                       p2p.Messenger
+	FullArchiveMessenger                p2p.Messenger
+	Store                               dataRetriever.StorageService
+	Marshalizer                         marshal.Marshalizer
+	DataPools                           dataRetriever.PoolsHolder
+	Uint64ByteSliceConverter            typeConverters.Uint64ByteSliceConverter
+	DataPacker                          dataRetriever.DataPacker
+	TriesContainer                      common.TriesHolder
+	InputAntifloodHandler               dataRetriever.P2PAntifloodHandler
+	OutputAntifloodHandler              dataRetriever.P2PAntifloodHandler
+	MainPreferredPeersHolder            p2p.PreferredPeersHolderHandler
+	FullArchivePreferredPeersHolder     p2p.PreferredPeersHolderHandler
+	SizeCheckDelta                      uint32
+	IsFullHistoryNode                   bool
+	PayloadValidator                    dataRetriever.PeerAuthenticationPayloadValidator
 }
diff --git a/dataRetriever/factory/resolverscontainer/baseResolversContainerFactory.go b/dataRetriever/factory/resolverscontainer/baseResolversContainerFactory.go
@@ -36,6 +36,7 @@ type baseResolversContainerFactory struct {
 	inputAntifloodHandler           dataRetriever.P2PAntifloodHandler
 	outputAntifloodHandler          dataRetriever.P2PAntifloodHandler
 	throttler                       dataRetriever.ResolverThrottler
+	trieNodesThrottler              dataRetriever.ResolverThrottler
 	intraShardTopic                 string
 	isFullHistoryNode               bool
 	mainPreferredPeersHolder        dataRetriever.PreferredPeersHolderHandler
@@ -78,7 +79,10 @@ func (brcf *baseResolversContainerFactory) checkParams() error {
 		return fmt.Errorf("%w for output", dataRetriever.ErrNilAntifloodHandler)
 	}
 	if check.IfNil(brcf.throttler) {
-		return dataRetriever.ErrNilThrottler
+		return fmt.Errorf("%w for the main throttler", dataRetriever.ErrNilThrottler)
+	}
+	if check.IfNil(brcf.trieNodesThrottler) {
+		return fmt.Errorf("%w for the trie nodes throttler", dataRetriever.ErrNilThrottler)
 	}
 	if check.IfNil(brcf.mainPreferredPeersHolder) {
 		return fmt.Errorf("%w for main network", dataRetriever.ErrNilPreferredPeersHolder)
@@ -351,7 +355,7 @@ func (brcf *baseResolversContainerFactory) createTrieNodesResolver(
 			SenderResolver:   resolverSender,
 			Marshaller:       brcf.marshalizer,
 			AntifloodHandler: brcf.inputAntifloodHandler,
-			Throttler:        brcf.throttler,
+			Throttler:        brcf.trieNodesThrottler,
 		},
 		TrieDataGetter: trie,
 	}

diff --git a/dataRetriever/factory/resolverscontainer/metaResolversContainerFactory.go b/dataRetriever/factory/resolverscontainer/metaResolversContainerFactory.go
@@ -27,7 +27,12 @@ func NewMetaResolversContainerFactory(
 		args.Marshalizer = marshal.NewSizeCheckUnmarshalizer(args.Marshalizer, args.SizeCheckDelta)
 	}
 
-	thr, err := throttler.NewNumGoRoutinesThrottler(args.NumConcurrentResolvingJobs)
+	mainThrottler, err := throttler.NewNumGoRoutinesThrottler(args.NumConcurrentResolvingJobs)
+	if err != nil {
+		return nil, err
+	}
+
+	trieNodesThrottler, err := throttler.NewNumGoRoutinesThrottler(args.NumConcurrentResolvingTrieNodesJobs)
 	if err != nil {
 		return nil, err
 	}
@@ -46,7 +51,8 @@ func NewMetaResolversContainerFactory(
 		triesContainer:                  args.TriesContainer,
 		inputAntifloodHandler:           args.InputAntifloodHandler,
 		outputAntifloodHandler:          args.OutputAntifloodHandler,
-		throttler:                       thr,
+		throttler:                       mainThrottler,
+		trieNodesThrottler:              trieNodesThrottler,
 		isFullHistoryNode:               args.IsFullHistoryNode,
 		mainPreferredPeersHolder:        args.MainPreferredPeersHolder,
 		fullArchivePreferredPeersHolder: args.FullArchivePreferredPeersHolder,