Merge remote-tracking branch 'origin/develop' into test/rewards-docke…

…rtest

components/inx/server_commitments.go

@@ -158,7 +158,14 @@ func (s *Server) ListenToCommitments(req *inx.SlotRangeRequest, srv inx.INX_List
 }
 
 func (s *Server) ForceCommitUntil(_ context.Context, slot *inx.SlotRequest) (*inx.NoParams, error) {
-	err := deps.Protocol.Engines.Main.Get().Notarization.ForceCommitUntil(slot.Unwrap())
+	// If the chain manager is aware of a commitments on the main chain, then do not force commit.
+	// The node should wait to warpsync those slots and use those commitments to avoid potentially creating a diverging commitment.
+	unwrappedSlot := slot.Unwrap()
+	if latestChainCommitment := deps.Protocol.Chains.Main.Get().LatestCommitment.Get(); latestChainCommitment.Slot() >= unwrappedSlot {
+		return nil, ierrors.Errorf("chain manager is aware of a newer commitment (%s) than target slot %d", latestChainCommitment, unwrappedSlot)
+	}
+
+	err := deps.Protocol.Engines.Main.Get().Notarization.ForceCommitUntil(unwrappedSlot)
 	if err != nil {
 		return nil, ierrors.Wrapf(err, "error while performing force commit until %d", slot.GetSlot())
 	}

components/prometheus/metrics_scheduler.go

@@ -3,7 +3,6 @@ package prometheus
 import (
 	"time"
 
-	"github.com/iotaledger/hive.go/ierrors"
 	"github.com/iotaledger/hive.go/runtime/event"
 	"github.com/iotaledger/iota-core/components/prometheus/collector"
 	"github.com/iotaledger/iota-core/pkg/protocol/engine/blocks"
@@ -16,7 +15,6 @@ const (
 	queueSizePerNodeCount          = "queue_size_per_node_count"
 	validatorQueueSizePerNodeCount = "validator_queue_size_per_node_count"
 	schedulerProcessedBlocks       = "processed_blocks"
-	manaAmountPerNode              = "mana_per_node"
 	scheduledBlockLabel            = "scheduled"
 	skippedBlockLabel              = "skipped"
 	droppedBlockLabel              = "dropped"
@@ -116,24 +114,6 @@ var SchedulerMetrics = collector.NewCollection(schedulerNamespace,
 			}, event.WithWorkerPool(Component.WorkerPool))
 		}),
 	)),
-	collector.WithMetric(collector.NewMetric(manaAmountPerNode,
-		collector.WithType(collector.Gauge),
-		collector.WithLabels("issuer_id"),
-		collector.WithPruningDelay(10*time.Minute),
-		collector.WithHelp("Current amount of mana of each issuer in the queue."),
-		collector.WithInitFunc(func() {
-			deps.Protocol.Events.Engine.Scheduler.BlockEnqueued.Hook(func(block *blocks.Block) {
-				mana, err := deps.Protocol.Engines.Main.Get().Ledger.ManaManager().GetManaOnAccount(block.ProtocolBlock().Header.IssuerID, block.SlotCommitmentID().Slot())
-				if err != nil {
-					deps.Protocol.Engines.Main.Get().ErrorHandler("metrics")(ierrors.Wrapf(err, "failed to retrieve mana on account %s for slot %d", block.ProtocolBlock().Header.IssuerID.ToHex(), block.SlotCommitmentID().Slot()))
-
-					return
-				}
-
-				deps.Collector.Update(schedulerNamespace, manaAmountPerNode, float64(mana), block.ProtocolBlock().Header.IssuerID.String())
-			}, event.WithWorkerPool(Component.WorkerPool))
-		}),
-	)),
 	collector.WithMetric(collector.NewMetric(schedulerProcessedBlocks,
 		collector.WithType(collector.Counter),
 		collector.WithLabels("state"),

components/protocol/component.go

@@ -90,6 +90,7 @@ func initConfigParams(c *dig.Container) error {
 		DatabaseEngine     db.Engine `name:"databaseEngine"`
 		BaseToken          *BaseToken
 		ProtocolParameters []iotago.ProtocolParameters
+		SnapshotFilePath   string `name:"snapshotFilePath"`
 	}
 
 	if err := c.Provide(func() cfgResult {
@@ -102,6 +103,7 @@ func initConfigParams(c *dig.Container) error {
 			DatabaseEngine:     dbEngine,
 			BaseToken:          &ParamsProtocol.BaseToken,
 			ProtocolParameters: readProtocolParameters(),
+			SnapshotFilePath:   ParamsProtocol.Snapshot.Path,
 		}
 	}); err != nil {
 		Component.LogPanic(err.Error())

components/restapi/management/component.go

@@ -35,6 +35,7 @@ type dependencies struct {
 	Protocol             *protocol.Protocol
 	PeeringConfigManager *p2p.ConfigManager
 	NetworkManager       network.Manager
+	SnapshotFilePath     string `name:"snapshotFilePath"`
 }
 
 func configure() error {
@@ -79,14 +80,14 @@ func configure() error {
 		return responseByHeader(c, resp, http.StatusOK)
 	})
 
-	// routeGroup.POST(api.ManagementEndpointSnapshotsCreate, func(c echo.Context) error {
-	//	resp, err := createSnapshots(c)
-	//	if err != nil {
-	//		return err
-	//	}
-	//
-	//	return responseByHeader(c, resp, http.StatusOK)
-	// })
+	routeGroup.POST(api.ManagementEndpointSnapshotsCreate, func(c echo.Context) error {
+		resp, err := createSnapshots(c)
+		if err != nil {
+			return err
+		}
+
+		return responseByHeader(c, resp, http.StatusOK)
+	})
 
 	return nil
 }

components/restapi/management/pruning.go

@@ -10,8 +10,8 @@ import (
 )
 
 func pruneDatabase(c echo.Context) (*api.PruneDatabaseResponse, error) {
-	if deps.Protocol.Engines.Main.Get().Storage.IsPruning() {
-		return nil, ierrors.WithMessage(echo.ErrServiceUnavailable, "node is already pruning")
+	if deps.Protocol.Engines.Main.Get().IsSnapshotting() || deps.Protocol.Engines.Main.Get().Storage.IsPruning() {
+		return nil, ierrors.WithMessage(echo.ErrServiceUnavailable, "node is already creating a snapshot or pruning is running")
 	}
 
 	request := &api.PruneDatabaseRequest{}

components/restapi/management/snapshots.go

@@ -0,0 +1,42 @@
+package management
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	"github.com/labstack/echo/v4"
+
+	"github.com/iotaledger/hive.go/ierrors"
+	"github.com/iotaledger/inx-app/pkg/httpserver"
+	"github.com/iotaledger/iota.go/v4/api"
+)
+
+func createSnapshots(c echo.Context) (*api.CreateSnapshotResponse, error) {
+	if deps.Protocol.Engines.Main.Get().IsSnapshotting() || deps.Protocol.Engines.Main.Get().Storage.IsPruning() {
+		return nil, ierrors.WithMessage(echo.ErrServiceUnavailable, "node is already creating a snapshot or pruning is running")
+	}
+
+	request := &api.CreateSnapshotRequest{}
+	if err := c.Bind(request); err != nil {
+		return nil, ierrors.WithMessagef(httpserver.ErrInvalidParameter, "invalid request: %w", err)
+	}
+	if request.Slot == 0 {
+		return nil, ierrors.WithMessage(httpserver.ErrInvalidParameter, "slot needs to be specified")
+	}
+
+	directory := filepath.Dir(deps.SnapshotFilePath)
+	fileName := filepath.Base(deps.SnapshotFilePath)
+	fileExt := filepath.Ext(fileName)
+	fileNameWithoutExt := strings.TrimSuffix(fileName, fileExt)
+	filePath := filepath.Join(directory, fmt.Sprintf("%s_%d%s", fileNameWithoutExt, request.Slot, fileExt))
+
+	if err := deps.Protocol.Engines.Main.Get().WriteSnapshot(filePath, request.Slot); err != nil {
+		return nil, ierrors.WithMessagef(echo.ErrInternalServerError, "creating snapshot failed: %w", err)
+	}
+
+	return &api.CreateSnapshotResponse{
+		Slot:     request.Slot,
+		FilePath: filePath,
+	}, nil
+}

deploy/ansible/roles/metrics/files/grafana/provisioning/dashboards/local_dashboard_old.json

@@ -3844,68 +3844,6 @@
       "title": "Scheduler metrics",
       "type": "row"
     },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green"
-              }
-            ]
-          },
-          "unit": "none"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 6,
-        "w": 15,
-        "x": 0,
-        "y": 115
-      },
-      "id": 146,
-      "options": {
-        "displayMode": "lcd",
-        "minVizHeight": 10,
-        "minVizWidth": 0,
-        "orientation": "horizontal",
-        "reduceOptions": {
-          "calcs": [
-            "last"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "showUnfilled": true,
-        "text": {},
-        "valueMode": "color"
-      },
-      "pluginVersion": "9.5.6",
-      "targets": [
-        {
-          "exemplar": true,
-          "expr": "topk(15, sum by (node_id) (scheduler_queue_size_per_node_work{instance=~\"$instance\"}/scheduler_mana_per_node{instance=~\"$instance\"}))",
-          "instant": true,
-          "interval": "",
-          "legendFormat": "Node: {{node_id}}",
-          "queryType": "randomWalk",
-          "refId": "A"
-        }
-      ],
-      "title": "Mana-scaled queue size",
-      "type": "bargauge"
-    },
     {
       "datasource": {
         "type": "prometheus",

deploy/ansible/roles/metrics/files/grafana/provisioning/dashboards/scheduler-metrics.json

@@ -46,75 +46,6 @@
       "title": "Scheduler metrics",
       "type": "row"
     },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          },
-          "unit": "none"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 6,
-        "w": 15,
-        "x": 0,
-        "y": 1
-      },
-      "id": 146,
-      "options": {
-        "displayMode": "lcd",
-        "minVizHeight": 10,
-        "minVizWidth": 0,
-        "orientation": "horizontal",
-        "reduceOptions": {
-          "calcs": [
-            "last"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "showUnfilled": true,
-        "text": {},
-        "valueMode": "color"
-      },
-      "pluginVersion": "9.5.6",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "editorMode": "code",
-          "exemplar": true,
-          "expr": "topk(15, sum by (issuer_id) (scheduler_queue_size_per_node_count{instance=~\"$instance\"}/scheduler_mana_per_node{instance=~\"$instance\"}))",
-          "hide": false,
-          "instant": true,
-          "interval": "",
-          "legendFormat": "Issuer: {{issuer_id}}",
-          "queryType": "randomWalk",
-          "refId": "A"
-        }
-      ],
-      "title": "Mana-scaled queue size",
-      "type": "bargauge"
-    },
     {
       "datasource": {
         "type": "prometheus",

go.mod

@@ -29,7 +29,7 @@ require (
 	github.com/iotaledger/hive.go/stringify v0.0.0-20240326102522-2e37ab3611a3
 	github.com/iotaledger/inx-app v1.0.0-rc.3.0.20240307101848-db58eb9353ec
 	github.com/iotaledger/inx/go v1.0.0-rc.2.0.20240307100839-48553e1d2022
-	github.com/iotaledger/iota.go/v4 v4.0.0-20240325092426-32979eef3205
+	github.com/iotaledger/iota.go/v4 v4.0.0-20240415115618-57e9e887bf49
 	github.com/labstack/echo/v4 v4.11.4
 	github.com/labstack/gommon v0.4.2
 	github.com/libp2p/go-libp2p v0.33.1

go.sum

@@ -326,8 +326,8 @@ github.com/iotaledger/inx/go v1.0.0-rc.2.0.20240307100839-48553e1d2022 h1:I178Sa
 github.com/iotaledger/inx/go v1.0.0-rc.2.0.20240307100839-48553e1d2022/go.mod h1:jTFxIWiMUdAwO263jlJCSWcNLqEkgYEVOFXfjp5aNJM=
 github.com/iotaledger/iota-crypto-demo v0.0.0-20240320124000-d02f37a4fdff h1:Do8fakxvFaj7dLckoo/z+mRyBdZo8QvT8HcgnQlG2Sg=
 github.com/iotaledger/iota-crypto-demo v0.0.0-20240320124000-d02f37a4fdff/go.mod h1:aVEutEWFnhDNJBxtVuzy2BeTN+8FAlnR83k7hKV0CFE=
-github.com/iotaledger/iota.go/v4 v4.0.0-20240325092426-32979eef3205 h1:nn7nCEezVLmFExiONAJ2XAgZKOJJ+iWrwfDBFdYTKSY=
-github.com/iotaledger/iota.go/v4 v4.0.0-20240325092426-32979eef3205/go.mod h1:qn/63CB0/jE1em6ewqDSiz+ovS+E/os7K5b7g2pmJFg=
+github.com/iotaledger/iota.go/v4 v4.0.0-20240415115618-57e9e887bf49 h1:1uYaqFeokRrmgkX813vYdn1KTLUGMa97OxJVcOfHm7c=
+github.com/iotaledger/iota.go/v4 v4.0.0-20240415115618-57e9e887bf49/go.mod h1:qn/63CB0/jE1em6ewqDSiz+ovS+E/os7K5b7g2pmJFg=
 github.com/ipfs/boxo v0.18.0 h1:MOL9/AgoV3e7jlVMInicaSdbgralfqSsbkc31dZ9tmw=
 github.com/ipfs/boxo v0.18.0/go.mod h1:pIZgTWdm3k3pLF9Uq6MB8JEcW07UDwNJjlXW1HELW80=
 github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s=

pkg/network/errors.go

@@ -11,8 +11,6 @@ var (
 	ErrLoopbackPeer = ierrors.New("loopback connection not allowed")
 	// ErrDuplicatePeer is returned when the same peer is added more than once.
 	ErrDuplicatePeer = ierrors.New("already connected")
-	// ErrFirstPacketNotReceived is returned when the first packet from a peer is not received.
-	ErrFirstPacketNotReceived = ierrors.New("first packet not received")
 	// ErrMaxAutopeeringPeersReached is returned when the maximum number of autopeering peers is reached.
 	ErrMaxAutopeeringPeersReached = ierrors.New("max autopeering peers reached")
 )