From 2afd17653d7f06a2ba790bdb8859f7d986134560 Mon Sep 17 00:00:00 2001 From: jkrvivian Date: Mon, 29 Apr 2024 22:40:10 +0800 Subject: [PATCH 1/7] Rename stopAdvertisingItNotNeeded to stopAdvertisingIfNotNeeded --- pkg/network/p2p/autopeering/autopeering.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/network/p2p/autopeering/autopeering.go b/pkg/network/p2p/autopeering/autopeering.go index 33af4c9a9..8ce950a28 100644 --- a/pkg/network/p2p/autopeering/autopeering.go +++ b/pkg/network/p2p/autopeering/autopeering.go @@ -118,7 +118,7 @@ func (m *Manager) Start(ctx context.Context, networkID string, bootstrapPeers [] }) onGossipNeighborAddedHook := m.networkManager.OnNeighborAdded(func(neighbor network.Neighbor) { m.logger.LogInfof("Gossip layer successfully connected with the peer %s", neighbor.Peer()) - m.stopAdvertisingItNotNeeded() + m.stopAdvertisingIfNotNeeded() }) m.stopFunc = func() { @@ -162,7 +162,7 @@ func (m *Manager) startAdvertisingIfNeeded() { } } -func (m *Manager) stopAdvertisingItNotNeeded() { +func (m *Manager) stopAdvertisingIfNotNeeded() { if len(m.networkManager.AutopeeringNeighbors()) >= m.maxPeers { m.stopAdvertising() } From a9be2be0fc9c5347001024c8206fdbf87bbb0378 Mon Sep 17 00:00:00 2001 From: muXxer Date: Tue, 30 Apr 2024 13:25:42 +0200 Subject: [PATCH 2/7] Use `SetAddrs` instead of `AddAddrs` in the libp2p peer store --- pkg/network/p2p/manager.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/network/p2p/manager.go b/pkg/network/p2p/manager.go index 26bc27f46..da101214a 100644 --- a/pkg/network/p2p/manager.go +++ b/pkg/network/p2p/manager.go @@ -124,8 +124,8 @@ func (m *Manager) DialPeer(ctx context.Context, peer *network.Peer) error { return ierrors.WithMessagef(network.ErrMaxAutopeeringPeersReached, "peer %s is not allowed", peer.ID.String()) } - // Adds the peer's multiaddresses to the peerstore, so that they can be used for dialing. - m.libp2pHost.Peerstore().AddAddrs(peer.ID, m.addrFilter(peer.PeerAddresses), peerstore.ConnectedAddrTTL) + // Set the current peer's multiaddresses to the peerstore, so that they can be used for dialing. + m.libp2pHost.Peerstore().SetAddrs(peer.ID, m.addrFilter(peer.PeerAddresses), 10*time.Minute) cancelCtx := ctx stream, err := m.P2PHost().NewStream(cancelCtx, peer.ID, network.CoreProtocolID) From 1ac4c2db81e6fd7c2af4d97b1cbb3f64dac5b618 Mon Sep 17 00:00:00 2001 From: muXxer Date: Tue, 30 Apr 2024 13:40:49 +0200 Subject: [PATCH 3/7] Remove unused peer database --- Dockerfile.dev | 2 +- components/p2p/component.go | 61 +---- components/p2p/params.go | 11 +- components/p2p/utils.go | 37 --- config_defaults.json | 4 +- .../templates/docker-compose-iota-core.yml.j2 | 2 +- documentation/configuration.md | 24 +- pkg/network/p2p/autopeering/autopeering.go | 19 +- pkg/network/p2p/manager.go | 21 +- pkg/network/peerdb.go | 217 ------------------ pkg/toolset/jwt.go | 23 +- pkg/toolset/p2p_identity_extract.go | 23 +- pkg/toolset/p2p_identity_gen.go | 26 +-- pkg/toolset/toolset.go | 6 +- tools/docker-network/.env | 2 +- 15 files changed, 57 insertions(+), 421 deletions(-) delete mode 100644 components/p2p/utils.go delete mode 100644 pkg/network/peerdb.go diff --git a/Dockerfile.dev b/Dockerfile.dev index 546f57027..e94a627ef 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -52,7 +52,7 @@ RUN --mount=type=cache,target=/go go build -o /app/iota-core -tags="$BUILD_TAGS" RUN cp ./config_defaults.json /app/config.json RUN cp ./peering.json /app/peering.json -RUN mkdir -p /app/data/peerdb +RUN mkdir -p /app/data/p2p ############################ # Runtime Image diff --git a/components/p2p/component.go b/components/p2p/component.go index 54a6cf7a2..322a5d4be 100644 --- a/components/p2p/component.go +++ b/components/p2p/component.go @@ -2,7 +2,6 @@ package p2p import ( "context" - "path/filepath" "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p/core/crypto" @@ -18,7 +17,6 @@ import ( hivep2p "github.com/iotaledger/hive.go/crypto/p2p" "github.com/iotaledger/hive.go/db" "github.com/iotaledger/hive.go/ierrors" - "github.com/iotaledger/hive.go/kvstore" "github.com/iotaledger/iota-core/pkg/daemon" "github.com/iotaledger/iota-core/pkg/network" "github.com/iotaledger/iota-core/pkg/network/p2p" @@ -47,21 +45,17 @@ type dependencies struct { PeeringConfig *configuration.Configuration `name:"peeringConfig"` PeeringConfigManager *p2p.ConfigManager NetworkManager network.Manager - PeerDB *network.DB Protocol *protocol.Protocol - PeerDBKVSTore kvstore.KVStore `name:"peerDBKVStore"` } func initConfigParams(c *dig.Container) error { type cfgResult struct { dig.Out - P2PDatabasePath string `name:"p2pDatabasePath"` P2PBindMultiAddresses []string `name:"p2pBindMultiAddresses"` } if err := c.Provide(func() cfgResult { return cfgResult{ - P2PDatabasePath: ParamsP2P.Database.Path, P2PBindMultiAddresses: ParamsP2P.BindMultiAddresses, } }); err != nil { @@ -72,27 +66,6 @@ func initConfigParams(c *dig.Container) error { } func provide(c *dig.Container) error { - type peerDatabaseResult struct { - dig.Out - - PeerDB *network.DB - PeerDBKVSTore kvstore.KVStore `name:"peerDBKVStore"` - } - - if err := c.Provide(func() peerDatabaseResult { - peerDB, peerDBKVStore, err := initPeerDB() - if err != nil { - Component.LogFatal(err.Error()) - } - - return peerDatabaseResult{ - PeerDB: peerDB, - PeerDBKVSTore: peerDBKVStore, - } - }); err != nil { - return err - } - type configManagerDeps struct { dig.In PeeringConfig *configuration.Configuration `name:"peeringConfig"` @@ -167,7 +140,6 @@ func provide(c *dig.Container) error { type p2pDeps struct { dig.In DatabaseEngine db.Engine `name:"databaseEngine"` - P2PDatabasePath string `name:"p2pDatabasePath"` P2PBindMultiAddresses []string `name:"p2pBindMultiAddresses"` } @@ -180,22 +152,20 @@ func provide(c *dig.Container) error { if err := c.Provide(func(deps p2pDeps) p2pResult { res := p2pResult{} - privKeyFilePath := filepath.Join(deps.P2PDatabasePath, IdentityPrivateKeyFileName) - // make sure nobody copies around the peer store since it contains the private key of the node - Component.LogInfof(`WARNING: never share your "%s" folder as it contains your node's private key!`, deps.P2PDatabasePath) + Component.LogInfof(`WARNING: never share the file "%s" as it contains your node's private key!`, ParamsP2P.IdentityPrivateKeyFilePath) // load up the previously generated identity or create a new one - nodePrivateKey, newlyCreated, err := hivep2p.LoadOrCreateIdentityPrivateKey(privKeyFilePath, ParamsP2P.IdentityPrivateKey) + nodePrivateKey, newlyCreated, err := hivep2p.LoadOrCreateIdentityPrivateKey(ParamsP2P.IdentityPrivateKeyFilePath, ParamsP2P.IdentityPrivateKey) if err != nil { Component.LogPanic(err.Error()) } res.NodePrivateKey = nodePrivateKey if newlyCreated { - Component.LogInfof(`stored new private key for peer identity under "%s"`, privKeyFilePath) + Component.LogInfof(`stored new private key for peer identity under "%s"`, ParamsP2P.IdentityPrivateKeyFilePath) } else { - Component.LogInfof(`loaded existing private key for peer identity from "%s"`, privKeyFilePath) + Component.LogInfof(`loaded existing private key for peer identity from "%s"`, ParamsP2P.IdentityPrivateKeyFilePath) } connManager, err := connmgr.NewConnManager( @@ -238,7 +208,6 @@ func provide(c *dig.Container) error { type p2pManagerDeps struct { dig.In Host host.Host - PeerDB *network.DB P2PMetrics *p2p.Metrics } @@ -247,31 +216,11 @@ func provide(c *dig.Container) error { inDeps.P2PMetrics.OutgoingBlocks.Add(1) } - return p2p.NewManager(Component.Logger, inDeps.Host, inDeps.PeerDB, ParamsP2P.Autopeering.MaxPeers, ParamsP2P.Autopeering.AllowLocalIPs, onBlockSentCallback) + return p2p.NewManager(Component.Logger, inDeps.Host, ParamsP2P.Autopeering.MaxPeers, ParamsP2P.Autopeering.AllowLocalIPs, onBlockSentCallback) }) } func configure() error { - if err := Component.Daemon().BackgroundWorker("Close p2p peer database", func(ctx context.Context) { - <-ctx.Done() - - closeDatabases := func() error { - if err := deps.PeerDBKVSTore.Flush(); err != nil { - return err - } - - return deps.PeerDBKVSTore.Close() - } - - Component.LogInfo("Syncing p2p peer database to disk ...") - if err := closeDatabases(); err != nil { - Component.LogPanicf("Syncing p2p peer database to disk ... failed: %s", err) - } - Component.LogInfo("Syncing p2p peer database to disk ... done") - }, daemon.PriorityCloseDatabase); err != nil { - Component.LogPanicf("failed to start worker: %s", err) - } - // log the p2p events deps.NetworkManager.OnNeighborAdded(func(neighbor network.Neighbor) { Component.LogInfof("neighbor added: %s / %s", neighbor.Peer().PeerAddresses, neighbor.Peer().ID) diff --git a/components/p2p/params.go b/components/p2p/params.go index 907df3678..7922ac484 100644 --- a/components/p2p/params.go +++ b/components/p2p/params.go @@ -6,8 +6,7 @@ import ( const ( // CfgPeers defines the static peers this node should retain a connection to (CLI). - CfgPeers = "peers" - IdentityPrivateKeyFileName = "identity.key" + CfgPeers = "peers" ) // ParametersP2P contains the definition of configuration parameters used by the p2p plugin. @@ -25,6 +24,9 @@ type ParametersP2P struct { // Defines the private key used to derive the node identity (optional). IdentityPrivateKey string `default:"" usage:"private key used to derive the node identity (optional)"` + // Defines the file path to the private key used to derive the node identity. + IdentityPrivateKeyFilePath string `default:"testnet/p2p/identity.key" usage:"the file path to the private key used to derive the node identity"` + Autopeering struct { // MaxPeers defines the max number of auto-peer connections. Set to 0 to disable auto-peering. MaxPeers int `default:"5" usage:"the max number of auto-peer connections. Set to 0 to disable auto-peering."` @@ -38,11 +40,6 @@ type ParametersP2P struct { // ExternalMultiAddress defines additional p2p multiaddresses to be advertised via DHT. ExternalMultiAddresses []string `default:"" usage:"external reacheable multi addresses advertised to the network"` } - - Database struct { - // Defines the path to the p2p database. - Path string `default:"testnet/p2pstore" usage:"the path to the p2p database"` - } `name:"db"` } // ParametersPeers contains the definition of the parameters used by peers. diff --git a/components/p2p/utils.go b/components/p2p/utils.go deleted file mode 100644 index 33c8ba802..000000000 --- a/components/p2p/utils.go +++ /dev/null @@ -1,37 +0,0 @@ -package p2p - -import ( - "path/filepath" - - "github.com/iotaledger/hive.go/ierrors" - "github.com/iotaledger/hive.go/kvstore" - "github.com/iotaledger/hive.go/kvstore/rocksdb" - "github.com/iotaledger/iota-core/pkg/network" - "github.com/iotaledger/iota-core/pkg/storage/database" -) - -// inits the peer database. -func initPeerDB() (peerDB *network.DB, peerDBKVStore kvstore.KVStore, err error) { - if err = checkValidPeerDBPath(); err != nil { - return nil, nil, ierrors.Wrap(err, "invalid peer database path") - } - - db, err := database.NewRocksDB(ParamsP2P.Database.Path) - if err != nil { - return nil, nil, ierrors.Wrap(err, "failed to create peer database") - } - - peerDBKVStore = rocksdb.New(db) - - return network.NewDB(peerDBKVStore), peerDBKVStore, nil -} - -// checks that the peer database path does not reside within the main database directory. -func checkValidPeerDBPath() error { - _, err := filepath.Abs(ParamsP2P.Database.Path) - if err != nil { - return ierrors.Wrapf(err, "cannot resolve absolute path of %s", ParamsP2P.Database.Path) - } - - return nil -} diff --git a/config_defaults.json b/config_defaults.json index f514d7c6f..680b8e63d 100644 --- a/config_defaults.json +++ b/config_defaults.json @@ -27,14 +27,12 @@ "lowWatermark": 5 }, "identityPrivateKey": "", + "identityPrivateKeyFilePath": "testnet/p2p/identity.key", "autopeering": { "maxPeers": 5, "bootstrapPeers": [], "allowLocalIPs": false, "externalMultiAddresses": [] - }, - "db": { - "path": "testnet/p2pstore" } }, "profiling": { diff --git a/deploy/ansible/roles/iota-core-node/templates/docker-compose-iota-core.yml.j2 b/deploy/ansible/roles/iota-core-node/templates/docker-compose-iota-core.yml.j2 index fca6c425e..8a669661d 100644 --- a/deploy/ansible/roles/iota-core-node/templates/docker-compose-iota-core.yml.j2 +++ b/deploy/ansible/roles/iota-core-node/templates/docker-compose-iota-core.yml.j2 @@ -41,7 +41,7 @@ services: --p2p.peers=/dns/node-01.feature/tcp/15600/p2p/12D3KooWCrjmh4dUCWfGVQT6ivzArieJB9Z3eKdy2mdEEN95NDPS --p2p.externalMultiAddresses={{ ips | join(',') }} --p2p.identityPrivateKey={{p2pIdentityPrvKey}} - --p2p.db.path=/app/data/peerdb + --p2p.identityPrivateKeyFilePath=/app/data/p2p/identity.key", --profiling.enabled=true --profiling.bindAddress=0.0.0.0:6061 --restAPI.bindAddress=0.0.0.0:14265 diff --git a/documentation/configuration.md b/documentation/configuration.md index b8230dbb2..a97d27714 100644 --- a/documentation/configuration.md +++ b/documentation/configuration.md @@ -93,13 +93,13 @@ Example: ## 3. Peer to Peer -| Name | Description | Type | Default value | -| ------------------------------------------- | ------------------------------------------------------- | ------ | -------------------------------------------- | -| bindMultiAddresses | The bind multi addresses for p2p connections | array | /ip4/0.0.0.0/tcp/15600
/ip6/::/tcp/15600 | -| [connectionManager](#p2p_connectionmanager) | Configuration for connectionManager | object | | -| identityPrivateKey | Private key used to derive the node identity (optional) | string | "" | -| [autopeering](#p2p_autopeering) | Configuration for autopeering | object | | -| [db](#p2p_db) | Configuration for Database | object | | +| Name | Description | Type | Default value | +| ------------------------------------------- | ----------------------------------------------------------------- | ------ | -------------------------------------------- | +| bindMultiAddresses | The bind multi addresses for p2p connections | array | /ip4/0.0.0.0/tcp/15600
/ip6/::/tcp/15600 | +| [connectionManager](#p2p_connectionmanager) | Configuration for connectionManager | object | | +| identityPrivateKey | Private key used to derive the node identity (optional) | string | "" | +| identityPrivateKeyFilePath | The file path to the private key used to derive the node identity | string | "testnet/p2p/identity.key" | +| [autopeering](#p2p_autopeering) | Configuration for autopeering | object | | ### ConnectionManager @@ -117,12 +117,6 @@ Example: | allowLocalIPs | Allow local IPs to be used for autopeering | boolean | false | | externalMultiAddresses | External reacheable multi addresses advertised to the network | array | | -### Database - -| Name | Description | Type | Default value | -| ---- | ---------------------------- | ------ | ------------------ | -| path | The path to the p2p database | string | "testnet/p2pstore" | - Example: ```json @@ -137,14 +131,12 @@ Example: "lowWatermark": 5 }, "identityPrivateKey": "", + "identityPrivateKeyFilePath": "testnet/p2p/identity.key", "autopeering": { "maxPeers": 5, "bootstrapPeers": [], "allowLocalIPs": false, "externalMultiAddresses": [] - }, - "db": { - "path": "testnet/p2pstore" } } } diff --git a/pkg/network/p2p/autopeering/autopeering.go b/pkg/network/p2p/autopeering/autopeering.go index 8ce950a28..4048554b3 100644 --- a/pkg/network/p2p/autopeering/autopeering.go +++ b/pkg/network/p2p/autopeering/autopeering.go @@ -27,7 +27,6 @@ type Manager struct { networkManager network.Manager logger log.Logger host host.Host - peerDB *network.DB startOnce sync.Once isStarted atomic.Bool stopOnce sync.Once @@ -42,12 +41,11 @@ type Manager struct { } // NewManager creates a new autopeering manager. -func NewManager(maxPeers int, networkManager network.Manager, host host.Host, peerDB *network.DB, addressFilter network.AddressFilter, logger log.Logger) *Manager { +func NewManager(maxPeers int, networkManager network.Manager, host host.Host, addressFilter network.AddressFilter, logger log.Logger) *Manager { return &Manager{ maxPeers: maxPeers, networkManager: networkManager, host: host, - peerDB: peerDB, logger: logger.NewChildLogger("Autopeering"), addrFilter: addressFilter, } @@ -94,21 +92,6 @@ func (m *Manager) Start(ctx context.Context, networkID string, bootstrapPeers [] m.ctx = dhtCtx - for _, seedPeer := range m.peerDB.SeedPeers() { - addrInfo := seedPeer.ToAddrInfo() - if innerErr := m.host.Connect(ctx, *addrInfo); innerErr != nil { - m.logger.LogInfof("Failed to connect to bootstrap node, peer: %s, error: %s", seedPeer, innerErr) - continue - } - - if _, innerErr := kademliaDHT.RoutingTable().TryAddPeer(addrInfo.ID, true, true); innerErr != nil { - m.logger.LogWarnf("Failed to add bootstrap node to routing table, error: %s", innerErr) - continue - } - - m.logger.LogDebugf("Connected to bootstrap node, peer: %s", seedPeer) - } - m.routingDiscovery = routing.NewRoutingDiscovery(kademliaDHT) m.startAdvertisingIfNeeded() go m.discoveryLoop() diff --git a/pkg/network/p2p/manager.go b/pkg/network/p2p/manager.go index da101214a..ceffb9fe4 100644 --- a/pkg/network/p2p/manager.go +++ b/pkg/network/p2p/manager.go @@ -2,11 +2,11 @@ package p2p import ( "context" + "time" "github.com/libp2p/go-libp2p/core/host" p2pnetwork "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" - "github.com/libp2p/go-libp2p/core/peerstore" "github.com/multiformats/go-multiaddr" "google.golang.org/protobuf/proto" @@ -37,7 +37,6 @@ type Manager struct { neighborRemoved *event.Event1[network.Neighbor] libp2pHost host.Host - peerDB *network.DB ctx context.Context @@ -58,11 +57,10 @@ type Manager struct { var _ network.Manager = (*Manager)(nil) // NewManager creates a new Manager. -func NewManager(logger log.Logger, libp2pHost host.Host, peerDB *network.DB, maxAutopeeringPeers int, allowLocalAutopeering bool, onBlockSentCallback func()) *Manager { +func NewManager(logger log.Logger, libp2pHost host.Host, maxAutopeeringPeers int, allowLocalAutopeering bool, onBlockSentCallback func()) *Manager { m := &Manager{ logger: logger, libp2pHost: libp2pHost, - peerDB: peerDB, neighborAdded: event.New1[network.Neighbor](), neighborRemoved: event.New1[network.Neighbor](), neighbors: shrinkingmap.New[peer.ID, *neighbor](), @@ -70,7 +68,7 @@ func NewManager(logger log.Logger, libp2pHost host.Host, peerDB *network.DB, max addrFilter: network.PublicOnlyAddressesFilter(allowLocalAutopeering), } - m.autoPeering = autopeering.NewManager(maxAutopeeringPeers, m, libp2pHost, peerDB, m.addrFilter, logger) + m.autoPeering = autopeering.NewManager(maxAutopeeringPeers, m, libp2pHost, m.addrFilter, logger) m.manualPeering = manualpeering.NewManager(m, logger) return m @@ -142,12 +140,6 @@ func (m *Manager) DialPeer(ctx context.Context, peer *network.Peer) error { m.logger.LogDebugf("outgoing stream negotiated, id: %s, addr: %s, proto: %s", peer.ID, ps.Conn().RemoteMultiaddr(), network.CoreProtocolID) - if err := m.peerDB.UpdatePeer(peer); err != nil { - m.closeStream(stream) - - return ierrors.Wrapf(err, "failed to update peer %s", peer.ID.String()) - } - if err := m.addNeighbor(peer, ps, m.onBlockSentCallback); err != nil { m.closeStream(stream) @@ -350,13 +342,6 @@ func (m *Manager) handleStream(stream p2pnetwork.Stream) { } networkPeer := network.NewPeerFromAddrInfo(peerAddrInfo) - if err := m.peerDB.UpdatePeer(networkPeer); err != nil { - m.logger.LogErrorf("failed to update peer in peer database, peerID: %s, error: %s", peerID.String(), err.Error()) - m.closeStream(stream) - - return - } - if err := m.addNeighbor(networkPeer, ps, m.onBlockSentCallback); err != nil { m.logger.LogErrorf("failed to add neighbor, peerID: %s, error: %s", peerID.String(), err.Error()) m.closeStream(stream) diff --git a/pkg/network/peerdb.go b/pkg/network/peerdb.go deleted file mode 100644 index cd7ba7b91..000000000 --- a/pkg/network/peerdb.go +++ /dev/null @@ -1,217 +0,0 @@ -package network - -import ( - "bytes" - "encoding/binary" - "math/rand" - "sync" - "time" - - "github.com/libp2p/go-libp2p/core/peer" - - "github.com/iotaledger/hive.go/kvstore" -) - -const ( - // remove peers from DB, when the last received ping was older than this. - peerExpiration = 30 * 24 * time.Hour - // interval in which expired peers are checked. - cleanupInterval = time.Hour - // number of peers used for bootstrapping. - seedCount = 10 -) - -// DB is the peer database, storing previously seen peers and any collected properties of them. -type DB struct { - store kvstore.KVStore - expirerWg sync.WaitGroup - quit chan struct{} // Channel to signal the expiring thread to stop -} - -// Keys in the node database. -const ( - dbNodePrefix = "n:" // Identifier to prefix node entries with - - dbNodeUpdated = "updated" -) - -// NewDB creates a new peer database. -func NewDB(store kvstore.KVStore) *DB { - pDB := &DB{ - store: store, - quit: make(chan struct{}), - } - - pDB.expirerWg.Add(1) - go pDB.expirer() - - return pDB -} - -// UpdatePeer updates a peer in the database. -func (db *DB) UpdatePeer(p *Peer) error { - data, err := p.Bytes() - if err != nil { - return err - } - - if err := db.store.Set(nodeKey(p.ID), data); err != nil { - return err - } - - if err := db.setInt64(nodeFieldKey(p.ID, dbNodeUpdated), time.Now().Unix()); err != nil { - return err - } - - return db.store.Flush() -} - -// Peer retrieves a peer from the database. -func (db *DB) Peer(id peer.ID) (*Peer, error) { - data, err := db.store.Get(nodeKey(id)) - if err != nil { - return nil, err - } - - return peerFromBytes(data) -} - -// SeedPeers retrieves random nodes to be used as potential bootstrap peers. -func (db *DB) SeedPeers() []*Peer { - return randomSubset(db.getPeers(), seedCount) -} - -// Close closes the peer database. -func (db *DB) Close() { - close(db.quit) - db.expirerWg.Wait() -} - -// expirer should be started in a go routine, and is responsible for looping ad -// infinitum and dropping stale data from the database. -func (db *DB) expirer() { - tick := time.NewTicker(cleanupInterval) - defer tick.Stop() - defer db.expirerWg.Done() - - for { - select { - case <-tick.C: - _ = db.expireNodes() - case <-db.quit: - return - } - } -} - -// expireNodes iterates over the database and deletes all nodes that have not -// been seen (i.e. received a pong from) for some time. -func (db *DB) expireNodes() error { - threshold := time.Now().Add(-peerExpiration).Unix() - batchedMuts, err := db.store.Batched() - if err != nil { - return err - } - - var innerErr error - if err := db.store.Iterate(kvstore.KeyPrefix(dbNodePrefix), func(key kvstore.Key, value kvstore.Value) bool { - if bytes.HasSuffix(key, []byte(dbNodeUpdated)) { - // if the peer has been updated before the threshold we expire it - if parseInt64(value) < threshold { - // delete update field of the peer - if err := batchedMuts.Delete(key); err != nil { - innerErr = err - - return false - } - - // delete peer - if err := batchedMuts.Delete(key[:len(key)-len(dbNodeUpdated)]); err != nil { - innerErr = err - - return false - } - } - } - - return true - }); err != nil { - batchedMuts.Cancel() - - return err - } - - if innerErr != nil { - batchedMuts.Cancel() - - return innerErr - } - - if err := batchedMuts.Commit(); err != nil { - return err - } - - return db.store.Flush() -} - -func (db *DB) getPeers() (peers []*Peer) { - if err := db.store.Iterate(kvstore.KeyPrefix(dbNodePrefix), func(key kvstore.Key, value kvstore.Value) bool { - // skip update fields - if bytes.HasSuffix(key, []byte(dbNodeUpdated)) { - return true - } - - if p, err := peerFromBytes(value); err == nil { - peers = append(peers, p) - } - - return true - }); err != nil { - return nil - } - - return peers -} - -// setInt64 stores an integer in the given key. -func (db *DB) setInt64(key []byte, n int64) error { - blob := make([]byte, binary.MaxVarintLen64) - blob = blob[:binary.PutVarint(blob, n)] - - return db.store.Set(key, blob) -} - -// nodeKey returns the database key for a node record. -func nodeKey(id peer.ID) []byte { - return append([]byte(dbNodePrefix), []byte(id)...) -} - -// nodeFieldKey returns the database key for a node metadata field. -func nodeFieldKey(id peer.ID, field string) []byte { - return append(nodeKey(id), []byte(field)...) -} - -func parseInt64(blob []byte) int64 { - val, read := binary.Varint(blob) - if read <= 0 { - return 0 - } - - return val -} - -func randomSubset(peers []*Peer, m int) []*Peer { - if len(peers) <= m { - return peers - } - - result := make([]*Peer, 0, m) - for i, p := range peers { - //nolint:gosec // we do not care about weak random numbers here - if rand.Intn(len(peers)-i) < m-len(result) { - result = append(result, p) - } - } - - return result -} diff --git a/pkg/toolset/jwt.go b/pkg/toolset/jwt.go index 4e7bbd4b8..b7c707ae8 100644 --- a/pkg/toolset/jwt.go +++ b/pkg/toolset/jwt.go @@ -3,7 +3,6 @@ package toolset import ( "fmt" "os" - "path/filepath" "github.com/libp2p/go-libp2p/core/peer" flag "github.com/spf13/pflag" @@ -12,13 +11,12 @@ import ( hivep2p "github.com/iotaledger/hive.go/crypto/p2p" "github.com/iotaledger/hive.go/crypto/pem" "github.com/iotaledger/hive.go/ierrors" - "github.com/iotaledger/iota-core/components/p2p" "github.com/iotaledger/iota-core/pkg/jwt" ) func generateJWTApiToken(args []string) error { fs := configuration.NewUnsortedFlagSet("", flag.ContinueOnError) - databasePathFlag := fs.String(FlagToolDatabasePath, DefaultValueP2PDatabasePath, "the path to the p2p database folder") + privKeyFilePath := fs.String(FlagToolIdentityPrivateKeyFilePath, DefaultValueIdentityPrivateKeyFilePath, "the file path to the identity private key file") apiJWTSaltFlag := fs.String(FlagToolSalt, DefaultValueAPIJWTTokenSalt, "salt used inside the JWT tokens for the REST API") outputJSONFlag := fs.Bool(FlagToolOutputJSON, false, FlagToolDescriptionOutputJSON) @@ -27,8 +25,8 @@ func generateJWTApiToken(args []string) error { fs.PrintDefaults() println(fmt.Sprintf("\nexample: %s --%s %s --%s %s", ToolJWTApi, - FlagToolDatabasePath, - DefaultValueP2PDatabasePath, + FlagToolIdentityPrivateKeyFilePath, + DefaultValueIdentityPrivateKeyFilePath, FlagToolSalt, DefaultValueAPIJWTTokenSalt)) } @@ -37,32 +35,29 @@ func generateJWTApiToken(args []string) error { return err } - if len(*databasePathFlag) == 0 { - return ierrors.Errorf("'%s' not specified", FlagToolDatabasePath) + if len(*privKeyFilePath) == 0 { + return ierrors.Errorf("'%s' not specified", FlagToolIdentityPrivateKeyFilePath) } if len(*apiJWTSaltFlag) == 0 { return ierrors.Errorf("'%s' not specified", FlagToolSalt) } - databasePath := *databasePathFlag - privKeyFilePath := filepath.Join(databasePath, p2p.IdentityPrivateKeyFileName) - salt := *apiJWTSaltFlag - _, err := os.Stat(privKeyFilePath) + _, err := os.Stat(*privKeyFilePath) switch { case os.IsNotExist(err): // private key does not exist - return ierrors.Errorf("private key file (%s) does not exist", privKeyFilePath) + return ierrors.Errorf("private key file (%s) does not exist", *privKeyFilePath) case err == nil || os.IsExist(err): // private key file exists default: - return ierrors.Wrapf(err, "unable to check private key file (%s)", privKeyFilePath) + return ierrors.Wrapf(err, "unable to check private key file (%s)", *privKeyFilePath) } - privKey, err := pem.ReadEd25519PrivateKeyFromPEMFile(privKeyFilePath) + privKey, err := pem.ReadEd25519PrivateKeyFromPEMFile(*privKeyFilePath) if err != nil { return ierrors.Wrap(err, "reading private key file for peer identity failed") } diff --git a/pkg/toolset/p2p_identity_extract.go b/pkg/toolset/p2p_identity_extract.go index 5e67ded89..4c5a4a6ae 100644 --- a/pkg/toolset/p2p_identity_extract.go +++ b/pkg/toolset/p2p_identity_extract.go @@ -3,7 +3,6 @@ package toolset import ( "fmt" "os" - "path/filepath" flag "github.com/spf13/pflag" @@ -11,12 +10,11 @@ import ( hivep2p "github.com/iotaledger/hive.go/crypto/p2p" "github.com/iotaledger/hive.go/crypto/pem" "github.com/iotaledger/hive.go/ierrors" - "github.com/iotaledger/iota-core/components/p2p" ) func extractP2PIdentity(args []string) error { fs := configuration.NewUnsortedFlagSet("", flag.ContinueOnError) - databasePathFlag := fs.String(FlagToolDatabasePath, DefaultValueP2PDatabasePath, "the path to the p2p database folder") + privKeyFilePath := fs.String(FlagToolIdentityPrivateKeyFilePath, DefaultValueIdentityPrivateKeyFilePath, "the file path to the identity private key file") outputJSONFlag := fs.Bool(FlagToolOutputJSON, false, FlagToolDescriptionOutputJSON) fs.Usage = func() { @@ -24,35 +22,32 @@ func extractP2PIdentity(args []string) error { fs.PrintDefaults() println(fmt.Sprintf("\nexample: %s --%s %s", ToolP2PExtractIdentity, - FlagToolDatabasePath, - DefaultValueP2PDatabasePath)) + FlagToolIdentityPrivateKeyFilePath, + DefaultValueIdentityPrivateKeyFilePath)) } if err := parseFlagSet(fs, args); err != nil { return err } - if len(*databasePathFlag) == 0 { - return ierrors.Errorf("'%s' not specified", FlagToolDatabasePath) + if len(*privKeyFilePath) == 0 { + return ierrors.Errorf("'%s' not specified", FlagToolIdentityPrivateKeyFilePath) } - databasePath := *databasePathFlag - privKeyFilePath := filepath.Join(databasePath, p2p.IdentityPrivateKeyFileName) - - _, err := os.Stat(privKeyFilePath) + _, err := os.Stat(*privKeyFilePath) switch { case os.IsNotExist(err): // private key does not exist - return ierrors.Errorf("private key file (%s) does not exist", privKeyFilePath) + return ierrors.Errorf("private key file (%s) does not exist", *privKeyFilePath) case err == nil || os.IsExist(err): // private key file exists default: - return ierrors.Wrapf(err, "unable to check private key file (%s)", privKeyFilePath) + return ierrors.Wrapf(err, "unable to check private key file (%s)", *privKeyFilePath) } - privKey, err := pem.ReadEd25519PrivateKeyFromPEMFile(privKeyFilePath) + privKey, err := pem.ReadEd25519PrivateKeyFromPEMFile(*privKeyFilePath) if err != nil { return ierrors.Wrap(err, "reading private key file for peer identity failed") } diff --git a/pkg/toolset/p2p_identity_gen.go b/pkg/toolset/p2p_identity_gen.go index c6a68f567..4a07dbc81 100644 --- a/pkg/toolset/p2p_identity_gen.go +++ b/pkg/toolset/p2p_identity_gen.go @@ -16,13 +16,12 @@ import ( hivecrypto "github.com/iotaledger/hive.go/crypto" "github.com/iotaledger/hive.go/crypto/pem" "github.com/iotaledger/hive.go/ierrors" - "github.com/iotaledger/iota-core/components/p2p" "github.com/iotaledger/iota.go/v4/hexutil" ) func generateP2PIdentity(args []string) error { fs := configuration.NewUnsortedFlagSet("", flag.ContinueOnError) - databasePathFlag := fs.String(FlagToolOutputPath, DefaultValueP2PDatabasePath, "the path to the output folder") + privKeyFilePath := fs.String(FlagToolIdentityPrivateKeyFilePath, DefaultValueIdentityPrivateKeyFilePath, "the file path to the identity private key file") privateKeyFlag := fs.String(FlagToolPrivateKey, "", "the p2p private key") outputJSONFlag := fs.Bool(FlagToolOutputJSON, false, FlagToolDescriptionOutputJSON) @@ -31,8 +30,8 @@ func generateP2PIdentity(args []string) error { fs.PrintDefaults() println(fmt.Sprintf("\nexample: %s --%s %s --%s %s", ToolP2PIdentityGen, - FlagToolDatabasePath, - DefaultValueP2PDatabasePath, + FlagToolIdentityPrivateKeyFilePath, + DefaultValueIdentityPrivateKeyFilePath, FlagToolPrivateKey, "[PRIVATE_KEY]", )) @@ -42,28 +41,25 @@ func generateP2PIdentity(args []string) error { return err } - if len(*databasePathFlag) == 0 { - return ierrors.Errorf("'%s' not specified", FlagToolDatabasePath) + if len(*privKeyFilePath) == 0 { + return ierrors.Errorf("'%s' not specified", FlagToolIdentityPrivateKeyFilePath) } - databasePath := *databasePathFlag - privKeyFilePath := filepath.Join(databasePath, p2p.IdentityPrivateKeyFileName) - - if err := os.MkdirAll(databasePath, 0700); err != nil { - return ierrors.Wrapf(err, "could not create peer store database dir '%s'", databasePath) + if err := os.MkdirAll(filepath.Dir(*privKeyFilePath), 0700); err != nil { + return ierrors.Wrapf(err, "could not create peer store database dir '%s'", filepath.Dir(*privKeyFilePath)) } - _, err := os.Stat(privKeyFilePath) + _, err := os.Stat(*privKeyFilePath) switch { case err == nil || os.IsExist(err): // private key file already exists - return ierrors.Errorf("private key file (%s) already exists", privKeyFilePath) + return ierrors.Errorf("private key file (%s) already exists", *privKeyFilePath) case os.IsNotExist(err): // private key file does not exist, create a new one default: - return ierrors.Wrapf(err, "unable to check private key file (%s)", privKeyFilePath) + return ierrors.Wrapf(err, "unable to check private key file (%s)", *privKeyFilePath) } var privKey ed25519.PrivateKey @@ -85,7 +81,7 @@ func generateP2PIdentity(args []string) error { return ierrors.Wrapf(err, "unable to convert given private key '%s'", hexutil.EncodeHex(privKey)) } - if err := pem.WriteEd25519PrivateKeyToPEMFile(privKeyFilePath, privKey); err != nil { + if err := pem.WriteEd25519PrivateKeyToPEMFile(*privKeyFilePath, privKey); err != nil { return ierrors.Wrap(err, "writing private key file for peer identity failed") } diff --git a/pkg/toolset/toolset.go b/pkg/toolset/toolset.go index 96f4b9d56..507373445 100644 --- a/pkg/toolset/toolset.go +++ b/pkg/toolset/toolset.go @@ -12,7 +12,7 @@ import ( ) const ( - FlagToolDatabasePath = "databasePath" + FlagToolIdentityPrivateKeyFilePath = "identityPrivateKeyFilePath" FlagToolOutputPath = "outputPath" @@ -40,8 +40,8 @@ const ( ) const ( - DefaultValueAPIJWTTokenSalt = "IOTA" - DefaultValueP2PDatabasePath = "testnet/p2pstore" + DefaultValueAPIJWTTokenSalt = "IOTA" + DefaultValueIdentityPrivateKeyFilePath = "testnet/p2p/identity.key" ) // ShouldHandleTools checks if tools were requested. diff --git a/tools/docker-network/.env b/tools/docker-network/.env index e965fe92e..92dd849c2 100644 --- a/tools/docker-network/.env +++ b/tools/docker-network/.env @@ -3,7 +3,7 @@ COMMON_CONFIG=" --logger.level=debug --logger.outputPaths=stdout --retainer.debugStoreErrorMessages=true ---p2p.db.path=/app/data/peerdb +--p2p.identityPrivateKeyFilePath=/app/data/p2p/identity.key --profiling.enabled=true --profiling.bindAddress=0.0.0.0:6061 --db.path=/app/data/database From f46e21dd9d47d4bc31aff6dfa348983d7d7e58d6 Mon Sep 17 00:00:00 2001 From: muXxer Date: Tue, 30 Apr 2024 13:41:05 +0200 Subject: [PATCH 4/7] Add MaxRecordAge for DHT entries --- pkg/network/p2p/autopeering/autopeering.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/network/p2p/autopeering/autopeering.go b/pkg/network/p2p/autopeering/autopeering.go index 4048554b3..7693b1967 100644 --- a/pkg/network/p2p/autopeering/autopeering.go +++ b/pkg/network/p2p/autopeering/autopeering.go @@ -73,6 +73,7 @@ func (m *Manager) Start(ctx context.Context, networkID string, bootstrapPeers [] dht.ProtocolExtension(extension), dht.AddressFilter(m.addrFilter), dht.BootstrapPeers(bootstrapPeers...), + dht.MaxRecordAge(10*time.Minute), ) if innerErr != nil { err = innerErr From 17ae11b3d53f19405406926b803a134cc1e21756 Mon Sep 17 00:00:00 2001 From: muXxer Date: Tue, 30 Apr 2024 13:50:59 +0200 Subject: [PATCH 5/7] Remove unused peer serialization code --- pkg/network/peer.go | 77 --------------------------------------------- 1 file changed, 77 deletions(-) diff --git a/pkg/network/peer.go b/pkg/network/peer.go index 5d32b9241..478184040 100644 --- a/pkg/network/peer.go +++ b/pkg/network/peer.go @@ -10,8 +10,6 @@ import ( "github.com/iotaledger/hive.go/crypto/ed25519" "github.com/iotaledger/hive.go/ierrors" - "github.com/iotaledger/hive.go/serializer/v2" - "github.com/iotaledger/hive.go/serializer/v2/stream" ) const DefaultReconnectInterval = 5 * time.Second @@ -74,81 +72,6 @@ func (p *Peer) SetConnStatus(cs ConnectionStatus) { p.ConnStatus.Store(cs) } -func (p *Peer) Bytes() ([]byte, error) { - byteBuffer := stream.NewByteBuffer() - - if err := stream.WriteObjectWithSize(byteBuffer, p.ID, serializer.SeriLengthPrefixTypeAsUint16, func(id peer.ID) ([]byte, error) { - return []byte(id), nil - }); err != nil { - return nil, ierrors.Wrap(err, "failed to write peer ID") - } - - if err := stream.WriteCollection(byteBuffer, serializer.SeriLengthPrefixTypeAsByte, func() (elementsCount int, err error) { - for _, addr := range p.PeerAddresses { - if err = stream.WriteObjectWithSize(byteBuffer, addr, serializer.SeriLengthPrefixTypeAsUint16, func(m multiaddr.Multiaddr) ([]byte, error) { - return m.Bytes(), nil - }); err != nil { - return 0, ierrors.Wrap(err, "failed to write peer address") - } - } - - return len(p.PeerAddresses), nil - }); err != nil { - return nil, ierrors.Wrap(err, "failed to write peer addresses") - } - - return byteBuffer.Bytes() -} - func (p *Peer) String() string { return fmt.Sprintf("Peer{ID: %s, Addrs: %v, ConnStatus: %s}", p.ID, p.PeerAddresses, p.GetConnStatus()) } - -// peerFromBytes parses a peer from a byte slice. -func peerFromBytes(bytes []byte) (*Peer, error) { - p := &Peer{ - PeerAddresses: make([]multiaddr.Multiaddr, 0), - ConnStatus: &atomic.Value{}, - RemoveCh: make(chan struct{}), - DoneCh: make(chan struct{}), - } - - var err error - byteReader := stream.NewByteReader(bytes) - - if p.ID, err = stream.ReadObjectWithSize(byteReader, serializer.SeriLengthPrefixTypeAsUint16, func(bytes []byte) (peer.ID, int, error) { - id, err := peer.IDFromBytes(bytes) - if err != nil { - return "", 0, ierrors.Wrap(err, "failed to parse peerID") - } - - return id, len(bytes), nil - }); err != nil { - return nil, ierrors.Wrap(err, "failed to read peer ID") - } - - p.SetConnStatus(ConnStatusDisconnected) - - //nolint:revive - if err = stream.ReadCollection(byteReader, serializer.SeriLengthPrefixTypeAsByte, func(i int) error { - addr, err := stream.ReadObjectWithSize(byteReader, serializer.SeriLengthPrefixTypeAsUint16, func(bytes []byte) (multiaddr.Multiaddr, int, error) { - m, err := multiaddr.NewMultiaddrBytes(bytes) - if err != nil { - return nil, 0, ierrors.Wrap(err, "failed to parse peer address") - } - - return m, len(bytes), nil - }) - if err != nil { - return ierrors.Wrap(err, "failed to read peer address") - } - - p.PeerAddresses = append(p.PeerAddresses, addr) - - return nil - }); err != nil { - return nil, ierrors.Wrap(err, "failed to read peer addresses") - } - - return p, nil -} From ab9f161c98f267248aee305ccd83d4faaa857aa6 Mon Sep 17 00:00:00 2001 From: muXxer Date: Tue, 30 Apr 2024 13:51:24 +0200 Subject: [PATCH 6/7] Remove unused provided config params --- components/p2p/component.go | 39 +++++++------------------------------ 1 file changed, 7 insertions(+), 32 deletions(-) diff --git a/components/p2p/component.go b/components/p2p/component.go index 322a5d4be..2b53f1094 100644 --- a/components/p2p/component.go +++ b/components/p2p/component.go @@ -15,7 +15,6 @@ import ( "github.com/iotaledger/hive.go/app" "github.com/iotaledger/hive.go/app/configuration" hivep2p "github.com/iotaledger/hive.go/crypto/p2p" - "github.com/iotaledger/hive.go/db" "github.com/iotaledger/hive.go/ierrors" "github.com/iotaledger/iota-core/pkg/daemon" "github.com/iotaledger/iota-core/pkg/network" @@ -25,13 +24,12 @@ import ( func init() { Component = &app.Component{ - Name: "P2P", - DepsFunc: func(cDeps dependencies) { deps = cDeps }, - Params: params, - InitConfigParams: initConfigParams, - Provide: provide, - Configure: configure, - Run: run, + Name: "P2P", + DepsFunc: func(cDeps dependencies) { deps = cDeps }, + Params: params, + Provide: provide, + Configure: configure, + Run: run, } } @@ -48,23 +46,6 @@ type dependencies struct { Protocol *protocol.Protocol } -func initConfigParams(c *dig.Container) error { - type cfgResult struct { - dig.Out - P2PBindMultiAddresses []string `name:"p2pBindMultiAddresses"` - } - - if err := c.Provide(func() cfgResult { - return cfgResult{ - P2PBindMultiAddresses: ParamsP2P.BindMultiAddresses, - } - }); err != nil { - Component.LogPanic(err.Error()) - } - - return nil -} - func provide(c *dig.Container) error { type configManagerDeps struct { dig.In @@ -137,19 +118,13 @@ func provide(c *dig.Container) error { Component.LogPanic(err.Error()) } - type p2pDeps struct { - dig.In - DatabaseEngine db.Engine `name:"databaseEngine"` - P2PBindMultiAddresses []string `name:"p2pBindMultiAddresses"` - } - type p2pResult struct { dig.Out NodePrivateKey crypto.PrivKey `name:"nodePrivateKey"` Host host.Host } - if err := c.Provide(func(deps p2pDeps) p2pResult { + if err := c.Provide(func() p2pResult { res := p2pResult{} // make sure nobody copies around the peer store since it contains the private key of the node From 756e3f70f1d7c382356304b3042619c475673285 Mon Sep 17 00:00:00 2001 From: muXxer Date: Tue, 30 Apr 2024 14:10:51 +0200 Subject: [PATCH 7/7] Correctly handle Peerstore entries --- pkg/network/p2p/manager.go | 2 -- pkg/network/p2p/manualpeering/manualpeering.go | 5 +++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/network/p2p/manager.go b/pkg/network/p2p/manager.go index ceffb9fe4..31ad21cc2 100644 --- a/pkg/network/p2p/manager.go +++ b/pkg/network/p2p/manager.go @@ -122,8 +122,6 @@ func (m *Manager) DialPeer(ctx context.Context, peer *network.Peer) error { return ierrors.WithMessagef(network.ErrMaxAutopeeringPeersReached, "peer %s is not allowed", peer.ID.String()) } - // Set the current peer's multiaddresses to the peerstore, so that they can be used for dialing. - m.libp2pHost.Peerstore().SetAddrs(peer.ID, m.addrFilter(peer.PeerAddresses), 10*time.Minute) cancelCtx := ctx stream, err := m.P2PHost().NewStream(cancelCtx, peer.ID, network.CoreProtocolID) diff --git a/pkg/network/p2p/manualpeering/manualpeering.go b/pkg/network/p2p/manualpeering/manualpeering.go index 6280d8fdb..f980491c6 100644 --- a/pkg/network/p2p/manualpeering/manualpeering.go +++ b/pkg/network/p2p/manualpeering/manualpeering.go @@ -79,6 +79,8 @@ func (m *Manager) RemovePeer(peerID peer.ID) error { return ierrors.Wrapf(err, "failed to drop known peer %s in the gossip layer", peerID.String()) } + m.networkManager.P2PHost().Peerstore().RemovePeer(peerID) + return nil } @@ -179,6 +181,9 @@ func (m *Manager) AddPeer(multiAddr multiaddr.Multiaddr) (*network.Peer, error) return peer, nil } + // Set the current peer's multiaddresses to the peerstore, so that they can be used for dialing. + m.networkManager.P2PHost().Peerstore().SetAddrs(newPeer.ID, newPeer.PeerAddresses, 10*time.Minute) + m.logger.LogInfof("Adding new peer to the list of known peers in manual peering %s", newPeer) m.knownPeers[newPeer.ID] = newPeer