diff --git a/pkg/network/p2p/manager.go b/pkg/network/p2p/manager.go
index ceffb9fe4..31ad21cc2 100644
--- a/pkg/network/p2p/manager.go
+++ b/pkg/network/p2p/manager.go
@@ -122,8 +122,6 @@ func (m *Manager) DialPeer(ctx context.Context, peer *network.Peer) error {
 		return ierrors.WithMessagef(network.ErrMaxAutopeeringPeersReached, "peer %s is not allowed", peer.ID.String())
 	}
 
-	// Set the current peer's multiaddresses to the peerstore, so that they can be used for dialing.
-	m.libp2pHost.Peerstore().SetAddrs(peer.ID, m.addrFilter(peer.PeerAddresses), 10*time.Minute)
 	cancelCtx := ctx
 
 	stream, err := m.P2PHost().NewStream(cancelCtx, peer.ID, network.CoreProtocolID)
diff --git a/pkg/network/p2p/manualpeering/manualpeering.go b/pkg/network/p2p/manualpeering/manualpeering.go
index 6280d8fdb..f980491c6 100644
--- a/pkg/network/p2p/manualpeering/manualpeering.go
+++ b/pkg/network/p2p/manualpeering/manualpeering.go
@@ -79,6 +79,8 @@ func (m *Manager) RemovePeer(peerID peer.ID) error {
 		return ierrors.Wrapf(err, "failed to drop known peer %s in the gossip layer", peerID.String())
 	}
 
+	m.networkManager.P2PHost().Peerstore().RemovePeer(peerID)
+
 	return nil
 }
 
@@ -179,6 +181,9 @@ func (m *Manager) AddPeer(multiAddr multiaddr.Multiaddr) (*network.Peer, error)
 		return peer, nil
 	}
 
+	// Set the current peer's multiaddresses to the peerstore, so that they can be used for dialing.
+	m.networkManager.P2PHost().Peerstore().SetAddrs(newPeer.ID, newPeer.PeerAddresses, 10*time.Minute)
+
 	m.logger.LogInfof("Adding new peer to the list of known peers in manual peering %s", newPeer)
 	m.knownPeers[newPeer.ID] = newPeer