Add explore state machine to expand population of routing table (#934)

* Improve query capabilities

* go mod tidy

* Review feedback

* go mod tidy

* Move coord packages to internal (#933)

* Move coord and kadt packages to internal

* go mod tidy

* go fmt

* Move kadt out of internal and add RoutingTable interface

* Add explore state machine to expand population of routing table

* Refactor schedule into separate type

* Add generation of random peer id for a given cpl

* go mod tidy

* Add prefixmap generator

* Use constants for various query ids

* go mod tidy

* Wire explore state machine into routing behaviour

* Remove some unnecessary conversions

* PR review updates

v2/go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/libp2p/go-msgio v0.3.0
 	github.com/multiformats/go-base32 v0.1.0
 	github.com/multiformats/go-multiaddr v0.11.0
-	github.com/multiformats/go-multihash v0.2.3 // indirect
+	github.com/multiformats/go-multihash v0.2.3
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/plprobelab/go-kademlia v0.0.0-20230913171354-443ec1f56080
 	github.com/prometheus/client_golang v1.16.0 // indirect

v2/internal/coord/behaviour_test.go

@@ -4,13 +4,6 @@ import (
 	"context"
 )
 
-type NullSM[E any, S any] struct{}
-
-func (NullSM[E, S]) Advance(context.Context, E) S {
-	var v S
-	return v
-}
-
 type RecordingSM[E any, S any] struct {
 	State    S
 	Received E

v2/internal/coord/coordinator.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/libp2p/go-libp2p-kad-dht/v2/internal/coord/brdcst"
 	"github.com/libp2p/go-libp2p-kad-dht/v2/internal/coord/coordt"
+	"github.com/libp2p/go-libp2p-kad-dht/v2/internal/coord/cplutil"
 	"github.com/libp2p/go-libp2p-kad-dht/v2/internal/coord/query"
 	"github.com/libp2p/go-libp2p-kad-dht/v2/internal/coord/routing"
 	"github.com/libp2p/go-libp2p-kad-dht/v2/kadt"
@@ -229,7 +230,22 @@ func NewCoordinator(self kadt.PeerID, rtr coordt.Router[kadt.Key, kadt.PeerID, *
 		return nil, fmt.Errorf("probe: %w", err)
 	}
 
-	routingBehaviour := NewRoutingBehaviour(self, bootstrap, include, probe, cfg.Logger, tele.Tracer)
+	exploreCfg := routing.DefaultExploreConfig()
+	exploreCfg.Clock = cfg.Clock
+	exploreCfg.Timeout = cfg.QueryTimeout
+
+	schedule, err := routing.NewDynamicExploreSchedule(14, cfg.Clock.Now(), time.Hour, 1, 0)
+	if err != nil {
+		return nil, fmt.Errorf("explore schedule: %w", err)
+	}
+
+	// TODO: expose more config
+	explore, err := routing.NewExplore[kadt.Key](self, rt, cplutil.GenRandPeerID, schedule, exploreCfg)
+	if err != nil {
+		return nil, fmt.Errorf("explore: %w", err)
+	}
+
+	routingBehaviour := NewRoutingBehaviour(self, bootstrap, include, probe, explore, cfg.Logger, tele.Tracer)
 
 	networkBehaviour := NewNetworkBehaviour(rtr, cfg.Logger, tele.Tracer)
 

v2/internal/coord/coordinator_test.go

@@ -99,11 +99,11 @@ func TestExhaustiveQuery(t *testing.T) {
 	// A (ids[0]) is looking for D (ids[3])
 	// A will first ask B, B will reply with C's address (and A's address)
 	// A will then ask C, C will reply with D's address (and B's address)
-	self := kadt.PeerID(nodes[0].NodeID)
+	self := nodes[0].NodeID
 	c, err := NewCoordinator(self, nodes[0].Router, nodes[0].RoutingTable, ccfg)
 	require.NoError(t, err)
 
-	target := kadt.PeerID(nodes[3].NodeID).Key()
+	target := nodes[3].NodeID.Key()
 
 	visited := make(map[string]int)
 
@@ -137,7 +137,7 @@ func TestRoutingUpdatedEventEmittedForCloserNodes(t *testing.T) {
 	// A (ids[0]) is looking for D (ids[3])
 	// A will first ask B, B will reply with C's address (and A's address)
 	// A will then ask C, C will reply with D's address (and B's address)
-	self := kadt.PeerID(nodes[0].NodeID)
+	self := nodes[0].NodeID
 	c, err := NewCoordinator(self, nodes[0].Router, nodes[0].RoutingTable, ccfg)
 	if err != nil {
 		log.Fatalf("unexpected error creating coordinator: %v", err)
@@ -194,7 +194,7 @@ func TestBootstrap(t *testing.T) {
 
 	ccfg.Clock = clk
 
-	self := kadt.PeerID(nodes[0].NodeID)
+	self := nodes[0].NodeID
 	d, err := NewCoordinator(self, nodes[0].Router, nodes[0].RoutingTable, ccfg)
 	require.NoError(t, err)
 

v2/internal/coord/cplutil/cpl.go

@@ -0,0 +1,58 @@
+package cplutil
+
+import (
+	"crypto/rand"
+	"encoding/binary"
+	"fmt"
+
+	mh "github.com/multiformats/go-multihash"
+
+	"github.com/libp2p/go-libp2p-kad-dht/v2/kadt"
+)
+
+//go:generate go run ./gen.go
+
+// GenRandPeerID generates a random [kadt.PeerID] whose key has a common prefix length of exactly cpl with the supplied key.
+// Ported from go-libp2p-kbucket
+func GenRandPeerID(k kadt.Key, cpl int) (kadt.PeerID, error) {
+	if cpl > 15 {
+		return "", fmt.Errorf("cannot generate peer ID for Cpl greater than 15")
+	}
+
+	targetPrefix := prefix(k, cpl)
+
+	// Convert to a known peer ID.
+	key := keyPrefixMap[targetPrefix]
+	id := [32 + 2]byte{mh.SHA2_256, 32}
+	binary.BigEndian.PutUint32(id[2:], key)
+	return kadt.PeerID(string(id[:])), nil
+}
+
+type keybit interface {
+	Bit(i int) uint
+}
+
+// prefix generates random bits that have a common prefix length of exactly cpl with the supplied key.
+func prefix(k keybit, cpl int) uint16 {
+	var p uint16
+	// copy the first cpl+1 bits so we can flip the last one
+	for i := 0; i < cpl+1; i++ {
+		bit := uint16(k.Bit(i)) << (15 - i)
+		p |= bit
+	}
+
+	// flip the bit at cpl (cpl 5 means bits 0-4 must be the same)
+	mask := uint16(1) << (15 - cpl)
+	p ^= mask
+
+	if cpl < 15 {
+		// pad with random data
+		var buf [2]byte
+		_, _ = rand.Read(buf[:])
+		r := binary.BigEndian.Uint16(buf[:])
+
+		mask = (^uint16(0)) << (15 - cpl)
+		p = (p & mask) | (r & ^mask)
+	}
+	return p
+}