Raft pre-vote extension implementation (#530)

* prevote initial implementation * add config and relevant tests * remove extra comments, fix a case where newer term is discovered for prevote * fix to reset timeout after pre-vote and fix split vote (pre-vote,vote) case. * fix a case where granted votes and prevotes don't reach quorum but the sum can reach quorum * add submodule and first iteration of multi-version tests rename test and pin submodule to version 1.5.0 rename test file * refactor test * clean up node init * clean up leader rolling upgrade * fix use of deprecate Leader method * extract cluster package * export cluster Type * clean up tests and add test utils * rename package to raftlatest * remove submodule * new submodule * fix go.mod * change inmemConfig to be not exported * remove unused func * add replace rolling upgrade tests * rename raft-latest to raft-previous * rename raft-latest to raft-previous submodule * fix submodule * remove printf * use same name for recycled servers, add other leave scenarios * prevote initial implementation * add config and relevant tests * remove extra comments, fix a case where newer term is discovered for prevote * fix to reset timeout after pre-vote and fix split vote (pre-vote,vote) case. * write upgrade tests that include prevotes * add more test cases * fix submodule version * go mod tidy * update pervious version to v1.6.0 * fix merge duplication * add submodule and first iteration of multi-version tests rename test and pin submodule to version 1.5.0 rename test file * refactor test * clean up node init * clean up leader rolling upgrade * fix use of deprecate Leader method * extract cluster package * export cluster Type * clean up tests and add test utils * rename package to raftlatest * remove submodule * new submodule * fix go.mod * change inmemConfig to be not exported * remove unused func * add replace rolling upgrade tests * rename raft-latest to raft-previous * rename raft-latest to raft-previous submodule * fix submodule * remove printf * use same name for recycled servers, add other leave scenarios * prevote initial implementation * add config and relevant tests * remove extra comments, fix a case where newer term is discovered for prevote * fix to reset timeout after pre-vote and fix split vote (pre-vote,vote) case. * write upgrade tests that include prevotes * add more test cases * fix submodule version * prevote initial implementation * add config and relevant tests * remove extra comments, fix a case where newer term is discovered for prevote * fix to reset timeout after pre-vote and fix split vote (pre-vote,vote) case. * fix a case where granted votes and prevotes don't reach quorum but the sum can reach quorum * go mod tidy * update pervious version to v1.6.0 * fix merge duplication * fix rebase issues * use a different RPC command for prevote. * fix prevote tests and add rollback tests * add a partitioned node prevote test * remove server from config before shutting down, fix raft submodule * remove extra comment * change `inmemConfig` to accept testing.TB * remove stray comment * fix comments and remove extra fields * remove duplicate var * remove leader transfer from pre-vote path, fix logs and comments. * make pre-vote enabled by default * remove `Candidate` field from pre-vote request * add warning when transport don't support prevote * panic if transport is not supported in preElectSelf. * Fix comments and log string Co-authored-by: Paul Banks <[email protected]> * Fix to log the right number for votesNeeded, added preVoteRefusedVotes to the log --------- Co-authored-by: Paul Banks <[email protected]>
hashicorp · Jun 6, 2024 · 181475c · 181475c
1 parent 341ea38
commit 181475c
Show file tree

Hide file tree

Showing 18 changed files with 931 additions and 101 deletions.
diff --git a/api.go b/api.go
@@ -213,6 +213,10 @@ type Raft struct {
 
 	// mainThreadSaturation measures the saturation of the main raft goroutine.
 	mainThreadSaturation *saturationMetric
+
+	// preVoteDisabled control if the pre-vote feature is activated,
+	// prevote feature is disabled if set to true.
+	preVoteDisabled bool
 }
 
 // BootstrapCluster initializes a server's storage with the given cluster
@@ -531,6 +535,7 @@ func NewRaft(conf *Config, fsm FSM, logs LogStore, stable StableStore, snaps Sna
 		applyCh = make(chan *logFuture, conf.MaxAppendEntries)
 	}
 
+	_, transportSupportPreVote := trans.(WithPreVote)
 	// Create Raft struct.
 	r := &Raft{
 		protocolVersion:       protocolVersion,
@@ -560,6 +565,10 @@ func NewRaft(conf *Config, fsm FSM, logs LogStore, stable StableStore, snaps Sna
 		leaderNotifyCh:        make(chan struct{}, 1),
 		followerNotifyCh:      make(chan struct{}, 1),
 		mainThreadSaturation:  newSaturationMetric([]string{"raft", "thread", "main", "saturation"}, 1*time.Second),
+		preVoteDisabled:       conf.PreVoteDisabled || !transportSupportPreVote,
+	}
+	if !transportSupportPreVote && !conf.PreVoteDisabled {
+		r.logger.Warn("pre-vote is disabled because it is not supported by the Transport")
 	}
 
 	r.conf.Store(*conf)

diff --git a/commands.go b/commands.go
@@ -120,6 +120,40 @@ func (r *RequestVoteResponse) GetRPCHeader() RPCHeader {
 	return r.RPCHeader
 }
 
+// RequestPreVoteRequest is the command used by a candidate to ask a Raft peer
+// for a vote in an election.
+type RequestPreVoteRequest struct {
+	RPCHeader
+
+	// Provide the term and our id
+	Term uint64
+
+	// Used to ensure safety
+	LastLogIndex uint64
+	LastLogTerm  uint64
+}
+
+// GetRPCHeader - See WithRPCHeader.
+func (r *RequestPreVoteRequest) GetRPCHeader() RPCHeader {
+	return r.RPCHeader
+}
+
+// RequestPreVoteResponse is the response returned from a RequestPreVoteRequest.
+type RequestPreVoteResponse struct {
+	RPCHeader
+
+	// Newer term if leader is out of date.
+	Term uint64
+
+	// Is the vote granted.
+	Granted bool
+}
+
+// GetRPCHeader - See WithRPCHeader.
+func (r *RequestPreVoteResponse) GetRPCHeader() RPCHeader {
+	return r.RPCHeader
+}
+
 // InstallSnapshotRequest is the command sent to a Raft peer to bootstrap its
 // log (and state machine) from a snapshot on another peer.
 type InstallSnapshotRequest struct {

diff --git a/config.go b/config.go
@@ -232,6 +232,9 @@ type Config struct {
 	// raft's configuration and index values.
 	NoSnapshotRestoreOnStart bool
 
+	// PreVoteDisabled deactivate the pre-vote feature when set to true
+	PreVoteDisabled bool
+
 	// skipStartup allows NewRaft() to bypass all background work goroutines
 	skipStartup bool
 }

diff --git a/fuzzy/go.mod b/fuzzy/go.mod
@@ -3,7 +3,7 @@ module github.com/hashicorp/raft/fuzzy
 go 1.20
 
 require (
-	github.com/hashicorp/go-hclog v1.5.0
+	github.com/hashicorp/go-hclog v1.6.2
 	github.com/hashicorp/go-msgpack/v2 v2.1.1
 	github.com/hashicorp/raft v1.2.0
 	github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea

diff --git a/fuzzy/go.sum b/fuzzy/go.sum
@@ -33,6 +33,7 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/
 github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c=
 github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
+github.com/hashicorp/go-hclog v1.6.2/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
 github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0=
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI=
@@ -91,7 +92,10 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=

diff --git a/fuzzy/transport.go b/fuzzy/transport.go
@@ -221,6 +221,11 @@ func (t *transport) RequestVote(id raft.ServerID, target raft.ServerAddress, arg
 	return t.sendRPC(string(target), args, resp)
 }
 
+// RequestPreVote sends the appropriate RPC to the target node.
+func (t *transport) RequestPreVote(id raft.ServerID, target raft.ServerAddress, args *raft.RequestPreVoteRequest, resp *raft.RequestPreVoteResponse) error {
+	return t.sendRPC(string(target), args, resp)
+}
+
 // InstallSnapshot is used to push a snapshot down to a follower. The data is read from
 // the ReadCloser and streamed to the client.
 func (t *transport) InstallSnapshot(id raft.ServerID, target raft.ServerAddress, args *raft.InstallSnapshotRequest, resp *raft.InstallSnapshotResponse, data io.Reader) error {

diff --git a/inmem_transport.go b/inmem_transport.go
@@ -125,6 +125,18 @@ func (i *InmemTransport) RequestVote(id ServerID, target ServerAddress, args *Re
 	return nil
 }
 
+func (i *InmemTransport) RequestPreVote(id ServerID, target ServerAddress, args *RequestPreVoteRequest, resp *RequestPreVoteResponse) error {
+	rpcResp, err := i.makeRPC(target, args, nil, i.timeout)
+	if err != nil {
+		return err
+	}
+
+	// Copy the result back
+	out := rpcResp.Response.(*RequestPreVoteResponse)
+	*resp = *out
+	return nil
+}
+
 // InstallSnapshot implements the Transport interface.
 func (i *InmemTransport) InstallSnapshot(id ServerID, target ServerAddress, args *InstallSnapshotRequest, resp *InstallSnapshotResponse, data io.Reader) error {
 	rpcResp, err := i.makeRPC(target, args, data, 10*i.timeout)

diff --git a/net_transport.go b/net_transport.go
@@ -24,6 +24,7 @@ const (
 	rpcRequestVote
 	rpcInstallSnapshot
 	rpcTimeoutNow
+	rpcRequestPreVote
 
 	// DefaultTimeoutScale is the default TimeoutScale in a NetworkTransport.
 	DefaultTimeoutScale = 256 * 1024 // 256KB
@@ -473,6 +474,11 @@ func (n *NetworkTransport) RequestVote(id ServerID, target ServerAddress, args *
 	return n.genericRPC(id, target, rpcRequestVote, args, resp)
 }
 
+// RequestPreVote implements the Transport interface.
+func (n *NetworkTransport) RequestPreVote(id ServerID, target ServerAddress, args *RequestPreVoteRequest, resp *RequestPreVoteResponse) error {
+	return n.genericRPC(id, target, rpcRequestPreVote, args, resp)
+}
+
 // genericRPC handles a simple request/response RPC.
 func (n *NetworkTransport) genericRPC(id ServerID, target ServerAddress, rpcType uint8, args interface{}, resp interface{}) error {
 	// Get a conn
@@ -685,6 +691,13 @@ func (n *NetworkTransport) handleCommand(r *bufio.Reader, dec *codec.Decoder, en
 		}
 		rpc.Command = &req
 		labels = []metrics.Label{{Name: "rpcType", Value: "RequestVote"}}
+	case rpcRequestPreVote:
+		var req RequestPreVoteRequest
+		if err := dec.Decode(&req); err != nil {
+			return err
+		}
+		rpc.Command = &req
+		labels = []metrics.Label{{Name: "rpcType", Value: "RequestPreVote"}}
 	case rpcInstallSnapshot:
 		var req InstallSnapshotRequest
 		if err := dec.Decode(&req); err != nil {

diff --git a/raft-compat/go.mod b/raft-compat/go.mod
@@ -7,13 +7,14 @@ require github.com/stretchr/testify v1.8.4
 require (
 	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/fatih/color v1.13.0 // indirect
-	github.com/hashicorp/go-hclog v1.5.0 // indirect
+	github.com/hashicorp/go-hclog v1.6.2 // indirect
 	github.com/hashicorp/go-immutable-radix v1.0.0 // indirect
 	github.com/hashicorp/go-msgpack v0.5.5 // indirect
+	github.com/hashicorp/go-msgpack/v2 v2.1.1 // indirect
 	github.com/hashicorp/golang-lru v0.5.0 // indirect
 	github.com/mattn/go-colorable v0.1.12 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
-	golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 // indirect
+	golang.org/x/sys v0.13.0 // indirect
 )
 
 replace github.com/hashicorp/raft-previous-version => ./raft-previous-version
@@ -22,7 +23,7 @@ replace github.com/hashicorp/raft => ../
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/hashicorp/raft v1.2.0
+	github.com/hashicorp/raft v1.6.1
 	github.com/hashicorp/raft-previous-version v1.2.0
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

diff --git a/raft-compat/go.sum b/raft-compat/go.sum
@@ -31,10 +31,14 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/
 github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c=
 github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
+github.com/hashicorp/go-hclog v1.6.2 h1:NOtoftovWkDheyUM/8JW3QMiXyxJK3uHRK7wV04nD2I=
+github.com/hashicorp/go-hclog v1.6.2/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
 github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0=
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI=
 github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
+github.com/hashicorp/go-msgpack/v2 v2.1.1 h1:xQEY9yB2wnHitoSzk/B9UjXWRQ67QKu5AOm8aFp8N3I=
+github.com/hashicorp/go-msgpack/v2 v2.1.1/go.mod h1:upybraOAblm4S7rx0+jeNy+CWWhzywQsSRV5033mMu4=
 github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
 github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM=
 github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
@@ -113,6 +117,8 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 h1:nonptSpoQ4vQjyraW20DXPAglgQfVnM9ZC6MmNLMR60=
 golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=