forked from valkey-io/valkey
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Fix data loss when replica do a failover with a old history repl offset
Our current replica can initiate a failover without restriction when it detects that the primary node is offline. This is generally not a problem. However, consider the following scenarios: 1. In slot migration, a primary loses its last slot and then becomes a replica. When it is fully synchronized with the new primary, the new primary downs. 2. In CLUSTER REPLICATE command, a replica becomes a replica of another primary. When it is fully synchronized with the new primary, the new primary downs. In the above scenario, case 1 may cause the empty primary to be elected as the new primary, resulting in primary data loss. Case 2 may cause the non-empty replica to be elected as the new primary, resulting in data loss and confusion. The reason is that we have cached primary logic, which is used for psync. In the above scenario, when clusterSetPrimary is called, myself will cache server.primary in server.cached_primary for psync. In replicationGetReplicaOffset, we get server.cached_primary->reploff for offset, gossip it and rank it, which causes the replica to use the old historical offset to initiate failover, and it get a good rank, initiates election first, and then is elected as the new primary. The main problem here is that when the replica has not completed full sync, it may get the historical offset in replicationGetReplicaOffset. The fix is to clear cached_primary in these places where full sync is obviously needed, and let the replica use offset == 0 to participate in the election. In this way, this unhealthy replica has a worse rank and is not easy to be elected. Of course, it is possible that it will be elected with offset == 0. In the future, we may need to prohibit the replica with offset == 0 from having the right to initiate elections. Signed-off-by: Binbin <[email protected]>
- Loading branch information
1 parent
7424620
commit e41044f
Showing
4 changed files
with
192 additions
and
25 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,141 @@ | ||
| # Allocate slot 0 to the last primary and evenly distribute the remaining | ||
| # slots to the remaining primary. | ||
| proc my_slot_allocation {masters replicas} { | ||
| set avg [expr double(16384) / [expr $masters-1]] | ||
| set slot_start 1 | ||
| for {set j 0} {$j < $masters-1} {incr j} { | ||
| set slot_end [expr int(ceil(($j + 1) * $avg) - 1)] | ||
| R $j cluster addslotsrange $slot_start $slot_end | ||
| set slot_start [expr $slot_end + 1] | ||
| } | ||
| R [expr $masters-1] cluster addslots 0 | ||
| } | ||
|
|
||
| start_cluster 4 4 {tags {external:skip cluster} overrides {cluster-node-timeout 1000 cluster-migration-barrier 999}} { | ||
| test "Write some data to primary 0, slot 1, make a small repl_offset" { | ||
| for {set i 0} {$i < 1024} {incr i} { | ||
| R 0 incr key_991803 | ||
| } | ||
| assert_equal {1024} [R 0 get key_991803] | ||
| } | ||
|
|
||
| test "Write some data to primary 4, slot 0, make a big repl_offset" { | ||
| for {set i 0} {$i < 10240} {incr i} { | ||
| R 3 incr key_977613 | ||
| } | ||
| assert_equal {10240} [R 3 get key_977613] | ||
| } | ||
|
|
||
| test "If a replica has not completed sync, it can not do the failover" { | ||
| # 10s, make sure primary 1 will hang in the save | ||
| R 0 config set rdb-key-save-delay 100000000 | ||
|
|
||
| # Move the slot 0 from primary 4 to primary 1 | ||
| set addr "[srv 0 host]:[srv 0 port]" | ||
| set myid [R 3 CLUSTER MYID] | ||
| set code [catch { | ||
| exec src/valkey-cli {*}[valkeycli_tls_config "./tests"] --cluster rebalance $addr --cluster-weight $myid=0 | ||
| } result] | ||
| if {$code != 0} { | ||
| fail "valkey-cli --cluster rebalance returns non-zero exit code, output below:\n$result" | ||
| } | ||
|
|
||
| # Let primary 4's primary and replica can convert to replicas when | ||
| # they lost the last slot. | ||
| R 3 config set cluster-replica-validity-factor 0 | ||
| R 7 config set cluster-replica-validity-factor 0 | ||
| R 3 config set cluster-allow-replica-migration yes | ||
| R 7 config set cluster-allow-replica-migration yes | ||
|
|
||
| # Shutdown the primary 1 | ||
| catch {R 0 shutdown nosave} | ||
|
|
||
| # Wait for the replica become a primary, and make sure | ||
| # the other primary become a replica. | ||
| wait_for_condition 1000 50 { | ||
| [s -4 role] eq {master} && | ||
| [s -3 role] eq {slave} && | ||
| [s -7 role] eq {slave} | ||
| } else { | ||
| puts "s -4 role: [s -4 role]" | ||
| puts "s -3 role: [s -3 role]" | ||
| puts "s -7 role: [s -7 role]" | ||
| fail "Failover does not happened" | ||
| } | ||
|
|
||
| # Make sure 3 / 7 get the lower rank and offset is 0. | ||
| verify_log_message -3 "*Start of election*offset 0*" 0 | ||
| verify_log_message -7 "*Start of election*offset 0*" 0 | ||
|
|
||
| # Make sure the right replica get the higher rank. | ||
| verify_log_message -4 "*Start of election*rank #0*" 0 | ||
|
|
||
| # Make sure the key is exists and consistent. | ||
| R 3 readonly | ||
| R 7 readonly | ||
| wait_for_condition 1000 50 { | ||
| [R 3 get key_991803] == 1024 && [R 3 get key_977613] == 10240 && | ||
| [R 4 get key_991803] == 1024 && [R 4 get key_977613] == 10240 && | ||
| [R 7 get key_991803] == 1024 && [R 7 get key_977613] == 10240 | ||
| } else { | ||
| puts "R 3: [R 3 keys *]" | ||
| puts "R 4: [R 4 keys *]" | ||
| puts "R 7: [R 7 keys *]" | ||
| fail "Key not consistent" | ||
| } | ||
| } | ||
| } my_slot_allocation cluster_allocate_replicas ;# start_cluster | ||
|
|
||
| start_cluster 4 4 {tags {external:skip cluster} overrides {cluster-node-timeout 1000 cluster-migration-barrier 999}} { | ||
| test "Write some data to primary 0, slot 1, make a small repl_offset" { | ||
| for {set i 0} {$i < 1024} {incr i} { | ||
| R 0 incr key_991803 | ||
| } | ||
| assert_equal {1024} [R 0 get key_991803] | ||
| } | ||
|
|
||
| test "Write some data to primary 4, slot 0, make a big repl_offset" { | ||
| for {set i 0} {$i < 10240} {incr i} { | ||
| R 3 incr key_977613 | ||
| } | ||
| assert_equal {10240} [R 3 get key_977613] | ||
| } | ||
|
|
||
| test "A old replica use CLUSTER REPLICA get a zero offset before the full sync is completed" { | ||
| # 10s, make sure primary 1 will hang in the save | ||
| R 0 config set rdb-key-save-delay 100000000 | ||
|
|
||
| # Let the replica do the replicate with primary 1. | ||
| R 7 config set cluster-replica-validity-factor 0 | ||
| R 7 config set cluster-allow-replica-migration yes | ||
| R 7 cluster replicate [R 0 cluster myid] | ||
|
|
||
| # Shutdown the primary 1. | ||
| catch {R 0 shutdown nosave} | ||
|
|
||
| # Wait for the replica become a primary. | ||
| wait_for_condition 1000 50 { | ||
| [s -4 role] eq {master} && | ||
| [s -7 role] eq {slave} | ||
| } else { | ||
| puts "s -4 role: [s -4 role]" | ||
| puts "s -7 role: [s -7 role]" | ||
| fail "Failover does not happened" | ||
| } | ||
|
|
||
| # Make sure 7 get the lower rank and it's offset is 0. | ||
| verify_log_message -4 "*Start of election*rank #0*" 0 | ||
| verify_log_message -7 "*Start of election*offset 0*" 0 | ||
|
|
||
| # Make sure the key is exists and consistence. | ||
| R 7 readonly | ||
| wait_for_condition 1000 50 { | ||
| [R 4 get key_991803] == 1024 && | ||
| [R 7 get key_991803] == 1024 | ||
| } else { | ||
| puts "R 4: [R 4 get key_991803]" | ||
| puts "R 7: [R 7 get key_991803]" | ||
| fail "Key not consistent" | ||
| } | ||
| } | ||
| } my_slot_allocation cluster_allocate_replicas ;# start_cluster |