Merge pull request #79 from ClickHouse/fix-small-race-ch

Fix race when removing a node
ClickHouse · Nov 19, 2024 · b4a3aec · b4a3aec
2 parents 0a765ec + a8314d9
commit b4a3aec
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 6 deletions.
diff --git a/include/libnuraft/raft_server.hxx b/include/libnuraft/raft_server.hxx
@@ -1462,7 +1462,7 @@ protected:
     /**
      * Lock of handling client request and role change.
      */
-    std::mutex cli_lock_;
+    std::recursive_mutex cli_lock_;
 
     /**
      * Condition variable to invoke BG commit thread.

diff --git a/src/handle_client_request.cxx b/src/handle_client_request.cxx
@@ -67,7 +67,7 @@ ptr<resp_msg> raft_server::handle_leader_status_req(req_msg& req) {
         case raft_params::dual_mutex:
         default: {
             // TODO: Use RW lock here.
-            auto_lock(cli_lock_);
+            recur_lock(cli_lock_);
             return get_leader_status();
         }
     }
@@ -89,7 +89,7 @@ ptr<resp_msg> raft_server::handle_cli_req_prelock(req_msg& req,
         case raft_params::dual_mutex:
         default: {
             // TODO: Use RW lock here.
-            auto_lock(cli_lock_);
+            recur_lock(cli_lock_);
             resp = handle_cli_req(req, ext_params, timestamp_us);
             break;
         }

diff --git a/src/raft_server.cxx b/src/raft_server.cxx
@@ -1030,7 +1030,7 @@ void raft_server::become_leader() {
     }
 
     ptr<raft_params> params = ctx_->get_params();
-    {   auto_lock(cli_lock_);
+    {   recur_lock(cli_lock_);
         role_ = srv_role::leader;
         leader_ = id_;
         srv_to_join_.reset();
@@ -1403,7 +1403,7 @@ bool raft_server::request_leadership() {
 void raft_server::become_follower() {
     // stop hb for all peers
     p_in("[BECOME FOLLOWER] term %" PRIu64 "", state_->get_term());
-    {   std::lock_guard<std::mutex> ll(cli_lock_);
+    {   std::lock_guard<std::recursive_mutex> ll(cli_lock_);
         for (peer_itor it = peers_.begin(); it != peers_.end(); ++it) {
             it->second->enable_hb(false);
         }
@@ -1460,7 +1460,7 @@ bool raft_server::update_term(ulong term) {
             //
             //   To avoid this issue, we acquire `cli_lock_`,
             //   and change `role_` first before setting the term.
-            std::lock_guard<std::mutex> ll(cli_lock_);
+            std::lock_guard<std::recursive_mutex> ll(cli_lock_);
             role_ = srv_role::follower;
             state_->set_term(term);
         }
@@ -1778,6 +1778,7 @@ ulong raft_server::store_log_entry(ptr<log_entry>& entry, ulong index) {
         }
 
         if ( role_ == srv_role::leader ) {
+            recur_lock(cli_lock_);
             // Need to progress precommit index for config.
             try_update_precommit_index(log_index);
         }