feat: add downgrade leader region step (#2792)

* feat: add downgrade leader region step

* chore: apply suggestions from CR

* chore: rename exist to exists

* chore: apply suggestions from CR

src/common/meta/src/instruction.rs

@@ -48,6 +48,27 @@ impl Display for RegionIdent {
     }
 }
 
+/// The result of downgrade leader region.
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)]
+pub struct DowngradeRegionReply {
+    /// Returns the `last_entry_id` if available.
+    pub last_entry_id: Option<u64>,
+    /// Indicates whether the region exists.
+    pub exists: bool,
+    /// Return error if any during the operation.
+    pub error: Option<String>,
+}
+
+impl Display for DowngradeRegionReply {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "(last_entry_id={:?}, exists={}, error={:?})",
+            self.last_entry_id, self.exists, self.error
+        )
+    }
+}
+
 #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)]
 pub struct SimpleReply {
     pub result: bool,
@@ -87,10 +108,23 @@ impl OpenRegion {
     }
 }
 
+/// The instruction of downgrading leader region.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct DowngradeRegion {
+    pub region_id: RegionId,
+}
+
+impl Display for DowngradeRegion {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(f, "DowngradeRegion(region_id={})", self.region_id)
+    }
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize, Display)]
 pub enum Instruction {
     OpenRegion(OpenRegion),
     CloseRegion(RegionIdent),
+    DowngradeRegion(DowngradeRegion),
     InvalidateTableIdCache(TableId),
     InvalidateTableNameCache(TableName),
 }
@@ -101,6 +135,7 @@ pub enum InstructionReply {
     OpenRegion(SimpleReply),
     CloseRegion(SimpleReply),
     InvalidateTableCache(SimpleReply),
+    DowngradeRegion(DowngradeRegionReply),
 }
 
 impl Display for InstructionReply {
@@ -111,6 +146,9 @@ impl Display for InstructionReply {
             Self::InvalidateTableCache(reply) => {
                 write!(f, "InstructionReply::Invalidate({})", reply)
             }
+            Self::DowngradeRegion(reply) => {
+                write!(f, "InstructionReply::DowngradeRegion({})", reply)
+            }
         }
     }
 }

src/datanode/src/heartbeat/handler.rs

@@ -65,6 +65,10 @@ impl RegionHeartbeatResponseHandler {
             Instruction::InvalidateTableIdCache(_) | Instruction::InvalidateTableNameCache(_) => {
                 InvalidHeartbeatResponseSnafu.fail()
             }
+            Instruction::DowngradeRegion(_) => {
+                // TODO(weny): add it later.
+                todo!()
+            }
         }
     }
 
@@ -88,6 +92,10 @@ impl RegionHeartbeatResponseHandler {
                     error: None,
                 })
             }
+            Instruction::DowngradeRegion(_) => {
+                // TODO(weny): add it later.
+                todo!()
+            }
         }
     }
 
@@ -114,6 +122,10 @@ impl RegionHeartbeatResponseHandler {
                 reply.result = success;
                 reply.error = error;
             }
+            InstructionReply::DowngradeRegion(_) => {
+                // TODO(weny): add it later.
+                todo!()
+            }
         }
 
         template

src/meta-srv/src/procedure/region_migration.rs

@@ -19,9 +19,11 @@ pub(crate) mod open_candidate_region;
 #[cfg(test)]
 pub(crate) mod test_util;
 pub(crate) mod update_metadata;
+pub(crate) mod upgrade_candidate_region;
 
 use std::any::Any;
 use std::fmt::Debug;
+use std::time::Duration;
 
 use common_meta::key::table_route::TableRouteValue;
 use common_meta::key::{DeserializedValueWithBytes, TableMetadataManagerRef};
@@ -34,6 +36,7 @@ use common_procedure::{Context as ProcedureContext, LockKey, Procedure, Status};
 use serde::{Deserialize, Serialize};
 use snafu::{location, Location, OptionExt, ResultExt};
 use store_api::storage::RegionId;
+use tokio::time::Instant;
 
 use self::migration_start::RegionMigrationStart;
 use crate::error::{self, Error, Result};
@@ -80,6 +83,29 @@ pub struct VolatileContext {
     opening_region_guard: Option<OpeningRegionGuard>,
     /// `table_route_info` is stored via previous steps for future use.
     table_route_info: Option<DeserializedValueWithBytes<TableRouteValue>>,
+    /// The deadline of leader region lease.
+    leader_region_lease_deadline: Option<Instant>,
+    /// The last_entry_id of leader region.
+    leader_region_last_entry_id: Option<u64>,
+}
+
+impl VolatileContext {
+    /// Sets the `leader_region_lease_deadline` if it does not exist.
+    pub fn set_leader_region_lease_deadline(&mut self, lease_timeout: Duration) {
+        if self.leader_region_lease_deadline.is_none() {
+            self.leader_region_lease_deadline = Some(Instant::now() + lease_timeout);
+        }
+    }
+
+    /// Resets the `leader_region_lease_deadline`.
+    pub fn reset_leader_region_lease_deadline(&mut self) {
+        self.leader_region_lease_deadline = None;
+    }
+
+    /// Sets the `leader_region_last_entry_id`.
+    pub fn set_last_entry_id(&mut self, last_entry_id: u64) {
+        self.leader_region_last_entry_id = Some(last_entry_id)
+    }
 }
 
 /// Used to generate new [Context].