Skip to content

Commit

Permalink
fix: remark reigon as inactive on leader changed
Browse files Browse the repository at this point in the history
  • Loading branch information
fengjiachun committed Sep 19, 2023
1 parent 0f79cca commit 3665b19
Show file tree
Hide file tree
Showing 2 changed files with 20 additions and 9 deletions.
19 changes: 15 additions & 4 deletions src/meta-srv/src/procedure/region_failover/activate_region.rs
Original file line number Diff line number Diff line change
Expand Up @@ -37,13 +37,18 @@ use crate::service::mailbox::{Channel, MailboxReceiver};
#[derive(Serialize, Deserialize, Debug)]
pub(super) struct ActivateRegion {
candidate: Peer,
// If the meta leader node dies during the execution of the procedure,
// the new leader node needs to remark the failed region as "inactive"
// to prevent it from renewing the lease.
remark_inactive_region: bool,
region_storage_path: Option<String>,
}

impl ActivateRegion {
pub(super) fn new(candidate: Peer) -> Self {
pub(super) fn new(candidate: Peer, remark_inactive_region: bool) -> Self {
Self {
candidate,
remark_inactive_region,
region_storage_path: None,
}
}
Expand All @@ -55,7 +60,6 @@ impl ActivateRegion {
timeout: Duration,
) -> Result<MailboxReceiver> {
let table_id = failed_region.table_id;
// TODO(weny): considers fetching table info only once.
let table_info = ctx
.table_metadata_manager
.table_info_manager()
Expand Down Expand Up @@ -168,6 +172,13 @@ impl State for ActivateRegion {
ctx: &RegionFailoverContext,
failed_region: &RegionIdent,
) -> Result<Box<dyn State>> {
if self.remark_inactive_region {
// Remark the fail region as inactive to prevent it from renewing the lease.
InactiveRegionManager::new(&ctx.in_memory)
.register_inactive_region(failed_region)
.await?;
}

let mailbox_receiver = self
.send_open_region_message(ctx, failed_region, OPEN_REGION_MESSAGE_TIMEOUT)
.await?;
Expand All @@ -192,7 +203,7 @@ mod tests {
let failed_region = env.failed_region(1).await;

let candidate = 2;
let mut state = ActivateRegion::new(Peer::new(candidate, ""));
let mut state = ActivateRegion::new(Peer::new(candidate, ""), false);
let mailbox_receiver = state
.send_open_region_message(&env.context, &failed_region, Duration::from_millis(100))
.await
Expand Down Expand Up @@ -263,7 +274,7 @@ mod tests {
let failed_region = env.failed_region(1).await;

let candidate = 2;
let mut state = ActivateRegion::new(Peer::new(candidate, ""));
let mut state = ActivateRegion::new(Peer::new(candidate, ""), false);
let mailbox_receiver = state
.send_open_region_message(&env.context, &failed_region, Duration::from_millis(100))
.await
Expand Down
10 changes: 5 additions & 5 deletions src/meta-srv/src/procedure/region_failover/deactivate_region.rs
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ impl DeactivateRegion {
.deregister_inactive_region(failed_region)
.await?;

Ok(Box::new(ActivateRegion::new(self.candidate.clone())))
Ok(Box::new(ActivateRegion::new(self.candidate.clone(), false)))
} else {
// Under rare circumstances would a Datanode fail to close a Region.
// So simply retry.
Expand All @@ -113,7 +113,7 @@ impl DeactivateRegion {
// the call and have disabled region lease renewal. Therefore, if a timeout error
// occurs, it can be concluded that the region has been closed. With this information,
// we can proceed confidently to the next step.
Ok(Box::new(ActivateRegion::new(self.candidate.clone())))
Ok(Box::new(ActivateRegion::new(self.candidate.clone(), true)))
}
Err(e) => Err(e),
}
Expand Down Expand Up @@ -146,7 +146,7 @@ impl State for DeactivateRegion {
);
// See the mailbox received timeout situation comments above.
self.wait_for_region_lease_expiry(ctx).await;
return Ok(Box::new(ActivateRegion::new(self.candidate.clone())));
return Ok(Box::new(ActivateRegion::new(self.candidate.clone(), true)));
}
Err(e) => return Err(e),
};
Expand Down Expand Up @@ -226,7 +226,7 @@ mod tests {
.unwrap();
assert_eq!(
format!("{next_state:?}"),
r#"ActivateRegion { candidate: Peer { id: 2, addr: "" }, region_storage_path: None }"#
r#"ActivateRegion { candidate: Peer { id: 2, addr: "" }, remark_inactive_region: false, region_storage_path: None }"#
);
}

Expand Down Expand Up @@ -268,7 +268,7 @@ mod tests {
// Timeout or not, proceed to `ActivateRegion`.
assert_eq!(
format!("{next_state:?}"),
r#"ActivateRegion { candidate: Peer { id: 2, addr: "" }, region_storage_path: None }"#
r#"ActivateRegion { candidate: Peer { id: 2, addr: "" }, remark_inactive_region: true, region_storage_path: None }"#
);
}
}

0 comments on commit 3665b19

Please sign in to comment.