bug: backward incompatible change in hummock table watermark for release-1.10

[hzxa21]

Describe the bug

We introduced a new field StateTableInfo in HummockVersion in #15896 and implemented a logic to init this new field via

risingwave/src/storage/hummock_sdk/src/version.rs

Line 324 in cc44276

pub fn may_fill_backward_compatible_state_table_info_delta(

and we leverage this new field to clean up stale hummock table watermark in

risingwave/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs

Line 649 in cc44276

&& let Some(Some(prev_table)) = changed_table_info.get(table_id)

.

However, given that may_fill_backward_compatible_state_table_info_delta happens after the initialization and the replay of the existing hummock version delta. The stale table watermark cleanup triggered by replaying old version deltas will be skipped and cause panic in

risingwave/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs

Line 218 in cc44276

assert!(

Error message/log

thread 'rw-main' panicked at src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs:218:17:
smallest epoch 6778399146639360 in table watermark should be at least safe epoch 6778399801933824
stack backtrace:
   0:     0xaaaae457d384 - std::backtrace_rs::backtrace::libunwind::trace::h64ef74c421ddf515
                               at /rustc/e4c626dd9a17a23270bf8e7158e59cf2b9c04840/library/std/src/../../backtrace/src/backtrace/libunwind.rs:104:5
   1:     0xaaaae457d384 - std::backtrace_rs::backtrace::trace_unsynchronized::h0948cfffd26b9adf
                               at /rustc/e4c626dd9a17a23270bf8e7158e59cf2b9c04840/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
   2:     0xaaaae457d384 - std::sys_common::backtrace::_print_fmt::hec7c7f44910dd893
                               at /rustc/e4c626dd9a17a23270bf8e7158e59cf2b9c04840/library/std/src/sys_common/backtrace.rs:68:5
   3:     0xaaaae457d384 - <std::sys_common::backtrace::_print::DisplayBacktrace as core::fmt::Display>::fmt::h6ef892d3f076e581
                               at /rustc/e4c626dd9a17a23270bf8e7158e59cf2b9c04840/library/std/src/sys_common/backtrace.rs:44:22
   4:     0xaaaada71c7cc - core::fmt::rt::Argument::fmt::hc7ee4c2da8957d17
                               at /rustc/e4c626dd9a17a23270bf8e7158e59cf2b9c04840/library/core/src/fmt/rt.rs:142:9
   5:     0xaaaada71c7cc - core::fmt::write::h0c4a627b12e3d78d
                               at /rustc/e4c626dd9a17a23270bf8e7158e59cf2b9c04840/library/core/src/fmt/mod.rs:1120:17
   6:     0xaaaae4576054 - std::io::Write::write_fmt::h318ea8fa9c57551f
                               at /rustc/e4c626dd9a17a23270bf8e7158e59cf2b9c04840/library/std/src/io/mod.rs:1810:15
   7:     0xaaaae457d1b0 - std::sys_common::backtrace::_print::ha8e58b5a72e68af5
                               at /rustc/e4c626dd9a17a23270bf8e7158e59cf2b9c04840/library/std/src/sys_common/backtrace.rs:47:5
   8:     0xaaaae457d1b0 - std::sys_common::backtrace::print::hbce77ec9c0ee5615
                               at /rustc/e4c626dd9a17a23270bf8e7158e59cf2b9c04840/library/std/src/sys_common/backtrace.rs:34:9
   9:     0xaaaae457e828 - std::panicking::default_hook::{{closure}}::h62de2f44289d99c3
  10:     0xaaaae457e55c - std::panicking::default_hook::h6808acaf9c18352a
                               at /rustc/e4c626dd9a17a23270bf8e7158e59cf2b9c04840/library/std/src/panicking.rs:292:9
  11:     0xaaaae21d9e60 - <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call::h51433673ade33678
                               at /rustc/e4c626dd9a17a23270bf8e7158e59cf2b9c04840/library/alloc/src/boxed.rs:2029:9
  12:     0xaaaae21d9e60 - risingwave_rt::panic_hook::set_panic_hook::{{closure}}::hf762bc4f9c141b43
                               at /risingwave/src/utils/runtime/src/panic_hook.rs:25:9
  13:     0xaaaae21d9e60 - std::panicking::update_hook::{{closure}}::h3add513d39e25f07
                               at /rustc/e4c626dd9a17a23270bf8e7158e59cf2b9c04840/library/std/src/panicking.rs:234:47
  14:     0xaaaae457ee74 - <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call::h861e29f50d7dc506
                               at /rustc/e4c626dd9a17a23270bf8e7158e59cf2b9c04840/library/alloc/src/boxed.rs:2029:9
  15:     0xaaaae457ee74 - std::panicking::rust_panic_with_hook::h7d9418a0e45d61fb
                               at /rustc/e4c626dd9a17a23270bf8e7158e59cf2b9c04840/library/std/src/panicking.rs:783:13
  16:     0xaaaae457ec1c - std::panicking::begin_panic_handler::{{closure}}::he5c28c60ba487bed
                               at /rustc/e4c626dd9a17a23270bf8e7158e59cf2b9c04840/library/std/src/panicking.rs:657:13
  17:     0xaaaae457d858 - std::sys_common::backtrace::__rust_end_short_backtrace::hcdcd48a39dadcd83
                               at /rustc/e4c626dd9a17a23270bf8e7158e59cf2b9c04840/library/std/src/sys_common/backtrace.rs:171:18
  18:     0xaaaae457e9c4 - rust_begin_unwind
                               at /rustc/e4c626dd9a17a23270bf8e7158e59cf2b9c04840/library/std/src/panicking.rs:645:5
  19:     0xaaaada719398 - core::panicking::panic_fmt::had704c91ae6f0365
                               at /rustc/e4c626dd9a17a23270bf8e7158e59cf2b9c04840/library/core/src/panicking.rs:72:14
  20:     0xaaaadfdb640c - risingwave_hummock_sdk::compaction_group::hummock_version_ext::<impl risingwave_hummock_sdk::version::HummockVersion>::safe_epoch_table_watermarks::extract_single_table_watermark::h735b8df5051243a3
                               at /risingwave/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs:218:17
  21:     0xaaaadfdb640c - risingwave_hummock_sdk::compaction_group::hummock_version_ext::<impl risingwave_hummock_sdk::version::HummockVersion>::safe_epoch_table_watermarks::{{closure}}::h5912b3250c6be3d8
                               at /risingwave/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs:249:21
  22:     0xaaaadfdb640c - core::ops::function::impls::<impl core::ops::function::FnMut<A> for &mut F>::call_mut::h131c5715d5da4468
                               at /rustc/e4c626dd9a17a23270bf8e7158e59cf2b9c04840/library/core/src/ops/function.rs:294:13
  23:     0xaaaadfda697c - core::iter::traits::iterator::Iterator::find_map::check::{{closure}}::h11b4845e6f10c218
                               at /rustc/e4c626dd9a17a23270bf8e7158e59cf2b9c04840/library/core/src/iter/traits/iterator.rs:2958:32
  24:     0xaaaadfda697c - core::iter::traits::iterator::Iterator::try_fold::h1d549c39b84d21f7
                               at /rustc/e4c626dd9a17a23270bf8e7158e59cf2b9c04840/library/core/src/iter/traits/iterator.rs:2462:21
  25:     0xaaaadfda697c - core::iter::traits::iterator::Iterator::find_map::h83e1e30780fda0f0
                               at /rustc/e4c626dd9a17a23270bf8e7158e59cf2b9c04840/library/core/src/iter/traits/iterator.rs:2964:9
  26:     0xaaaadfda697c - <core::iter::adapters::filter_map::FilterMap<I,F> as core::iter::traits::iterator::Iterator>::next::h5f329ee7b1dead32
                               at /rustc/e4c626dd9a17a23270bf8e7158e59cf2b9c04840/library/core/src/iter/adapters/filter_map.rs:63:9
  27:     0xaaaadfda697c - <alloc::vec::Vec<T> as alloc::vec::spec_from_iter_nested::SpecFromIterNested<T,I>>::from_iter::h1e8f504eb2a57fca
                               at /rustc/e4c626dd9a17a23270bf8e7158e59cf2b9c04840/library/alloc/src/vec/spec_from_iter_nested.rs:26:32
  28:     0xaaaadfda697c - <alloc::vec::Vec<T> as alloc::vec::spec_from_iter::SpecFromIter<T,I>>::from_iter::h623a693550cf64bf
                               at /rustc/e4c626dd9a17a23270bf8e7158e59cf2b9c04840/library/alloc/src/vec/spec_from_iter.rs:33:9
  29:     0xaaaadfdc1c38 - <alloc::vec::Vec<T> as core::iter::traits::collect::FromIterator<T>>::from_iter::h2de77b407c2f40f2
                               at /rustc/e4c626dd9a17a23270bf8e7158e59cf2b9c04840/library/alloc/src/vec/mod.rs:2791:9
  30:     0xaaaadfdc1c38 - core::iter::traits::iterator::Iterator::collect::h0dd014bab7446be5
                               at /rustc/e4c626dd9a17a23270bf8e7158e59cf2b9c04840/library/core/src/iter/traits/iterator.rs:2054:9
  31:     0xaaaadfdc1c38 - <alloc::collections::btree::map::BTreeMap<K,V> as core::iter::traits::collect::FromIterator<(K,V)>>::from_iter::h22863f7e61ac45e3
                               at /rustc/e4c626dd9a17a23270bf8e7158e59cf2b9c04840/library/alloc/src/collections/btree/map.rs:2221:34
  32:     0xaaaae08b52a8 - core::iter::traits::iterator::Iterator::collect::hc8cf914a27c78765
                               at /rustc/e4c626dd9a17a23270bf8e7158e59cf2b9c04840/library/core/src/iter/traits/iterator.rs:2054:9
  33:     0xaaaae08b52a8 - risingwave_hummock_sdk::compaction_group::hummock_version_ext::<impl risingwave_hummock_sdk::version::HummockVersion>::safe_epoch_table_watermarks::hff3ad4a6f0956dce
                               at /risingwave/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs:253:14
  34:     0xaaaae08b52a8 - risingwave_meta::hummock::manager::HummockManager::get_compact_tasks_impl::{{closure}}::h9bd75ef635aefcd8
                               at /risingwave/src/meta/src/hummock/manager/mod.rs:1098:53
  35:     0xaaaae08a149c - risingwave_meta::hummock::manager::HummockManager::get_compact_tasks::{{closure}}::ha9e8a093b2be0b8d
                               at /risingwave/src/meta/src/hummock/manager/mod.rs:1299:14
  36:     0xaaaae08a149c - risingwave_meta::hummock::manager::compaction::<impl risingwave_meta::hummock::manager::HummockManager>::handle_pull_task_event::{{closure}}::he8a25d7b1a1b2cef
                               at /risingwave/src/meta/src/hummock/manager/compaction.rs:154:26
  37:     0xaaaae0881a98 - risingwave_meta::hummock::manager::compaction::<impl risingwave_meta::hummock::manager::HummockManager>::compact_task_dedicated_event_handler::{{closure}}::{{closure}}::h92063555084f3035
                               at /risingwave/src/meta/src/hummock/manager/compaction.rs:228:184
  38:     0xaaaae0881a98 - risingwave_meta::hummock::manager::compaction::<impl risingwave_meta::hummock::manager::HummockManager>::compact_task_dedicated_event_handler::{{closure}}::{{closure}}::hd651ad10bad6b33b
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/macros/select.rs:524:49
  39:     0xaaaae0881a98 - <tokio::future::poll_fn::PollFn<F> as core::future::future::Future>::poll::hcd5e18ed2b6be97f
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/future/poll_fn.rs:58:9
  40:     0xaaaae0052fcc - risingwave_meta::hummock::manager::compaction::<impl risingwave_meta::hummock::manager::HummockManager>::compact_task_dedicated_event_handler::{{closure}}::h8451d65cf91d27bc
                               at /risingwave/src/meta/src/hummock/manager/compaction.rs:219:9
  41:     0xaaaae0052fcc - risingwave_meta::hummock::manager::HummockManager::compaction_event_loop::{{closure}}::h94822e1ae9bdbadb
                               at /risingwave/src/meta/src/hummock/manager/mod.rs:2910:14
  42:     0xaaaae0052fcc - <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll::h476c090aac889abd
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9
  43:     0xaaaae03aaf78 - tokio::runtime::task::core::Core<T,S>::poll::{{closure}}::h9770a785d0492d53
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/task/core.rs:328:17
  44:     0xaaaae03aaf78 - tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut::h6ba8ea6ea1484f41
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/loom/std/unsafe_cell.rs:16:9
  45:     0xaaaae03aaf78 - tokio::runtime::task::core::Core<T,S>::poll::h9e4a8288a7a9ed7a
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/task/core.rs:317:13
  46:     0xaaaae03d3078 - tokio::runtime::task::harness::poll_future::{{closure}}::h2e4e125eef586c1d
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/task/harness.rs:485:19
  47:     0xaaaae03d3078 - <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once::hb06318a14d7444b6
                               at /rustc/e4c626dd9a17a23270bf8e7158e59cf2b9c04840/library/core/src/panic/unwind_safe.rs:272:9
  48:     0xaaaae03d3078 - std::panicking::try::do_call::h170d083669dac603
                               at /rustc/e4c626dd9a17a23270bf8e7158e59cf2b9c04840/library/std/src/panicking.rs:552:40
  49:     0xaaaae03d3078 - std::panicking::try::h4accf5821f2a8db3
                               at /rustc/e4c626dd9a17a23270bf8e7158e59cf2b9c04840/library/std/src/panicking.rs:516:19
  50:     0xaaaae03d3078 - std::panic::catch_unwind::hbbc6946fbeb1a56e
                               at /rustc/e4c626dd9a17a23270bf8e7158e59cf2b9c04840/library/std/src/panic.rs:142:14
  51:     0xaaaae03d3078 - tokio::runtime::task::harness::poll_future::hfa4a562e702bf19a
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/task/harness.rs:473:18

To Reproduce

No response

Expected behavior

No response

How did you deploy RisingWave?

No response

The version of RisingWave

No response

Additional context

No response