refactor: simplify block cache and file cache with foyer v0.8 (#16500)

Signed-off-by: MrCroxx <[email protected]>

Cargo.toml

@@ -78,7 +78,7 @@ license = "Apache-2.0"
 repository = "https://github.com/risingwavelabs/risingwave"
 
 [workspace.dependencies]
-foyer = "0.6"
+foyer = { version = "0.8.6", features = ["nightly"] }
 auto_enums = { version = "0.8", features = ["futures03", "tokio1"] }
 await-tree = "0.2.1"
 aws-config = { version = "1", default-features = false, features = [

src/batch/src/executor/insert.rs

@@ -267,7 +267,7 @@ mod tests {
     use std::sync::Arc;
 
     use assert_matches::assert_matches;
-    use foyer::memory::CacheContext;
+    use foyer::CacheContext;
     use futures::StreamExt;
     use itertools::Itertools;
     use risingwave_common::array::{Array, ArrayImpl, I32Array, StructArray};

src/common/src/catalog/mod.rs

@@ -33,6 +33,7 @@ use risingwave_pb::catalog::{
 };
 use risingwave_pb::plan_common::ColumnDescVersion;
 pub use schema::{test_utils as schema_test_utils, Field, FieldDisplay, Schema};
+use serde::{Deserialize, Serialize};
 
 use crate::array::DataChunk;
 pub use crate::constants::hummock;
@@ -228,7 +229,20 @@ impl From<SchemaId> for u32 {
     }
 }
 
-#[derive(Clone, Copy, Debug, Display, Default, Hash, PartialOrd, PartialEq, Eq, Ord)]
+#[derive(
+    Clone,
+    Copy,
+    Debug,
+    Display,
+    Default,
+    Hash,
+    PartialOrd,
+    PartialEq,
+    Eq,
+    Ord,
+    Serialize,
+    Deserialize,
+)]
 #[display("{table_id}")]
 pub struct TableId {
     pub table_id: u32,

src/common/src/config.rs

@@ -24,7 +24,7 @@ use std::num::NonZeroUsize;
 use anyhow::Context;
 use clap::ValueEnum;
 use educe::Educe;
-use foyer::memory::{LfuConfig, LruConfig, S3FifoConfig};
+use foyer::{LfuConfig, LruConfig, S3FifoConfig};
 use risingwave_common_proc_macro::ConfigDoc;
 pub use risingwave_common_proc_macro::OverrideConfig;
 use risingwave_pb::meta::SystemParams;
@@ -583,6 +583,8 @@ pub enum CacheEvictionConfig {
     },
     S3Fifo {
         small_queue_capacity_ratio_in_percent: Option<usize>,
+        ghost_queue_capacity_ratio_in_percent: Option<usize>,
+        small_to_main_freq_threshold: Option<u8>,
     },
 }
 
@@ -1347,6 +1349,14 @@ pub mod default {
             10
         }
 
+        pub fn ghost_queue_capacity_ratio_in_percent() -> usize {
+            1000
+        }
+
+        pub fn small_to_main_freq_threshold() -> u8 {
+            1
+        }
+
         pub fn meta_cache_capacity_mb() -> usize {
             128
         }
@@ -1889,6 +1899,16 @@ impl EvictionConfig {
     }
 }
 
+impl From<EvictionConfig> for foyer::EvictionConfig {
+    fn from(value: EvictionConfig) -> Self {
+        match value {
+            EvictionConfig::Lru(lru) => foyer::EvictionConfig::Lru(lru),
+            EvictionConfig::Lfu(lfu) => foyer::EvictionConfig::Lfu(lfu),
+            EvictionConfig::S3Fifo(s3fifo) => foyer::EvictionConfig::S3Fifo(s3fifo),
+        }
+    }
+}
+
 pub struct StorageMemoryConfig {
     pub block_cache_capacity_mb: usize,
     pub block_cache_shard_num: usize,
@@ -1975,11 +1995,19 @@ pub fn extract_storage_memory_config(s: &RwConfig) -> StorageMemoryConfig {
             }),
             CacheEvictionConfig::S3Fifo {
                 small_queue_capacity_ratio_in_percent,
+                ghost_queue_capacity_ratio_in_percent,
+                small_to_main_freq_threshold,
             } => EvictionConfig::S3Fifo(S3FifoConfig {
                 small_queue_capacity_ratio: small_queue_capacity_ratio_in_percent
                     .unwrap_or(default::storage::small_queue_capacity_ratio_in_percent())
                     as f64
                     / 100.0,
+                ghost_queue_capacity_ratio: ghost_queue_capacity_ratio_in_percent
+                    .unwrap_or(default::storage::ghost_queue_capacity_ratio_in_percent())
+                    as f64
+                    / 100.0,
+                small_to_main_freq_threshold: small_to_main_freq_threshold
+                    .unwrap_or(default::storage::small_to_main_freq_threshold()),
             }),
         }
     };

src/compute/src/memory/config.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use foyer::memory::{LfuConfig, LruConfig, S3FifoConfig};
+use foyer::{LfuConfig, LruConfig, S3FifoConfig};
 use risingwave_common::config::{
     CacheEvictionConfig, EvictionConfig, StorageConfig, StorageMemoryConfig,
     MAX_BLOCK_CACHE_SHARD_BITS, MAX_META_CACHE_SHARD_BITS, MIN_BUFFER_SIZE_PER_SHARD,
@@ -211,12 +211,22 @@ pub fn storage_memory_config(
         }),
         CacheEvictionConfig::S3Fifo {
             small_queue_capacity_ratio_in_percent,
+            ghost_queue_capacity_ratio_in_percent,
+            small_to_main_freq_threshold,
         } => EvictionConfig::S3Fifo(S3FifoConfig {
             small_queue_capacity_ratio: small_queue_capacity_ratio_in_percent.unwrap_or(
                 risingwave_common::config::default::storage::small_queue_capacity_ratio_in_percent(
                 ),
             ) as f64
                 / 100.0,
+            ghost_queue_capacity_ratio: ghost_queue_capacity_ratio_in_percent.unwrap_or(
+                risingwave_common::config::default::storage::ghost_queue_capacity_ratio_in_percent(
+                ),
+            ) as f64
+                / 100.0,
+            small_to_main_freq_threshold: small_to_main_freq_threshold.unwrap_or(
+                risingwave_common::config::default::storage::small_to_main_freq_threshold(),
+            ),
         }),
     };
 

src/ctl/Cargo.toml

@@ -20,6 +20,7 @@ chrono = "0.4"
 clap = { workspace = true }
 comfy-table = "7"
 etcd-client = { workspace = true }
+foyer ={ workspace = true }
 futures = { version = "0.3", default-features = false, features = ["alloc"] }
 hex = "0.4"
 inquire = "0.7.0"