feat: improve object storage cache (#2522)

* feat: refactor object storage cache with moka * chore: minor fixes * fix: concurrent issues and invalidate cache after write/delete * chore: minor changes * fix: cargo lock * refactor: rename * chore: change DEFAULT_OBJECT_STORE_CACHE_SIZE to 256Mib * fix: typo * chore: style * fix: toml format * chore: toml * fix: toml format * Update src/object-store/src/layers/lru_cache/read_cache.rs Co-authored-by: Ruihang Xia <[email protected]> * chore: update Cargo.toml Co-authored-by: Yingwen <[email protected]> * chore: update src/object-store/Cargo.toml Co-authored-by: Yingwen <[email protected]> * chore: refactor and apply suggestions * fix: typo * feat: adds back allow list for caching * chore: cr suggestion Co-authored-by: Yingwen <[email protected]> * chore: cr suggestion Co-authored-by: Yingwen <[email protected]> * refactor: wrap inner Accessor with Arc * chore: remove run_pending_task in read and write path * chore: the arc is unnecessary --------- Co-authored-by: Ruihang Xia <[email protected]> Co-authored-by: Yingwen <[email protected]>
GreptimeTeam · Oct 8, 2023 · ff15bc4 · ff15bc4
1 parent 657542c
commit ff15bc4
Show file tree

Hide file tree

Showing 15 changed files with 500 additions and 305 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -82,7 +82,7 @@ greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", r
 humantime-serde = "1.1"
 itertools = "0.10"
 lazy_static = "1.4"
-moka = { version = "0.11" }
+moka = "0.12"
 once_cell = "1.18"
 opentelemetry-proto = { version = "0.2", features = ["gen-tonic", "metrics"] }
 parquet = "43.0"

diff --git a/config/datanode.example.toml b/config/datanode.example.toml
@@ -47,6 +47,12 @@ type = "File"
 # TTL for all tables. Disabled by default.
 # global_ttl = "7d"
 
+# Cache configuration for object storage such as 'S3' etc.
+# The local file cache directory
+# cache_path = "/path/local_cache"
+# The local file cache capacity in bytes.
+# cache_capacity = "256Mib"
+
 # Compaction options, see `standalone.example.toml`.
 [storage.compaction]
 max_inflight_tasks = 4

diff --git a/config/standalone.example.toml b/config/standalone.example.toml
@@ -115,6 +115,10 @@ data_home = "/tmp/greptimedb/"
 type = "File"
 # TTL for all tables. Disabled by default.
 # global_ttl = "7d"
+# Cache configuration for object storage such as 'S3' etc.
+# cache_path = "/path/local_cache"
+# The local file cache capacity in bytes.
+# cache_capacity = "256Mib"
 
 # Compaction options.
 [storage.compaction]

diff --git a/src/datanode/src/config.rs b/src/datanode/src/config.rs
@@ -37,7 +37,7 @@ use storage::config::{
 };
 use storage::scheduler::SchedulerConfig;
 
-pub const DEFAULT_OBJECT_STORE_CACHE_SIZE: ReadableSize = ReadableSize(1024);
+pub const DEFAULT_OBJECT_STORE_CACHE_SIZE: ReadableSize = ReadableSize::mb(256);
 
 /// Default data home in file storage
 const DEFAULT_DATA_HOME: &str = "/tmp/greptimedb";
@@ -90,6 +90,15 @@ impl Default for StorageConfig {
 #[serde(default)]
 pub struct FileConfig {}
 
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+#[serde(default)]
+pub struct ObjectStorageCacheConfig {
+    /// The local file cache directory
+    pub cache_path: Option<String>,
+    /// The cache capacity in bytes
+    pub cache_capacity: Option<ReadableSize>,
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize)]
 #[serde(default)]
 pub struct S3Config {
@@ -101,8 +110,8 @@ pub struct S3Config {
     pub secret_access_key: SecretString,
     pub endpoint: Option<String>,
     pub region: Option<String>,
-    pub cache_path: Option<String>,
-    pub cache_capacity: Option<ReadableSize>,
+    #[serde(flatten)]
+    pub cache: ObjectStorageCacheConfig,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -115,8 +124,8 @@ pub struct OssConfig {
     #[serde(skip_serializing)]
     pub access_key_secret: SecretString,
     pub endpoint: String,
-    pub cache_path: Option<String>,
-    pub cache_capacity: Option<ReadableSize>,
+    #[serde(flatten)]
+    pub cache: ObjectStorageCacheConfig,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -130,8 +139,8 @@ pub struct AzblobConfig {
     pub account_key: SecretString,
     pub endpoint: String,
     pub sas_token: Option<String>,
-    pub cache_path: Option<String>,
-    pub cache_capacity: Option<ReadableSize>,
+    #[serde(flatten)]
+    pub cache: ObjectStorageCacheConfig,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -143,8 +152,8 @@ pub struct GcsConfig {
     #[serde(skip_serializing)]
     pub credential_path: SecretString,
     pub endpoint: String,
-    pub cache_path: Option<String>,
-    pub cache_capacity: Option<ReadableSize>,
+    #[serde(flatten)]
+    pub cache: ObjectStorageCacheConfig,
 }
 
 impl Default for S3Config {
@@ -156,8 +165,7 @@ impl Default for S3Config {
             secret_access_key: SecretString::from(String::default()),
             endpoint: Option::default(),
             region: Option::default(),
-            cache_path: Option::default(),
-            cache_capacity: Option::default(),
+            cache: ObjectStorageCacheConfig::default(),
         }
     }
 }
@@ -170,8 +178,7 @@ impl Default for OssConfig {
             access_key_id: SecretString::from(String::default()),
             access_key_secret: SecretString::from(String::default()),
             endpoint: String::default(),
-            cache_path: Option::default(),
-            cache_capacity: Option::default(),
+            cache: ObjectStorageCacheConfig::default(),
         }
     }
 }
@@ -184,9 +191,8 @@ impl Default for AzblobConfig {
             account_name: SecretString::from(String::default()),
             account_key: SecretString::from(String::default()),
             endpoint: String::default(),
-            cache_path: Option::default(),
-            cache_capacity: Option::default(),
             sas_token: Option::default(),
+            cache: ObjectStorageCacheConfig::default(),
         }
     }
 }
@@ -199,8 +205,7 @@ impl Default for GcsConfig {
             scope: String::default(),
             credential_path: SecretString::from(String::default()),
             endpoint: String::default(),
-            cache_path: Option::default(),
-            cache_capacity: Option::default(),
+            cache: ObjectStorageCacheConfig::default(),
         }
     }
 }

diff --git a/src/datanode/src/datanode.rs b/src/datanode/src/datanode.rs
@@ -19,7 +19,6 @@ use std::sync::Arc;
 
 use catalog::kvbackend::MetaKvBackend;
 use catalog::memory::MemoryCatalogManager;
-use common_base::readable_size::ReadableSize;
 use common_base::Plugins;
 use common_error::ext::BoxedError;
 use common_greptimedb_telemetry::GreptimeDBTelemetryTask;
@@ -62,8 +61,6 @@ use crate::region_server::RegionServer;
 use crate::server::Services;
 use crate::store;
 
-pub const DEFAULT_OBJECT_STORE_CACHE_SIZE: ReadableSize = ReadableSize(1024);
-
 const OPEN_REGION_PARALLELISM: usize = 16;
 
 /// Datanode service.