cache(cache_mode): Tune eviction threshold in cache mode

fixes dragonflydb#4011

Signed-off-by: Stepan Bagritsevich <[email protected]>

src/server/engine_shard.cc

@@ -704,6 +704,7 @@ void EngineShard::RetireExpiredAndEvict() {
   { std::unique_lock lk(db_slice.GetSerializationMutex()); }
   constexpr double kTtlDeleteLimit = 200;
   constexpr double kRedLimitFactor = 0.1;
+  constexpr double kDenyOomThresholdMargin = 0.05;
 
   uint32_t traversed = GetMovingSum6(TTL_TRAVERSE);
   uint32_t deleted = GetMovingSum6(TTL_DELETE);
@@ -717,7 +718,11 @@ void EngineShard::RetireExpiredAndEvict() {
     ttl_delete_target = kTtlDeleteLimit * double(deleted) / (double(traversed) + 10);
   }
 
-  ssize_t eviction_redline = size_t(max_memory_limit * kRedLimitFactor) / shard_set->size();
+  const double deny_oom_threshold = 1.0 - std::max(ServerState::tlocal()->oom_deny_ratio, 0.0);
+  const double eviction_threshold =
+      std::max(deny_oom_threshold + kDenyOomThresholdMargin, kRedLimitFactor);
+
+  ssize_t eviction_redline = size_t(max_memory_limit * eviction_threshold) / shard_set->size();
 
   DbContext db_cntx;
   db_cntx.time_now_ms = GetCurrentTimeMs();
@@ -764,7 +769,10 @@ void EngineShard::CacheStats() {
   size_t delta = used_mem - last_cached_used_memory_;
   last_cached_used_memory_ = used_mem;
   size_t current = used_mem_current.fetch_add(delta, memory_order_relaxed) + delta;
-  ssize_t free_mem = max_memory_limit - current;
+
+  ssize_t max_memory = static_cast<ssize_t>(static_cast<double>(max_memory_limit) *
+                                            ServerState::tlocal()->oom_deny_ratio);
+  ssize_t free_mem = max_memory - static_cast<ssize_t>(current);
 
   size_t entries = db_slice.entries_count();
   size_t table_memory = db_slice.table_memory();

tests/dragonfly/memory_test.py

@@ -107,3 +107,44 @@ async def test_rss_oom_ratio(df_factory: DflyInstanceFactory, admin_port):
     # new client create shoud not fail after memory usage decrease
     client = df_server.client()
     await client.execute_command("set x y")
+
+
+@pytest.mark.asyncio
+@dfly_args(
+    {
+        "proactor_threads": 1,
+        "cache_mode": "true",
+        "maxmemory": "256mb",
+        "oom_deny_ratio": 0.3,
+        "enable_heartbeat_eviction": "true",
+    }
+)
+async def test_cache_eviction_with_deny_oom(
+    df_server: DflyInstance,
+):
+    """
+    Test to verify that cache eviction is triggered before reaching the deny OOM threshold.
+    """
+
+    client = df_server.client()
+
+    max_memory = 256 * 1024 * 1024  # 256 MB in bytes
+    oom_deny_ratio = 0.3
+    deny_oom_threshold = int(max_memory * oom_deny_ratio)  # 77 MB
+    key_size = 1048576  # 1 MB per key
+
+    # Populate the cache up to the deny OOM threshold (77 MB)
+    await client.execute_command("DEBUG POPULATE", deny_oom_threshold // key_size, "size", key_size)
+
+    await asyncio.sleep(1)  # Wait for RSS heartbeat update in Dragonfly
+
+    info_before_eviction = await client.info("memory")
+    assert info_before_eviction["evicted_keys"] == 0, "No eviction should have occurred yet."
+
+    # Populate the cache further to trigger eviction
+    await client.execute_command("DEBUG POPULATE 200 size", key_size)
+
+    await asyncio.sleep(1)  # Wait for RSS heartbeat update in Dragonfly
+
+    info_after_eviction = await client.info("memory")
+    assert info_after_eviction["evicted_keys"] > 0, "Eviction should have occurred."