Introduce Thread-local storage variable to to reduce atomic contentio…

…n when update used_memory metrics.

Signed-off-by: Lipeng Zhu <[email protected]>

src/evict.c

@@ -676,12 +676,12 @@ int performEvictions(void) {
              * AOF and Output buffer memory will be freed eventually so
              * we only care about memory used by the key space. */
             enterExecutionUnit(1, 0);
-            delta = (long long)zmalloc_used_memory();
+            delta = (long long)zmalloc_used_memory_with_thread_delta();
             latencyStartMonitor(eviction_latency);
             dbGenericDelete(db, keyobj, server.lazyfree_lazy_eviction, DB_FLAG_KEY_EVICTED);
             latencyEndMonitor(eviction_latency);
             latencyAddSampleIfNeeded("eviction-del", eviction_latency);
-            delta -= (long long)zmalloc_used_memory();
+            delta -= (long long)zmalloc_used_memory_with_thread_delta();
             mem_freed += delta;
             server.stat_evictedkeys++;
             signalModifiedKey(NULL, db, keyobj);

src/zmalloc.c

@@ -87,10 +87,26 @@ void zlibc_free(void *ptr) {
 #define dallocx(ptr, flags) je_dallocx(ptr, flags)
 #endif
 
-#define update_zmalloc_stat_alloc(__n) atomic_fetch_add_explicit(&used_memory, (__n), memory_order_relaxed)
-#define update_zmalloc_stat_free(__n) atomic_fetch_sub_explicit(&used_memory, (__n), memory_order_relaxed)
+static _Atomic int64_t used_memory = 0;
+static __thread int64_t thread_used_memory_delta = 0;
 
-static _Atomic size_t used_memory = 0;
+#define THREAD_MEM_MAX_DELTA (1024 * 1024)
+
+static inline void update_zmalloc_stat_alloc(size_t size) {
+    thread_used_memory_delta += size;
+    if (thread_used_memory_delta >= THREAD_MEM_MAX_DELTA) {
+        atomic_fetch_add_explicit(&used_memory, thread_used_memory_delta, memory_order_relaxed);
+        thread_used_memory_delta = 0;
+    }
+}
+
+static inline void update_zmalloc_stat_free(size_t size) {
+    thread_used_memory_delta -= size;
+    if (thread_used_memory_delta <= THREAD_MEM_MAX_DELTA) {
+        atomic_fetch_add_explicit(&used_memory, thread_used_memory_delta, memory_order_relaxed);
+        thread_used_memory_delta = 0;
+    }
+}
 
 static void zmalloc_default_oom(size_t size) {
     fprintf(stderr, "zmalloc: Out of memory trying to allocate %zu bytes\n", size);
@@ -388,8 +404,16 @@ char *zstrdup(const char *s) {
 }
 
 size_t zmalloc_used_memory(void) {
-    size_t um = atomic_load_explicit(&used_memory, memory_order_relaxed);
-    return um;
+    /* Maybe become negative */
+    int64_t um = atomic_load_explicit(&used_memory, memory_order_relaxed);
+    return um < 0 ? 0 : um;
+}
+
+/* Flush the thread_used_memory_delta to global counter. */
+size_t zmalloc_used_memory_with_thread_delta(void) {
+    atomic_fetch_add_explicit(&used_memory, thread_used_memory_delta, memory_order_relaxed);
+    thread_used_memory_delta = 0;
+    return zmalloc_used_memory();
 }
 
 void zmalloc_set_oom_handler(void (*oom_handler)(size_t)) {

src/zmalloc.h

@@ -126,6 +126,7 @@ void *ztrycalloc_usable(size_t size, size_t *usable);
 void *ztryrealloc_usable(void *ptr, size_t size, size_t *usable);
 __attribute__((malloc)) char *zstrdup(const char *s);
 size_t zmalloc_used_memory(void);
+size_t zmalloc_used_memory_with_thread_delta(void);
 void zmalloc_set_oom_handler(void (*oom_handler)(size_t));
 size_t zmalloc_get_rss(void);
 int zmalloc_get_allocator_info(size_t *allocated,