From 7027569b0d5f17fbb767ce5663526bb3224dc6b1 Mon Sep 17 00:00:00 2001
From: Andrey Gubarev <60990599+andreygubarev@users.noreply.github.com>
Date: Fri, 6 Dec 2024 16:34:41 +0400
Subject: [PATCH] Performance improvements: use `multiSearchAny` instead of
 `IN` to support Clickhouse projections for logs volumes (#1052)

---
 CHANGELOG.md                  |  4 ++++
 src/data/CHDatasource.test.ts | 14 +++++++-------
 src/data/CHDatasource.ts      |  2 +-
 src/data/logs.test.ts         | 14 +++++++-------
 src/data/logs.ts              |  4 ++--
 5 files changed, 21 insertions(+), 17 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index bb132686..eedbbff9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,10 @@
 
 ## Unreleased
 
+### Fixes
+
+- Optimized performance for log volumes processing using ClickHouse `multiSearchAny`
+
 ## 4.5.1
 
 ### Fixes
diff --git a/src/data/CHDatasource.test.ts b/src/data/CHDatasource.test.ts
index 15996fc4..558ebc2f 100644
--- a/src/data/CHDatasource.test.ts
+++ b/src/data/CHDatasource.test.ts
@@ -477,13 +477,13 @@ describe('ClickHouseDatasource', () => {
         const result = datasource.getSupplementaryLogsVolumeQuery(request, query);
         expect(result?.rawSql).toEqual(
           `SELECT toStartOfInterval("created_at", INTERVAL 1 DAY) as "time", ` +
-            `sum(toString("level") IN ('critical','fatal','crit','alert','emerg','CRITICAL','FATAL','CRIT','ALERT','EMERG','Critical','Fatal','Crit','Alert','Emerg')) as critical, ` +
-            `sum(toString("level") IN ('error','err','eror','ERROR','ERR','EROR','Error','Err','Eror')) as error, ` +
-            `sum(toString("level") IN ('warn','warning','WARN','WARNING','Warn','Warning')) as warn, ` +
-            `sum(toString("level") IN ('info','information','informational','INFO','INFORMATION','INFORMATIONAL','Info','Information','Informational')) as info, ` +
-            `sum(toString("level") IN ('debug','dbug','DEBUG','DBUG','Debug','Dbug')) as debug, ` +
-            `sum(toString("level") IN ('trace','TRACE','Trace')) as trace, ` +
-            `sum(toString("level") IN ('unknown','UNKNOWN','Unknown')) as unknown ` +
+            `sum(multiSearchAny(toString("level"), ['critical','fatal','crit','alert','emerg','CRITICAL','FATAL','CRIT','ALERT','EMERG','Critical','Fatal','Crit','Alert','Emerg'])) as critical, ` +
+            `sum(multiSearchAny(toString("level"), ['error','err','eror','ERROR','ERR','EROR','Error','Err','Eror'])) as error, ` +
+            `sum(multiSearchAny(toString("level"), ['warn','warning','WARN','WARNING','Warn','Warning'])) as warn, ` +
+            `sum(multiSearchAny(toString("level"), ['info','information','informational','INFO','INFORMATION','INFORMATIONAL','Info','Information','Informational'])) as info, ` +
+            `sum(multiSearchAny(toString("level"), ['debug','dbug','DEBUG','DBUG','Debug','Dbug'])) as debug, ` +
+            `sum(multiSearchAny(toString("level"), ['trace','TRACE','Trace'])) as trace, ` +
+            `sum(multiSearchAny(toString("level"), ['unknown','UNKNOWN','Unknown'])) as unknown ` +
             `FROM "default"."logs" ` +
             `GROUP BY time ` +
             `ORDER BY time ASC`
diff --git a/src/data/CHDatasource.ts b/src/data/CHDatasource.ts
index ad41970b..0af13f63 100644
--- a/src/data/CHDatasource.ts
+++ b/src/data/CHDatasource.ts
@@ -157,7 +157,7 @@ export class Datasource
       const llf = `toString("${logLevelColumn.name}")`;
       let level: keyof typeof LOG_LEVEL_TO_IN_CLAUSE;
       for (level in LOG_LEVEL_TO_IN_CLAUSE) {
-        aggregates.push({ aggregateType: AggregateType.Sum, column: `${llf} ${LOG_LEVEL_TO_IN_CLAUSE[level]}`, alias: level });
+        aggregates.push({ aggregateType: AggregateType.Sum, column: `multiSearchAny(${llf}, [${LOG_LEVEL_TO_IN_CLAUSE[level]}])`, alias: level });
       }
     } else {
       // Count all logs if level column isn't selected
diff --git a/src/data/logs.test.ts b/src/data/logs.test.ts
index bb65483a..2766c99e 100644
--- a/src/data/logs.test.ts
+++ b/src/data/logs.test.ts
@@ -226,13 +226,13 @@ describe('logs', () => {
     it('should generate correct IN clauses', async () => {
       expect(LOG_LEVEL_TO_IN_CLAUSE).toEqual({
         critical:
-          "IN ('critical','fatal','crit','alert','emerg','CRITICAL','FATAL','CRIT','ALERT','EMERG','Critical','Fatal','Crit','Alert','Emerg')",
-        debug: "IN ('debug','dbug','DEBUG','DBUG','Debug','Dbug')",
-        error: "IN ('error','err','eror','ERROR','ERR','EROR','Error','Err','Eror')",
-        info: "IN ('info','information','informational','INFO','INFORMATION','INFORMATIONAL','Info','Information','Informational')",
-        trace: "IN ('trace','TRACE','Trace')",
-        unknown: "IN ('unknown','UNKNOWN','Unknown')",
-        warn: "IN ('warn','warning','WARN','WARNING','Warn','Warning')",
+          "'critical','fatal','crit','alert','emerg','CRITICAL','FATAL','CRIT','ALERT','EMERG','Critical','Fatal','Crit','Alert','Emerg'",
+        debug: "'debug','dbug','DEBUG','DBUG','Debug','Dbug'",
+        error: "'error','err','eror','ERROR','ERR','EROR','Error','Err','Eror'",
+        info: "'info','information','informational','INFO','INFORMATION','INFORMATIONAL','Info','Information','Informational'",
+        trace: "'trace','TRACE','Trace'",
+        unknown: "'unknown','UNKNOWN','Unknown'",
+        warn: "'warn','warning','WARN','WARNING','Warn','Warning'",
       });
     });
   });
diff --git a/src/data/logs.ts b/src/data/logs.ts
index 81f051eb..b0766d88 100644
--- a/src/data/logs.ts
+++ b/src/data/logs.ts
@@ -245,11 +245,11 @@ export const LOG_LEVEL_TO_IN_CLAUSE: LogLevelToInClause = (() => {
     unknown: ['unknown'],
   };
   return (Object.keys(levels) as Array<keyof typeof levels>).reduce((allLevels, level) => {
-    allLevels[level] = `IN (${[
+    allLevels[level] = `${[
       ...levels[level].map((l) => `'${l}'`),
       ...levels[level].map((l) => `'${l.toUpperCase()}'`),
       ...levels[level].map((l) => `'${l.charAt(0).toUpperCase() + l.slice(1)}'`),
-    ].join(',')})`;
+    ].join(',')}`;
     return allLevels;
   }, {} as LogLevelToInClause);
 })();