- fixed search handler

- upgraded libraries
EinsamHauer · Oct 31, 2022 · ed38701 · ed38701
1 parent 970dc50
commit ed38701
Show file tree

Hide file tree

Showing 5 changed files with 81 additions and 11 deletions.
diff --git a/pom.xml b/pom.xml
@@ -4,7 +4,7 @@
     <groupId>net.iponweb.disthene.reader</groupId>
     <artifactId>disthene-reader</artifactId>
     <packaging>jar</packaging>
-    <version>2.0.15</version>
+    <version>2.0.16</version>
     <name>disthene-reader</name>
     <url>https://maven.apache.org</url>
 
@@ -18,7 +18,7 @@
         <dependency>
             <groupId>io.netty</groupId>
             <artifactId>netty-all</artifactId>
-            <version>4.1.82.Final</version>
+            <version>4.1.84.Final</version>
         </dependency>
         <dependency>
             <groupId>commons-cli</groupId>
@@ -33,7 +33,7 @@
         <dependency>
             <groupId>com.google.code.gson</groupId>
             <artifactId>gson</artifactId>
-            <version>2.9.0</version>
+            <version>2.10</version>
         </dependency>
         <dependency>
             <groupId>com.google.guava</groupId>
@@ -43,7 +43,7 @@
         <dependency>
             <groupId>joda-time</groupId>
             <artifactId>joda-time</artifactId>
-            <version>2.11.2</version>
+            <version>2.12.0</version>
         </dependency>
         <dependency>
             <groupId>org.lz4</groupId>

diff --git a/src/main/java/net/iponweb/disthene/reader/DistheneReader.java b/src/main/java/net/iponweb/disthene/reader/DistheneReader.java
@@ -117,7 +117,7 @@ private void run() {
             readerServer.registerHandler(RENDER_PATH, renderHandler);
 
             logger.info("Creating search handler");
-            SearchHandler searchHandler = new SearchHandler(indexService, statsService);
+            SearchHandler searchHandler = new SearchHandler(indexService, statsService, distheneReaderConfiguration.getReader());
             readerServer.registerHandler(SEARCH_PATH, searchHandler);
 
             logger.info("Creating path stats handler");

diff --git a/src/main/java/net/iponweb/disthene/reader/config/IndexConfiguration.java b/src/main/java/net/iponweb/disthene/reader/config/IndexConfiguration.java
@@ -13,6 +13,7 @@ public class IndexConfiguration {
     private int scroll;
     private int timeout;
     private int maxPaths;
+    private int maxSearchPaths = 100;
 
     public String getIndex() {
         return index;
@@ -62,6 +63,14 @@ public void setMaxPaths(int maxPaths) {
         this.maxPaths = maxPaths;
     }
 
+    public int getMaxSearchPaths() {
+        return maxSearchPaths;
+    }
+
+    public void setMaxSearchPaths(int maxSearchPaths) {
+        this.maxSearchPaths = maxSearchPaths;
+    }
+
     @Override
     public String toString() {
         return "IndexConfiguration{" +
@@ -71,6 +80,7 @@ public String toString() {
                 ", scroll=" + scroll +
                 ", timeout=" + timeout +
                 ", maxPaths=" + maxPaths +
+                ", maxSearchPaths=" + maxSearchPaths +
                 '}';
     }
 }
diff --git a/src/main/java/net/iponweb/disthene/reader/handler/SearchHandler.java b/src/main/java/net/iponweb/disthene/reader/handler/SearchHandler.java
@@ -1,8 +1,13 @@
 package net.iponweb.disthene.reader.handler;
 
 import com.google.common.base.Joiner;
+import com.google.common.base.Stopwatch;
+import com.google.common.util.concurrent.SimpleTimeLimiter;
+import com.google.common.util.concurrent.TimeLimiter;
+import com.google.common.util.concurrent.UncheckedTimeoutException;
 import io.netty.buffer.Unpooled;
 import io.netty.handler.codec.http.*;
+import net.iponweb.disthene.reader.config.ReaderConfiguration;
 import net.iponweb.disthene.reader.exceptions.MissingParameterException;
 import net.iponweb.disthene.reader.exceptions.ParameterParsingException;
 import net.iponweb.disthene.reader.exceptions.TooMuchDataExpectedException;
@@ -12,30 +17,57 @@
 import org.apache.logging.log4j.Logger;
 
 import java.io.IOException;
+import java.util.concurrent.*;
 
 /**
  * @author Andrei Ivanov
  */
+@SuppressWarnings("UnstableApiUsage")
 public class SearchHandler implements DistheneReaderHandler {
     private final static Logger logger = LogManager.getLogger(SearchHandler.class);
 
-    private final static int SEARCH_LIMIT = 100;
-
     private final IndexService indexService;
     private final StatsService statsService;
 
-    public SearchHandler(IndexService indexService, StatsService statsService) {
+    private final ReaderConfiguration readerConfiguration;
+
+    private static final ExecutorService executor = Executors.newCachedThreadPool();
+    private final TimeLimiter timeLimiter = SimpleTimeLimiter.create(executor);
+
+    public SearchHandler(IndexService indexService, StatsService statsService, ReaderConfiguration readerConfiguration) {
         this.indexService = indexService;
         this.statsService = statsService;
+        this.readerConfiguration = readerConfiguration;
     }
 
     @Override
     public FullHttpResponse handle(HttpRequest request) throws ParameterParsingException, IOException, TooMuchDataExpectedException {
         SearchParameters parameters = parse(request);
+        logger.debug("Got request: " + parameters);
+
+        Stopwatch timer = Stopwatch.createStarted();
 
         statsService.incPathsRequests(parameters.getTenant());
 
-        String pathsAsString = indexService.getSearchPathsAsString(parameters.getTenant(), parameters.getQuery(), SEARCH_LIMIT);
+        FullHttpResponse response;
+        try {
+            response = timeLimiter.callWithTimeout(() -> handleInternal(parameters), readerConfiguration.getRequestTimeout(), TimeUnit.SECONDS);
+        } catch (UncheckedTimeoutException e) {
+            logger.debug("Request timed out: " + parameters);
+            statsService.incTimedOutRequests(parameters.getTenant());
+            response = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE);
+        } catch (Exception e) {
+            response = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.BAD_REQUEST, Unpooled.wrappedBuffer(("Ohoho.. We have a weird problem: " + e.getCause().getMessage()).getBytes()));
+        }
+
+        timer.stop();
+        logger.debug("Request took " + timer.elapsed(TimeUnit.MILLISECONDS) + " milliseconds (" + parameters + ")");
+
+        return response;
+    }
+
+    private FullHttpResponse handleInternal(SearchParameters parameters) throws IOException {
+        String pathsAsString = indexService.getSearchPathsAsString(parameters.getTenant(), parameters.getQuery());
 
         FullHttpResponse response = new DefaultFullHttpResponse(
                 HttpVersion.HTTP_1_1,
@@ -96,5 +128,13 @@ String getQuery() {
         void setQuery(String query) {
             this.query = query;
         }
+
+        @Override
+        public String toString() {
+            return "SearchParameters{" +
+                    "tenant='" + tenant + '\'' +
+                    ", query='" + query + '\'' +
+                    '}';
+        }
     }
 }
diff --git a/src/main/java/net/iponweb/disthene/reader/service/index/IndexService.java b/src/main/java/net/iponweb/disthene/reader/service/index/IndexService.java
@@ -185,8 +185,28 @@ public String getPathsAsJsonArray(String tenant, String wildcard) throws TooMuch
         return "[" + joiner.join(paths) + "]";
     }
 
-    public String getSearchPathsAsString(String tenant, String regEx, int limit) throws IOException, TooMuchDataExpectedException {
-        return Joiner.on(",").skipNulls().join(getPathsFromRegExs(tenant, List.of(regEx), false, limit));
+    public String getSearchPathsAsString(String tenant, String regEx) throws IOException {
+        BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery()
+                .must(QueryBuilders.regexpQuery("path.keyword", regEx))
+                .filter(QueryBuilders.termQuery("tenant.keyword", tenant));
+
+        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder()
+                .fetchSource("path", null)
+                .query(queryBuilder)
+                .size(Math.min(indexConfiguration.getMaxSearchPaths(), maxResultWindow));
+
+        SearchRequest request = new SearchRequest(indexConfiguration.getIndex())
+                .source(sourceBuilder)
+                .requestCache(true);
+
+        SearchResponse response = client.search(request, RequestOptions.DEFAULT);
+
+        List<String> paths = new ArrayList<>();
+        for (SearchHit hit : response.getHits()) {
+            paths.add(hit.getSourceAsString());
+        }
+
+        return Joiner.on(",").skipNulls().join(paths);
     }
 
     public String getPathsWithStats(String tenant, String wildcard) throws TooMuchDataExpectedException, IOException {