diff --git a/pom.xml b/pom.xml
index c3b26caa..90a3aac2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
net.iponweb.disthene.reader
disthene-reader
jar
- 2.0.15
+ 2.0.16
disthene-reader
https://maven.apache.org
@@ -18,7 +18,7 @@
io.netty
netty-all
- 4.1.82.Final
+ 4.1.84.Final
commons-cli
@@ -33,7 +33,7 @@
com.google.code.gson
gson
- 2.9.0
+ 2.10
com.google.guava
@@ -43,7 +43,7 @@
joda-time
joda-time
- 2.11.2
+ 2.12.0
org.lz4
diff --git a/src/main/java/net/iponweb/disthene/reader/DistheneReader.java b/src/main/java/net/iponweb/disthene/reader/DistheneReader.java
index 1f351982..75c56648 100644
--- a/src/main/java/net/iponweb/disthene/reader/DistheneReader.java
+++ b/src/main/java/net/iponweb/disthene/reader/DistheneReader.java
@@ -117,7 +117,7 @@ private void run() {
readerServer.registerHandler(RENDER_PATH, renderHandler);
logger.info("Creating search handler");
- SearchHandler searchHandler = new SearchHandler(indexService, statsService);
+ SearchHandler searchHandler = new SearchHandler(indexService, statsService, distheneReaderConfiguration.getReader());
readerServer.registerHandler(SEARCH_PATH, searchHandler);
logger.info("Creating path stats handler");
diff --git a/src/main/java/net/iponweb/disthene/reader/config/IndexConfiguration.java b/src/main/java/net/iponweb/disthene/reader/config/IndexConfiguration.java
index 9391c0a8..69889e66 100644
--- a/src/main/java/net/iponweb/disthene/reader/config/IndexConfiguration.java
+++ b/src/main/java/net/iponweb/disthene/reader/config/IndexConfiguration.java
@@ -13,6 +13,7 @@ public class IndexConfiguration {
private int scroll;
private int timeout;
private int maxPaths;
+ private int maxSearchPaths = 100;
public String getIndex() {
return index;
@@ -62,6 +63,14 @@ public void setMaxPaths(int maxPaths) {
this.maxPaths = maxPaths;
}
+ public int getMaxSearchPaths() {
+ return maxSearchPaths;
+ }
+
+ public void setMaxSearchPaths(int maxSearchPaths) {
+ this.maxSearchPaths = maxSearchPaths;
+ }
+
@Override
public String toString() {
return "IndexConfiguration{" +
@@ -71,6 +80,7 @@ public String toString() {
", scroll=" + scroll +
", timeout=" + timeout +
", maxPaths=" + maxPaths +
+ ", maxSearchPaths=" + maxSearchPaths +
'}';
}
}
diff --git a/src/main/java/net/iponweb/disthene/reader/handler/SearchHandler.java b/src/main/java/net/iponweb/disthene/reader/handler/SearchHandler.java
index 58ccf77d..8a8c940c 100644
--- a/src/main/java/net/iponweb/disthene/reader/handler/SearchHandler.java
+++ b/src/main/java/net/iponweb/disthene/reader/handler/SearchHandler.java
@@ -1,8 +1,13 @@
package net.iponweb.disthene.reader.handler;
import com.google.common.base.Joiner;
+import com.google.common.base.Stopwatch;
+import com.google.common.util.concurrent.SimpleTimeLimiter;
+import com.google.common.util.concurrent.TimeLimiter;
+import com.google.common.util.concurrent.UncheckedTimeoutException;
import io.netty.buffer.Unpooled;
import io.netty.handler.codec.http.*;
+import net.iponweb.disthene.reader.config.ReaderConfiguration;
import net.iponweb.disthene.reader.exceptions.MissingParameterException;
import net.iponweb.disthene.reader.exceptions.ParameterParsingException;
import net.iponweb.disthene.reader.exceptions.TooMuchDataExpectedException;
@@ -12,30 +17,57 @@
import org.apache.logging.log4j.Logger;
import java.io.IOException;
+import java.util.concurrent.*;
/**
* @author Andrei Ivanov
*/
+@SuppressWarnings("UnstableApiUsage")
public class SearchHandler implements DistheneReaderHandler {
private final static Logger logger = LogManager.getLogger(SearchHandler.class);
- private final static int SEARCH_LIMIT = 100;
-
private final IndexService indexService;
private final StatsService statsService;
- public SearchHandler(IndexService indexService, StatsService statsService) {
+ private final ReaderConfiguration readerConfiguration;
+
+ private static final ExecutorService executor = Executors.newCachedThreadPool();
+ private final TimeLimiter timeLimiter = SimpleTimeLimiter.create(executor);
+
+ public SearchHandler(IndexService indexService, StatsService statsService, ReaderConfiguration readerConfiguration) {
this.indexService = indexService;
this.statsService = statsService;
+ this.readerConfiguration = readerConfiguration;
}
@Override
public FullHttpResponse handle(HttpRequest request) throws ParameterParsingException, IOException, TooMuchDataExpectedException {
SearchParameters parameters = parse(request);
+ logger.debug("Got request: " + parameters);
+
+ Stopwatch timer = Stopwatch.createStarted();
statsService.incPathsRequests(parameters.getTenant());
- String pathsAsString = indexService.getSearchPathsAsString(parameters.getTenant(), parameters.getQuery(), SEARCH_LIMIT);
+ FullHttpResponse response;
+ try {
+ response = timeLimiter.callWithTimeout(() -> handleInternal(parameters), readerConfiguration.getRequestTimeout(), TimeUnit.SECONDS);
+ } catch (UncheckedTimeoutException e) {
+ logger.debug("Request timed out: " + parameters);
+ statsService.incTimedOutRequests(parameters.getTenant());
+ response = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE);
+ } catch (Exception e) {
+ response = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.BAD_REQUEST, Unpooled.wrappedBuffer(("Ohoho.. We have a weird problem: " + e.getCause().getMessage()).getBytes()));
+ }
+
+ timer.stop();
+ logger.debug("Request took " + timer.elapsed(TimeUnit.MILLISECONDS) + " milliseconds (" + parameters + ")");
+
+ return response;
+ }
+
+ private FullHttpResponse handleInternal(SearchParameters parameters) throws IOException {
+ String pathsAsString = indexService.getSearchPathsAsString(parameters.getTenant(), parameters.getQuery());
FullHttpResponse response = new DefaultFullHttpResponse(
HttpVersion.HTTP_1_1,
@@ -96,5 +128,13 @@ String getQuery() {
void setQuery(String query) {
this.query = query;
}
+
+ @Override
+ public String toString() {
+ return "SearchParameters{" +
+ "tenant='" + tenant + '\'' +
+ ", query='" + query + '\'' +
+ '}';
+ }
}
}
diff --git a/src/main/java/net/iponweb/disthene/reader/service/index/IndexService.java b/src/main/java/net/iponweb/disthene/reader/service/index/IndexService.java
index b4dcc946..17097640 100644
--- a/src/main/java/net/iponweb/disthene/reader/service/index/IndexService.java
+++ b/src/main/java/net/iponweb/disthene/reader/service/index/IndexService.java
@@ -185,8 +185,28 @@ public String getPathsAsJsonArray(String tenant, String wildcard) throws TooMuch
return "[" + joiner.join(paths) + "]";
}
- public String getSearchPathsAsString(String tenant, String regEx, int limit) throws IOException, TooMuchDataExpectedException {
- return Joiner.on(",").skipNulls().join(getPathsFromRegExs(tenant, List.of(regEx), false, limit));
+ public String getSearchPathsAsString(String tenant, String regEx) throws IOException {
+ BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery()
+ .must(QueryBuilders.regexpQuery("path.keyword", regEx))
+ .filter(QueryBuilders.termQuery("tenant.keyword", tenant));
+
+ SearchSourceBuilder sourceBuilder = new SearchSourceBuilder()
+ .fetchSource("path", null)
+ .query(queryBuilder)
+ .size(Math.min(indexConfiguration.getMaxSearchPaths(), maxResultWindow));
+
+ SearchRequest request = new SearchRequest(indexConfiguration.getIndex())
+ .source(sourceBuilder)
+ .requestCache(true);
+
+ SearchResponse response = client.search(request, RequestOptions.DEFAULT);
+
+ List paths = new ArrayList<>();
+ for (SearchHit hit : response.getHits()) {
+ paths.add(hit.getSourceAsString());
+ }
+
+ return Joiner.on(",").skipNulls().join(paths);
}
public String getPathsWithStats(String tenant, String wildcard) throws TooMuchDataExpectedException, IOException {