diff --git a/pom.xml b/pom.xml index c3b26caa..90a3aac2 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ net.iponweb.disthene.reader disthene-reader jar - 2.0.15 + 2.0.16 disthene-reader https://maven.apache.org @@ -18,7 +18,7 @@ io.netty netty-all - 4.1.82.Final + 4.1.84.Final commons-cli @@ -33,7 +33,7 @@ com.google.code.gson gson - 2.9.0 + 2.10 com.google.guava @@ -43,7 +43,7 @@ joda-time joda-time - 2.11.2 + 2.12.0 org.lz4 diff --git a/src/main/java/net/iponweb/disthene/reader/DistheneReader.java b/src/main/java/net/iponweb/disthene/reader/DistheneReader.java index 1f351982..75c56648 100644 --- a/src/main/java/net/iponweb/disthene/reader/DistheneReader.java +++ b/src/main/java/net/iponweb/disthene/reader/DistheneReader.java @@ -117,7 +117,7 @@ private void run() { readerServer.registerHandler(RENDER_PATH, renderHandler); logger.info("Creating search handler"); - SearchHandler searchHandler = new SearchHandler(indexService, statsService); + SearchHandler searchHandler = new SearchHandler(indexService, statsService, distheneReaderConfiguration.getReader()); readerServer.registerHandler(SEARCH_PATH, searchHandler); logger.info("Creating path stats handler"); diff --git a/src/main/java/net/iponweb/disthene/reader/config/IndexConfiguration.java b/src/main/java/net/iponweb/disthene/reader/config/IndexConfiguration.java index 9391c0a8..69889e66 100644 --- a/src/main/java/net/iponweb/disthene/reader/config/IndexConfiguration.java +++ b/src/main/java/net/iponweb/disthene/reader/config/IndexConfiguration.java @@ -13,6 +13,7 @@ public class IndexConfiguration { private int scroll; private int timeout; private int maxPaths; + private int maxSearchPaths = 100; public String getIndex() { return index; @@ -62,6 +63,14 @@ public void setMaxPaths(int maxPaths) { this.maxPaths = maxPaths; } + public int getMaxSearchPaths() { + return maxSearchPaths; + } + + public void setMaxSearchPaths(int maxSearchPaths) { + this.maxSearchPaths = maxSearchPaths; + } + @Override public String toString() { return "IndexConfiguration{" + @@ -71,6 +80,7 @@ public String toString() { ", scroll=" + scroll + ", timeout=" + timeout + ", maxPaths=" + maxPaths + + ", maxSearchPaths=" + maxSearchPaths + '}'; } } diff --git a/src/main/java/net/iponweb/disthene/reader/handler/SearchHandler.java b/src/main/java/net/iponweb/disthene/reader/handler/SearchHandler.java index 58ccf77d..8a8c940c 100644 --- a/src/main/java/net/iponweb/disthene/reader/handler/SearchHandler.java +++ b/src/main/java/net/iponweb/disthene/reader/handler/SearchHandler.java @@ -1,8 +1,13 @@ package net.iponweb.disthene.reader.handler; import com.google.common.base.Joiner; +import com.google.common.base.Stopwatch; +import com.google.common.util.concurrent.SimpleTimeLimiter; +import com.google.common.util.concurrent.TimeLimiter; +import com.google.common.util.concurrent.UncheckedTimeoutException; import io.netty.buffer.Unpooled; import io.netty.handler.codec.http.*; +import net.iponweb.disthene.reader.config.ReaderConfiguration; import net.iponweb.disthene.reader.exceptions.MissingParameterException; import net.iponweb.disthene.reader.exceptions.ParameterParsingException; import net.iponweb.disthene.reader.exceptions.TooMuchDataExpectedException; @@ -12,30 +17,57 @@ import org.apache.logging.log4j.Logger; import java.io.IOException; +import java.util.concurrent.*; /** * @author Andrei Ivanov */ +@SuppressWarnings("UnstableApiUsage") public class SearchHandler implements DistheneReaderHandler { private final static Logger logger = LogManager.getLogger(SearchHandler.class); - private final static int SEARCH_LIMIT = 100; - private final IndexService indexService; private final StatsService statsService; - public SearchHandler(IndexService indexService, StatsService statsService) { + private final ReaderConfiguration readerConfiguration; + + private static final ExecutorService executor = Executors.newCachedThreadPool(); + private final TimeLimiter timeLimiter = SimpleTimeLimiter.create(executor); + + public SearchHandler(IndexService indexService, StatsService statsService, ReaderConfiguration readerConfiguration) { this.indexService = indexService; this.statsService = statsService; + this.readerConfiguration = readerConfiguration; } @Override public FullHttpResponse handle(HttpRequest request) throws ParameterParsingException, IOException, TooMuchDataExpectedException { SearchParameters parameters = parse(request); + logger.debug("Got request: " + parameters); + + Stopwatch timer = Stopwatch.createStarted(); statsService.incPathsRequests(parameters.getTenant()); - String pathsAsString = indexService.getSearchPathsAsString(parameters.getTenant(), parameters.getQuery(), SEARCH_LIMIT); + FullHttpResponse response; + try { + response = timeLimiter.callWithTimeout(() -> handleInternal(parameters), readerConfiguration.getRequestTimeout(), TimeUnit.SECONDS); + } catch (UncheckedTimeoutException e) { + logger.debug("Request timed out: " + parameters); + statsService.incTimedOutRequests(parameters.getTenant()); + response = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE); + } catch (Exception e) { + response = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.BAD_REQUEST, Unpooled.wrappedBuffer(("Ohoho.. We have a weird problem: " + e.getCause().getMessage()).getBytes())); + } + + timer.stop(); + logger.debug("Request took " + timer.elapsed(TimeUnit.MILLISECONDS) + " milliseconds (" + parameters + ")"); + + return response; + } + + private FullHttpResponse handleInternal(SearchParameters parameters) throws IOException { + String pathsAsString = indexService.getSearchPathsAsString(parameters.getTenant(), parameters.getQuery()); FullHttpResponse response = new DefaultFullHttpResponse( HttpVersion.HTTP_1_1, @@ -96,5 +128,13 @@ String getQuery() { void setQuery(String query) { this.query = query; } + + @Override + public String toString() { + return "SearchParameters{" + + "tenant='" + tenant + '\'' + + ", query='" + query + '\'' + + '}'; + } } } diff --git a/src/main/java/net/iponweb/disthene/reader/service/index/IndexService.java b/src/main/java/net/iponweb/disthene/reader/service/index/IndexService.java index b4dcc946..17097640 100644 --- a/src/main/java/net/iponweb/disthene/reader/service/index/IndexService.java +++ b/src/main/java/net/iponweb/disthene/reader/service/index/IndexService.java @@ -185,8 +185,28 @@ public String getPathsAsJsonArray(String tenant, String wildcard) throws TooMuch return "[" + joiner.join(paths) + "]"; } - public String getSearchPathsAsString(String tenant, String regEx, int limit) throws IOException, TooMuchDataExpectedException { - return Joiner.on(",").skipNulls().join(getPathsFromRegExs(tenant, List.of(regEx), false, limit)); + public String getSearchPathsAsString(String tenant, String regEx) throws IOException { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery() + .must(QueryBuilders.regexpQuery("path.keyword", regEx)) + .filter(QueryBuilders.termQuery("tenant.keyword", tenant)); + + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() + .fetchSource("path", null) + .query(queryBuilder) + .size(Math.min(indexConfiguration.getMaxSearchPaths(), maxResultWindow)); + + SearchRequest request = new SearchRequest(indexConfiguration.getIndex()) + .source(sourceBuilder) + .requestCache(true); + + SearchResponse response = client.search(request, RequestOptions.DEFAULT); + + List paths = new ArrayList<>(); + for (SearchHit hit : response.getHits()) { + paths.add(hit.getSourceAsString()); + } + + return Joiner.on(",").skipNulls().join(paths); } public String getPathsWithStats(String tenant, String wildcard) throws TooMuchDataExpectedException, IOException {