From 605331012b713baf94bb184c1f3a545198ad91b6 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Thu, 2 May 2024 11:07:40 -0400 Subject: [PATCH 1/7] Bump 2.x to 2.15.0 (#13495) (#13506) Signed-off-by: Andriy Redko --- .ci/bwcVersions | 1 + libs/core/src/main/java/org/opensearch/Version.java | 1 + 2 files changed, 2 insertions(+) diff --git a/.ci/bwcVersions b/.ci/bwcVersions index f3a9aa2787a80..e490c6ae6b4ea 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -32,3 +32,4 @@ BWC_VERSION: - "2.13.0" - "2.13.1" - "2.14.0" + - "2.15.0" diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java index f312c484a4842..823eb110eaa10 100644 --- a/libs/core/src/main/java/org/opensearch/Version.java +++ b/libs/core/src/main/java/org/opensearch/Version.java @@ -103,6 +103,7 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_2_13_0 = new Version(2130099, org.apache.lucene.util.Version.LUCENE_9_10_0); public static final Version V_2_13_1 = new Version(2130199, org.apache.lucene.util.Version.LUCENE_9_10_0); public static final Version V_2_14_0 = new Version(2140099, org.apache.lucene.util.Version.LUCENE_9_10_0); + public static final Version V_2_15_0 = new Version(2150099, org.apache.lucene.util.Version.LUCENE_9_10_0); public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_11_0); public static final Version CURRENT = V_3_0_0; From b09d9ef9d499119e20eacbf8ed459f929ac1b648 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Thu, 2 May 2024 13:12:59 -0400 Subject: [PATCH 2/7] Fix org.opensearch.backwards.MixedClusterClientYamlTestSuiteIT.test {p0=search.aggregation/400_inner_hits/Include inner hits in top hits} (#13510) Signed-off-by: Andriy Redko --- .../rest-api-spec/test/search.aggregation/400_inner_hits.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/400_inner_hits.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/400_inner_hits.yml index d4584a251816e..cbfc92a40835c 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/400_inner_hits.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/400_inner_hits.yml @@ -36,6 +36,9 @@ setup: --- "Include inner hits in top hits": + - skip: + version: " - 2.13.99" + reason: "the regression was fixed in 2.14.0" - do: search: rest_total_hits_as_int: true From 5ff50a7e00698b0f448bc4840461c8518500d591 Mon Sep 17 00:00:00 2001 From: Martin Gaievski Date: Thu, 2 May 2024 11:37:53 -0700 Subject: [PATCH 3/7] Refactor implementations of query phase searcher, add empty QueryCollectorContext (#13481) * Refactor implementations of query hpase searcher by adding overloaded searchWith method Signed-off-by: Martin Gaievski * Switched to Empty context add rescoring interface Signed-off-by: Martin Gaievski * Changed if by simple null check for querycontext argument Signed-off-by: Martin Gaievski * Added Override annotation for searchWith method Signed-off-by: Martin Gaievski * Remove old override method from Concurrent Query Phase Searcher Signed-off-by: Martin Gaievski --------- Signed-off-by: Martin Gaievski --- CHANGELOG.md | 1 + .../query/ConcurrentQueryPhaseSearcher.java | 26 +++++++++----- .../search/query/QueryCollectorContext.java | 23 ++++++++++++ .../opensearch/search/query/QueryPhase.java | 36 +++++++++++++++---- .../query/RescoringQueryCollectorContext.java | 27 ++++++++++++++ .../search/query/TopDocsCollectorContext.java | 2 +- 6 files changed, 100 insertions(+), 15 deletions(-) create mode 100644 server/src/main/java/org/opensearch/search/query/RescoringQueryCollectorContext.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bff49af99473..ca2b174c98001 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,6 +71,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [BWC and API enforcement] Reconsider the breaking changes check policy to detect breaking changes against released versions ([#13292](https://github.com/opensearch-project/OpenSearch/pull/13292)) - Switch to macos-13 runner for precommit and assemble github actions due to macos-latest is now arm64 ([#13412](https://github.com/opensearch-project/OpenSearch/pull/13412)) - [Revert] Prevent unnecessary fetch sub phase processor initialization during fetch phase execution ([#12503](https://github.com/opensearch-project/OpenSearch/pull/12503)) +- Refactor implementations of query phase searcher, allow QueryCollectorContext to have zero collectors ([#13481](https://github.com/opensearch-project/OpenSearch/pull/13481)) ### Deprecated diff --git a/server/src/main/java/org/opensearch/search/query/ConcurrentQueryPhaseSearcher.java b/server/src/main/java/org/opensearch/search/query/ConcurrentQueryPhaseSearcher.java index e22f766d3894c..771ac60dfb5e5 100644 --- a/server/src/main/java/org/opensearch/search/query/ConcurrentQueryPhaseSearcher.java +++ b/server/src/main/java/org/opensearch/search/query/ConcurrentQueryPhaseSearcher.java @@ -23,10 +23,9 @@ import java.io.IOException; import java.util.LinkedList; +import java.util.Objects; import java.util.concurrent.ExecutionException; -import static org.opensearch.search.query.TopDocsCollectorContext.createTopDocsCollectorContext; - /** * The implementation of the {@link QueryPhaseSearcher} which attempts to use concurrent * search of Apache Lucene segments if it has been enabled. @@ -46,10 +45,19 @@ protected boolean searchWithCollector( ContextIndexSearcher searcher, Query query, LinkedList collectors, + QueryCollectorContext queryCollectorContext, boolean hasFilterCollector, boolean hasTimeout ) throws IOException { - return searchWithCollectorManager(searchContext, searcher, query, collectors, hasFilterCollector, hasTimeout); + return searchWithCollectorManager( + searchContext, + searcher, + query, + collectors, + queryCollectorContext, + hasFilterCollector, + hasTimeout + ); } private static boolean searchWithCollectorManager( @@ -57,13 +65,12 @@ private static boolean searchWithCollectorManager( ContextIndexSearcher searcher, Query query, LinkedList collectorContexts, + QueryCollectorContext queryCollectorContext, boolean hasFilterCollector, boolean timeoutSet ) throws IOException { - // create the top docs collector last when the other collectors are known - final TopDocsCollectorContext topDocsFactory = createTopDocsCollectorContext(searchContext, hasFilterCollector); - // add the top docs collector, the first collector context in the chain - collectorContexts.addFirst(topDocsFactory); + // add the passed collector, the first collector context in the chain + collectorContexts.addFirst(Objects.requireNonNull(queryCollectorContext)); final QuerySearchResult queryResult = searchContext.queryResult(); final CollectorManager collectorManager; @@ -95,7 +102,10 @@ private static boolean searchWithCollectorManager( queryResult.terminatedEarly(false); } - return topDocsFactory.shouldRescore(); + if (queryCollectorContext instanceof RescoringQueryCollectorContext) { + return ((RescoringQueryCollectorContext) queryCollectorContext).shouldRescore(); + } + return false; } @Override diff --git a/server/src/main/java/org/opensearch/search/query/QueryCollectorContext.java b/server/src/main/java/org/opensearch/search/query/QueryCollectorContext.java index 91762bee2ac08..08b048cf682bb 100644 --- a/server/src/main/java/org/opensearch/search/query/QueryCollectorContext.java +++ b/server/src/main/java/org/opensearch/search/query/QueryCollectorContext.java @@ -77,6 +77,29 @@ public ScoreMode scoreMode() { } }; + public static final QueryCollectorContext EMPTY_CONTEXT = new QueryCollectorContext("empty") { + + @Override + Collector create(Collector in) throws IOException { + return EMPTY_COLLECTOR; + } + + @Override + CollectorManager createManager(CollectorManager in) throws IOException { + return new CollectorManager() { + @Override + public Collector newCollector() throws IOException { + return EMPTY_COLLECTOR; + } + + @Override + public ReduceableSearchResult reduce(Collection collectors) throws IOException { + return result -> {}; + } + }; + } + }; + private String profilerName; QueryCollectorContext(String profilerName) { diff --git a/server/src/main/java/org/opensearch/search/query/QueryPhase.java b/server/src/main/java/org/opensearch/search/query/QueryPhase.java index 8f98f0d9efbd4..608649ad22b23 100644 --- a/server/src/main/java/org/opensearch/search/query/QueryPhase.java +++ b/server/src/main/java/org/opensearch/search/query/QueryPhase.java @@ -335,13 +335,12 @@ private static boolean searchWithCollector( ContextIndexSearcher searcher, Query query, LinkedList collectors, + QueryCollectorContext queryCollectorContext, boolean hasFilterCollector, boolean timeoutSet ) throws IOException { - // create the top docs collector last when the other collectors are known - final TopDocsCollectorContext topDocsFactory = createTopDocsCollectorContext(searchContext, hasFilterCollector); - // add the top docs collector, the first collector context in the chain - collectors.addFirst(topDocsFactory); + // add passed collector, the first collector context in the chain + collectors.addFirst(Objects.requireNonNull(queryCollectorContext)); final Collector queryCollector; if (searchContext.getProfilers() != null) { @@ -370,7 +369,10 @@ private static boolean searchWithCollector( for (QueryCollectorContext ctx : collectors) { ctx.postProcess(queryResult); } - return topDocsFactory.shouldRescore(); + if (queryCollectorContext instanceof RescoringQueryCollectorContext) { + return ((RescoringQueryCollectorContext) queryCollectorContext).shouldRescore(); + } + return false; } /** @@ -440,7 +442,29 @@ protected boolean searchWithCollector( boolean hasFilterCollector, boolean hasTimeout ) throws IOException { - return QueryPhase.searchWithCollector(searchContext, searcher, query, collectors, hasFilterCollector, hasTimeout); + // create the top docs collector last when the other collectors are known + final TopDocsCollectorContext topDocsFactory = createTopDocsCollectorContext(searchContext, hasFilterCollector); + return searchWithCollector(searchContext, searcher, query, collectors, topDocsFactory, hasFilterCollector, hasTimeout); + } + + protected boolean searchWithCollector( + SearchContext searchContext, + ContextIndexSearcher searcher, + Query query, + LinkedList collectors, + QueryCollectorContext queryCollectorContext, + boolean hasFilterCollector, + boolean hasTimeout + ) throws IOException { + return QueryPhase.searchWithCollector( + searchContext, + searcher, + query, + collectors, + queryCollectorContext, + hasFilterCollector, + hasTimeout + ); } } } diff --git a/server/src/main/java/org/opensearch/search/query/RescoringQueryCollectorContext.java b/server/src/main/java/org/opensearch/search/query/RescoringQueryCollectorContext.java new file mode 100644 index 0000000000000..dc47ee80dba42 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/query/RescoringQueryCollectorContext.java @@ -0,0 +1,27 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.query; + +import org.opensearch.common.annotation.PublicApi; + +/** + * Abstraction that allows indication of whether results should be rescored or not based on + * custom logic of exact {@link QueryCollectorContext} implementation. + * + * @opensearch.api + */ +@PublicApi(since = "2.15.0") +public interface RescoringQueryCollectorContext { + + /** + * Indicates if results from the query context should be rescored + * @return true if results must be rescored, false otherwise + */ + boolean shouldRescore(); +} diff --git a/server/src/main/java/org/opensearch/search/query/TopDocsCollectorContext.java b/server/src/main/java/org/opensearch/search/query/TopDocsCollectorContext.java index 65d3948c8401e..f780f6fe32af2 100644 --- a/server/src/main/java/org/opensearch/search/query/TopDocsCollectorContext.java +++ b/server/src/main/java/org/opensearch/search/query/TopDocsCollectorContext.java @@ -95,7 +95,7 @@ * * @opensearch.internal */ -public abstract class TopDocsCollectorContext extends QueryCollectorContext { +public abstract class TopDocsCollectorContext extends QueryCollectorContext implements RescoringQueryCollectorContext { protected final int numHits; TopDocsCollectorContext(String profilerName, int numHits) { From 3b0cd0aef0097715041b65f5f0c942d1451b71d3 Mon Sep 17 00:00:00 2001 From: Rahul Karajgikar <50844303+rahulkarajgikar@users.noreply.github.com> Date: Fri, 3 May 2024 00:49:16 +0530 Subject: [PATCH 4/7] Increase cluster publish timeout in flaky test testStaleClusterManagerNotHijackingMajority (#13463) Signed-off-by: Rahul Karajgikar Co-authored-by: Rahul Karajgikar --- .../opensearch/discovery/StableClusterManagerDisruptionIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/discovery/StableClusterManagerDisruptionIT.java b/server/src/internalClusterTest/java/org/opensearch/discovery/StableClusterManagerDisruptionIT.java index f6f2b2dbfd096..4de79a31d0146 100644 --- a/server/src/internalClusterTest/java/org/opensearch/discovery/StableClusterManagerDisruptionIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/discovery/StableClusterManagerDisruptionIT.java @@ -203,7 +203,7 @@ public void testStaleClusterManagerNotHijackingMajority() throws Exception { 3, Settings.builder() .put(LeaderChecker.LEADER_CHECK_TIMEOUT_SETTING.getKey(), "1s") - .put(Coordinator.PUBLISH_TIMEOUT_SETTING.getKey(), "1s") + .put(Coordinator.PUBLISH_TIMEOUT_SETTING.getKey(), "2s") .build() ); ensureStableCluster(3); From dc77c46de2cb3dd4aac8ba1eac667466ee3d0b4e Mon Sep 17 00:00:00 2001 From: Harsha Vamsi Kalluri Date: Thu, 2 May 2024 13:15:46 -0700 Subject: [PATCH 5/7] Allow doc value search on unindexed boolean and date fields (#11650) * Making fields searchable Signed-off-by: Harsha Vamsi Kalluri * Adding tests for boolean fields Signed-off-by: Harsha Vamsi Kalluri * Added tests for date fields Signed-off-by: Harsha Vamsi Kalluri * Updating termsQuery logic Signed-off-by: Harsha Vamsi Kalluri * Spotless Signed-off-by: Harsha Vamsi Kalluri * Updating terms test Signed-off-by: Harsha Vamsi Kalluri * Spotless Signed-off-by: Harsha Vamsi Kalluri * Ensure that the points are intersecting for doc_values Signed-off-by: Harsha Vamsi Kalluri * Spotless Signed-off-by: Harsha Vamsi Kalluri * Adding missing javadocs Signed-off-by: Harsha Vamsi Kalluri * Adding more tests for indexedValueForSearch Signed-off-by: Harsha Vamsi Kalluri * Adding MatchAllDocsQuery to asserts Signed-off-by: Harsha Vamsi Kalluri * Fix changelog Signed-off-by: Harsha Vamsi Kalluri * Simplify loop criteria + remove IndexOrDocValuesQuery Signed-off-by: Harsha Vamsi Kalluri * Add some comments Signed-off-by: Harsha Vamsi Kalluri * Fix indendation Signed-off-by: Harsha Vamsi Kalluri * Fixing boolean field tests Signed-off-by: Harsha Vamsi Kalluri * Revert to correct logic for termsQuery Signed-off-by: Harsha Vamsi Kalluri * Update terms logic to be more succinct Signed-off-by: Harsha Vamsi Kalluri * Fix terms test Signed-off-by: Harsha Vamsi Kalluri * Adding tests for boolean range query + fix range query to use term query inside Signed-off-by: Harsha Vamsi Kalluri * Checking if upper and lower terms are valid Signed-off-by: Harsha Vamsi Kalluri * Checking if upper and lower terms are valid Signed-off-by: Harsha Vamsi Kalluri * Fixing changelog Signed-off-by: Harsha Vamsi Kalluri --------- Signed-off-by: Harsha Vamsi Kalluri --- CHANGELOG.md | 1 + .../test/search/340_doc_values_field.yml | 381 +++++++++++++++++- .../index/mapper/BooleanFieldMapper.java | 92 ++++- .../index/mapper/DateFieldMapper.java | 21 +- .../index/mapper/BooleanFieldMapperTests.java | 25 ++ .../index/mapper/BooleanFieldTypeTests.java | 69 +++- .../index/mapper/DateFieldTypeTests.java | 8 +- .../index/query/TermsQueryBuilderTests.java | 4 + 8 files changed, 573 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca2b174c98001..0ac0d64558e75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,6 +70,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Refactoring globMatch using simpleMatchWithNormalizedStrings from Regex ([#13104](https://github.com/opensearch-project/OpenSearch/pull/13104)) - [BWC and API enforcement] Reconsider the breaking changes check policy to detect breaking changes against released versions ([#13292](https://github.com/opensearch-project/OpenSearch/pull/13292)) - Switch to macos-13 runner for precommit and assemble github actions due to macos-latest is now arm64 ([#13412](https://github.com/opensearch-project/OpenSearch/pull/13412)) +- Add ability for Boolean and date field queries to run when only doc_values are enabled ([#11650](https://github.com/opensearch-project/OpenSearch/pull/11650)) - [Revert] Prevent unnecessary fetch sub phase processor initialization during fetch phase execution ([#12503](https://github.com/opensearch-project/OpenSearch/pull/12503)) - Refactor implementations of query phase searcher, allow QueryCollectorContext to have zero collectors ([#13481](https://github.com/opensearch-project/OpenSearch/pull/13481)) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml index d5ece1719dc48..a133060f07c6f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml @@ -46,6 +46,18 @@ type: ip index: true doc_values: true + boolean: + type: boolean + index: true + doc_values: true + date: + type: date + index: true + doc_values: true + date_nanos: + type: date_nanos + index: true + doc_values: true - do: bulk: @@ -53,11 +65,11 @@ refresh: true body: - '{"index": {"_index": "test-iodvq", "_id": "1" }}' - - '{ "some_keyword": "ingesting some random keyword data", "byte": 120, "double": 100.0, "float": "800.0", "half_float": "400.0", "integer": 1290, "long": 13456, "short": 150, "unsigned_long": 10223372036854775800, "ip_field": "192.168.0.1" }' + - '{ "some_keyword": "ingesting some random keyword data", "byte": 120, "double": 100.0, "float": "800.0", "half_float": "400.0", "integer": 1290, "long": 13456, "short": 150, "unsigned_long": 10223372036854775800, "ip_field": "192.168.0.1", "boolean": true, "date_nanos": "2018-10-29T12:12:12.123456789Z", "date": "2018-10-29T12:12:12.987Z" }' - '{ "index": { "_index": "test-iodvq", "_id": "2" }}' - - '{ "some_keyword": "400", "byte": 121, "double": 101.0, "float": "801.0", "half_float": "401.0", "integer": 1291, "long": 13457, "short": 151, "unsigned_long": 10223372036854775801, "ip_field": "192.168.0.2" }' + - '{ "some_keyword": "400", "byte": 121, "double": 101.0, "float": "801.0", "half_float": "401.0", "integer": 1291, "long": 13457, "short": 151, "unsigned_long": 10223372036854775801, "ip_field": "192.168.0.2", "boolean": true, "date_nanos": "2020-10-29T12:12:12.987654321Z", "date": "2020-10-29T12:12:12.987Z" }' - '{ "index": { "_index": "test-iodvq", "_id": "3" } }' - - '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802, "ip_field": "192.168.0.3" }' + - '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802, "ip_field": "192.168.0.3", "boolean": false, "date_nanos": "2024-10-29T12:12:12.987654321Z", "date": "2024-10-29T12:12:12.987Z" }' - do: search: @@ -183,6 +195,17 @@ - match: {hits.total: 1} + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + term: + boolean: true + + - match: { hits.total: 2 } + - do: search: rest_total_hits_as_int: true @@ -282,6 +305,17 @@ - match: { hits.total: 2 } + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + terms: + boolean: [true, false] + + - match: { hits.total: 3 } + - do: search: rest_total_hits_as_int: true @@ -415,6 +449,92 @@ - match: { hits.total: 2 } + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + range: { + date_nanos: { + gte: "2018-10-29T12:12:12.123456789Z" + }, + } + + - match: { hits.total: 3 } + + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + range: { + date: { + gte: "2018-10-29T12:12:12.987Z", + lte: "2020-10-29T12:12:12.987Z" + }, + } + + - match: { hits.total: 2 } + + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + range: { + boolean: { + gte: true + }, + } + + - match: { hits.total: 2 } + + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + range: { + boolean: { + lte: true + }, + } + + - match: { hits.total: 3 } + + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + range: { + boolean: { + lte: true, + gte: false + }, + } + + - match: { hits.total: 3 } + + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + range: { + boolean: { + lte: false, + gte: true + }, + } + + - match: { hits.total: 0 } --- "search on fields with only index enabled": - do: @@ -463,6 +583,18 @@ type: ip index: true doc_values: false + boolean: + type: boolean + index: true + doc_values: false + date_nanos: + type: date_nanos + index: true + doc_values: false + date: + type: date + index: true + doc_values: false - do: bulk: @@ -470,11 +602,11 @@ refresh: true body: - '{"index": {"_index": "test-index", "_id": "1" }}' - - '{ "some_keyword": "ingesting some random keyword data", "byte": 120, "double": 100.0, "float": "800.0", "half_float": "400.0", "integer": 1290, "long": 13456, "short": 150, "unsigned_long": 10223372036854775800, "ip_field": "192.168.0.1" }' + - '{ "some_keyword": "ingesting some random keyword data", "byte": 120, "double": 100.0, "float": "800.0", "half_float": "400.0", "integer": 1290, "long": 13456, "short": 150, "unsigned_long": 10223372036854775800, "ip_field": "192.168.0.1", "boolean": true, "date_nanos": "2018-10-29T12:12:12.123456789Z", "date": "2018-10-29T12:12:12.987Z" }' - '{ "index": { "_index": "test-index", "_id": "2" }}' - - '{ "some_keyword": "400", "byte": 121, "double": 101.0, "float": "801.0", "half_float": "401.0", "integer": 1291, "long": 13457, "short": 151, "unsigned_long": 10223372036854775801, "ip_field": "192.168.0.2" }' + - '{ "some_keyword": "400", "byte": 121, "double": 101.0, "float": "801.0", "half_float": "401.0", "integer": 1291, "long": 13457, "short": 151, "unsigned_long": 10223372036854775801, "ip_field": "192.168.0.2", "boolean": true, "date_nanos": "2020-10-29T12:12:12.123456789Z", "date": "2020-10-29T12:12:12.987Z" }' - '{ "index": { "_index": "test-index", "_id": "3" } }' - - '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802, "ip_field": "192.168.0.3" }' + - '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802, "ip_field": "192.168.0.3", "boolean": false, "date_nanos": "2024-10-29T12:12:12.123456789Z", "date": "2024-10-29T12:12:12.987Z" }' - do: search: @@ -600,6 +732,18 @@ - match: {hits.total: 1} + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + term: + boolean: true + + - match: { hits.total: 2 } + + - do: search: rest_total_hits_as_int: true @@ -699,6 +843,17 @@ - match: { hits.total: 2 } + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + terms: + boolean: [true, false] + + - match: { hits.total: 3 } + - do: search: rest_total_hits_as_int: true @@ -831,6 +986,93 @@ lte: "192.168.0.2" - match: { hits.total: 2 } + + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + range: { + date_nanos: { + gte: "2018-10-29T12:12:12.123456789Z" + }, + } + + - match: { hits.total: 3 } + + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + range: { + date: { + gte: "2018-10-29T12:12:12.987Z", + lte: "2020-10-29T12:12:12.987Z" + }, + } + + - match: { hits.total: 2 } + + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + range: { + boolean: { + gte: true + }, + } + + - match: { hits.total: 2 } + + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + range: { + boolean: { + lte: true + }, + } + + - match: { hits.total: 3 } + + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + range: { + boolean: { + lte: true, + gte: false + }, + } + + - match: { hits.total: 3 } + + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + range: { + boolean: { + lte: false, + gte: true + }, + } + + - match: { hits.total: 0 } --- "search on fields with only doc_values enabled": - skip: @@ -883,6 +1125,18 @@ type: ip index: false doc_values: true + boolean: + type: boolean + index: false + doc_values: true + date_nanos: + type: date_nanos + index: false + doc_values: true + date: + type: date + index: false + doc_values: true - do: bulk: @@ -890,11 +1144,11 @@ refresh: true body: - '{"index": {"_index": "test-doc-values", "_id": "1" }}' - - '{ "some_keyword": "ingesting some random keyword data", "byte": 120, "double": 100.0, "float": "800.0", "half_float": "400.0", "integer": 1290, "long": 13456, "short": 150, "unsigned_long": 10223372036854775800, "ip_field": "192.168.0.1" }' + - '{ "some_keyword": "ingesting some random keyword data", "byte": 120, "double": 100.0, "float": "800.0", "half_float": "400.0", "integer": 1290, "long": 13456, "short": 150, "unsigned_long": 10223372036854775800, "ip_field": "192.168.0.1", "boolean": true, "date_nanos": "2018-10-29T12:12:12.123456789Z", "date": "2018-10-29T12:12:12.987Z" }' - '{ "index": { "_index": "test-doc-values", "_id": "2" }}' - - '{ "some_keyword": "400", "byte": 121, "double": 101.0, "float": "801.0", "half_float": "401.0", "integer": 1291, "long": 13457, "short": 151, "unsigned_long": 10223372036854775801, "ip_field": "192.168.0.2" }' + - '{ "some_keyword": "400", "byte": 121, "double": 101.0, "float": "801.0", "half_float": "401.0", "integer": 1291, "long": 13457, "short": 151, "unsigned_long": 10223372036854775801, "ip_field": "192.168.0.2", "boolean": true, "date_nanos": "2020-10-29T12:12:12.123456789Z", "date": "2020-10-29T12:12:12.987Z" }' - '{ "index": { "_index": "test-doc-values", "_id": "3" } }' - - '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802, "ip_field": "192.168.0.3" }' + - '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802, "ip_field": "192.168.0.3", "boolean": false, "date_nanos": "2024-10-29T12:12:12.123456789Z", "date": "2024-10-29T12:12:12.987Z" }' - do: search: @@ -1019,6 +1273,17 @@ - match: { hits.total: 1 } + - do: + search: + rest_total_hits_as_int: true + index: test-doc-values + body: + query: + term: + boolean: false + + - match: { hits.total: 1 } + - do: search: rest_total_hits_as_int: true @@ -1107,6 +1372,17 @@ - match: { hits.total: 2 } + - do: + search: + rest_total_hits_as_int: true + index: test-doc-values + body: + query: + terms: + boolean: [true, false] + + - match: { hits.total: 3 } + - do: search: rest_total_hits_as_int: true @@ -1239,3 +1515,90 @@ lte: "192.168.0.2" - match: { hits.total: 2 } + + - do: + search: + rest_total_hits_as_int: true + index: test-doc-values + body: + query: + range: { + date_nanos: { + gte: "2018-10-29T12:12:12.123456789Z" + }, + } + + - match: { hits.total: 3 } + + - do: + search: + rest_total_hits_as_int: true + index: test-doc-values + body: + query: + range: { + date: { + gte: "2018-10-29T12:12:12.987Z", + lte: "2020-10-29T12:12:12.987Z" + }, + } + + - match: { hits.total: 2 } + + - do: + search: + rest_total_hits_as_int: true + index: test-doc-values + body: + query: + range: { + boolean: { + gte: true + }, + } + + - match: { hits.total: 2 } + + - do: + search: + rest_total_hits_as_int: true + index: test-doc-values + body: + query: + range: { + boolean: { + lte: true + }, + } + + - match: { hits.total: 3 } + + - do: + search: + rest_total_hits_as_int: true + index: test-doc-values + body: + query: + range: { + boolean: { + lte: true, + gte: false + }, + } + + - match: { hits.total: 3 } + + - do: + search: + rest_total_hits_as_int: true + index: test-doc-values + body: + query: + range: { + boolean: { + lte: false, + gte: true + }, + } + + - match: { hits.total: 0 } diff --git a/server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java index 3c7925809415a..b4cf585c1329d 100644 --- a/server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java @@ -37,8 +37,11 @@ import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BoostQuery; +import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.TermRangeQuery; +import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRef; import org.opensearch.common.Booleans; import org.opensearch.common.Nullable; @@ -55,8 +58,10 @@ import java.time.ZoneId; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Supplier; /** @@ -175,6 +180,10 @@ public BooleanFieldType(String name, boolean searchable) { this(name, searchable, false, true, false, Collections.emptyMap()); } + public BooleanFieldType(String name, boolean searchable, boolean hasDocValues) { + this(name, searchable, false, hasDocValues, false, Collections.emptyMap()); + } + @Override public String typeName() { return CONTENT_TYPE; @@ -257,16 +266,81 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) { return DocValueFormat.BOOLEAN; } + @Override + public Query termQuery(Object value, QueryShardContext context) { + failIfNotIndexedAndNoDocValues(); + if (!isSearchable()) { + return SortedNumericDocValuesField.newSlowExactQuery(name(), Values.TRUE.bytesEquals(indexedValueForSearch(value)) ? 1 : 0); + } + Query query = new TermQuery(new Term(name(), indexedValueForSearch(value))); + if (boost() != 1f) { + query = new BoostQuery(query, boost()); + } + return query; + } + + @Override + public Query termsQuery(List values, QueryShardContext context) { + failIfNotIndexedAndNoDocValues(); + int distinct = 0; + Set distinctValues = new HashSet<>(values); + for (Object value : distinctValues) { + if (Values.TRUE.equals(indexedValueForSearch(value))) { + distinct |= 2; + } else if (Values.FALSE.equals(indexedValueForSearch(value))) { + distinct |= 1; + } + if (distinct == 3) { + return this.existsQuery(context); + } + } + switch (distinct) { + case 1: + return termQuery("false", context); + case 2: + return termQuery("true", context); + } + + return new MatchNoDocsQuery("Values did not contain True or False"); + } + @Override public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) { - failIfNotIndexed(); - return new TermRangeQuery( - name(), - lowerTerm == null ? null : indexedValueForSearch(lowerTerm), - upperTerm == null ? null : indexedValueForSearch(upperTerm), - includeLower, - includeUpper - ); + failIfNotIndexedAndNoDocValues(); + if (lowerTerm == null) { + lowerTerm = false; + includeLower = true; + + } + if (upperTerm == null) { + upperTerm = true; + includeUpper = true; + + } + + lowerTerm = indexedValueForSearch(lowerTerm); + upperTerm = indexedValueForSearch(upperTerm); + + if (lowerTerm == upperTerm) { + if (!includeLower || !includeUpper) { + return new MatchNoDocsQuery(); + } + return termQuery(lowerTerm.equals(Values.TRUE), context); + } + + if (lowerTerm.equals(Values.TRUE)) { + return new MatchNoDocsQuery(); + } + if (!includeLower && !includeUpper) { + return new MatchNoDocsQuery(); + } else if (!includeLower) { + return termQuery(true, context); + } else if (!includeUpper) { + return termQuery(false, context); + } else { + return this.existsQuery(context); + } + } } diff --git a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java index d98e6ea6af83d..b7ee3bb8ca3e3 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java @@ -457,22 +457,30 @@ public Query rangeQuery( @Nullable DateMathParser forcedDateParser, QueryShardContext context ) { - failIfNotIndexed(); + failIfNotIndexedAndNoDocValues(); if (relation == ShapeRelation.DISJOINT) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] does not support DISJOINT ranges"); } DateMathParser parser = forcedDateParser == null ? dateMathParser : forcedDateParser; return dateRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, parser, context, resolution, (l, u) -> { - Query query = LongPoint.newRangeQuery(name(), l, u); - if (hasDocValues()) { + if (isSearchable() && hasDocValues()) { + Query query = LongPoint.newRangeQuery(name(), l, u); Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u); query = new IndexOrDocValuesQuery(query, dvQuery); if (context.indexSortedOnField(name())) { query = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query); } + return query; + } + if (hasDocValues()) { + Query query = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u); + if (context.indexSortedOnField(name())) { + query = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query); + } + return query; } - return query; + return LongPoint.newRangeQuery(name(), l, u); }); } @@ -543,6 +551,7 @@ public static long parseToLong( @Override public Query distanceFeatureQuery(Object origin, String pivot, float boost, QueryShardContext context) { + failIfNotIndexedAndNoDocValues(); long originLong = parseToLong(origin, true, null, null, context::nowInMillis); TimeValue pivotTime = TimeValue.parseTimeValue(pivot, "distance_feature.pivot"); return resolution.distanceFeatureQuery(name(), boost, originLong, pivotTime); @@ -559,6 +568,10 @@ public Relation isFieldWithinQuery( DateMathParser dateParser, QueryRewriteContext context ) throws IOException { + // if we have only doc_values enabled we do not look at the BKD so we return an INTERSECTS by default + if (isSearchable() == false && hasDocValues()) { + return Relation.INTERSECTS; + } if (dateParser == null) { dateParser = this.dateMathParser; } diff --git a/server/src/test/java/org/opensearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/BooleanFieldMapperTests.java index 8dec03a353d16..5392bd6c358d3 100644 --- a/server/src/test/java/org/opensearch/index/mapper/BooleanFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/BooleanFieldMapperTests.java @@ -209,4 +209,29 @@ public void testBoosts() throws Exception { assertEquals(new BoostQuery(new TermQuery(new Term("field", "T")), 2.0f), ft.termQuery("true", null)); assertParseMaximalWarnings(); } + + public void testIndexedValueForSearch() throws Exception { + assertEquals(new BooleanFieldMapper.BooleanFieldType("bool").indexedValueForSearch(null), BooleanFieldMapper.Values.FALSE); + + assertEquals(new BooleanFieldMapper.BooleanFieldType("bool").indexedValueForSearch(false), BooleanFieldMapper.Values.FALSE); + + assertEquals(new BooleanFieldMapper.BooleanFieldType("bool").indexedValueForSearch(true), BooleanFieldMapper.Values.TRUE); + + assertEquals( + new BooleanFieldMapper.BooleanFieldType("bool").indexedValueForSearch(new BytesRef("true")), + BooleanFieldMapper.Values.TRUE + ); + + assertEquals( + new BooleanFieldMapper.BooleanFieldType("bool").indexedValueForSearch(new BytesRef("false")), + BooleanFieldMapper.Values.FALSE + ); + + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> new BooleanFieldMapper.BooleanFieldType("bool").indexedValueForSearch(new BytesRef("random")) + ); + + assertEquals("Can't parse boolean value [random], expected [true] or [false]", e.getMessage()); + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/BooleanFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/BooleanFieldTypeTests.java index 14092706411cb..aab63fc30efd7 100644 --- a/server/src/test/java/org/opensearch/index/mapper/BooleanFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/BooleanFieldTypeTests.java @@ -31,11 +31,18 @@ package org.opensearch.index.mapper; +import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.index.Term; +import org.apache.lucene.search.BoostQuery; +import org.apache.lucene.search.DocValuesFieldExistsQuery; +import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.TermQuery; +import org.apache.lucene.util.BytesRef; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; public class BooleanFieldTypeTests extends FieldTypeTestCase { @@ -59,9 +66,67 @@ public void testTermQuery() { assertEquals(new TermQuery(new Term("field", "T")), ft.termQuery("true", null)); assertEquals(new TermQuery(new Term("field", "F")), ft.termQuery("false", null)); - MappedFieldType unsearchable = new BooleanFieldMapper.BooleanFieldType("field", false); + MappedFieldType doc_ft = new BooleanFieldMapper.BooleanFieldType("field", false, true); + assertEquals(SortedNumericDocValuesField.newSlowExactQuery("field", 1), doc_ft.termQuery("true", null)); + assertEquals(SortedNumericDocValuesField.newSlowExactQuery("field", 0), doc_ft.termQuery("false", null)); + + MappedFieldType boost_ft = new BooleanFieldMapper.BooleanFieldType("field"); + boost_ft.setBoost(2f); + assertEquals(new BoostQuery(new TermQuery(new Term("field", "T")), 2f), boost_ft.termQuery("true", null)); + assertEquals(new BoostQuery(new TermQuery(new Term("field", "F")), 2f), boost_ft.termQuery("false", null)); + + MappedFieldType unsearchable = new BooleanFieldMapper.BooleanFieldType("field", false, false); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("true", null)); - assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage()); + assertEquals("Cannot search on field [field] since it is both not indexed, and does not have doc_values enabled.", e.getMessage()); + } + + public void testTermsQuery() { + MappedFieldType ft = new BooleanFieldMapper.BooleanFieldType("field"); + List terms = new ArrayList<>(); + terms.add(new BytesRef("true")); + terms.add(new BytesRef("false")); + assertEquals(new DocValuesFieldExistsQuery("field"), ft.termsQuery(terms, null)); + + List newTerms = new ArrayList<>(); + newTerms.add(new BytesRef("true")); + assertEquals(new TermQuery(new Term("field", "T")), ft.termsQuery(newTerms, null)); + + List incorrectTerms = new ArrayList<>(); + incorrectTerms.add(new BytesRef("true")); + incorrectTerms.add(new BytesRef("random")); + IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> ft.termsQuery(incorrectTerms, null)); + assertEquals("Can't parse boolean value [random], expected [true] or [false]", ex.getMessage()); + + MappedFieldType doc_only_ft = new BooleanFieldMapper.BooleanFieldType("field", false, true); + + assertEquals(SortedNumericDocValuesField.newSlowExactQuery("field", 1), doc_only_ft.termsQuery(newTerms, null)); + + MappedFieldType unsearchable = new BooleanFieldMapper.BooleanFieldType("field", false, false); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termsQuery(terms, null)); + assertEquals("Cannot search on field [field] since it is both not indexed, and does not have doc_values enabled.", e.getMessage()); + } + + public void testRangeQuery() { + BooleanFieldMapper.BooleanFieldType ft = new BooleanFieldMapper.BooleanFieldType("field"); + assertEquals(new DocValuesFieldExistsQuery("field"), ft.rangeQuery(false, true, true, true, null)); + + assertEquals(new TermQuery(new Term("field", "T")), ft.rangeQuery(false, true, false, true, null)); + + assertEquals(new TermQuery(new Term("field", "F")), ft.rangeQuery(false, true, true, false, null)); + + assertEquals(new MatchNoDocsQuery(), ft.rangeQuery(false, true, false, false, null)); + + assertEquals(new MatchNoDocsQuery(), ft.rangeQuery(false, true, false, false, null)); + + assertEquals(new TermQuery(new Term("field", "F")), ft.rangeQuery(false, false, true, true, null)); + + assertEquals(new TermQuery(new Term("field", "F")), ft.rangeQuery(null, false, true, true, null)); + + assertEquals(new DocValuesFieldExistsQuery("field"), ft.rangeQuery(false, null, true, true, null)); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> ft.rangeQuery("random", null, true, true, null)); + + assertEquals("Can't parse boolean value [random], expected [true] or [false]", e.getMessage()); } public void testFetchSourceValue() throws IOException { diff --git a/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java index ab53ae81ab0ce..db5e1e419de93 100644 --- a/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java @@ -216,14 +216,14 @@ public void testTermQuery() { "field", false, false, - true, + false, DateFieldMapper.getDefaultDateTimeFormatter(), Resolution.MILLISECONDS, null, Collections.emptyMap() ); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery(date, context)); - assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage()); + assertEquals("Cannot search on field [field] since it is both not indexed, and does not have doc_values enabled.", e.getMessage()); } public void testRangeQuery() throws IOException { @@ -279,7 +279,7 @@ public void testRangeQuery() throws IOException { "field", false, false, - true, + false, DateFieldMapper.getDefaultDateTimeFormatter(), Resolution.MILLISECONDS, null, @@ -289,7 +289,7 @@ public void testRangeQuery() throws IOException { IllegalArgumentException.class, () -> unsearchable.rangeQuery(date1, date2, true, true, null, null, null, context) ); - assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage()); + assertEquals("Cannot search on field [field] since it is both not indexed, and does not have doc_values enabled.", e.getMessage()); } public void testRangeQueryWithIndexSort() { diff --git a/server/src/test/java/org/opensearch/index/query/TermsQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/TermsQueryBuilderTests.java index 32bf290627b63..61f7dc5facebe 100644 --- a/server/src/test/java/org/opensearch/index/query/TermsQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/TermsQueryBuilderTests.java @@ -34,7 +34,9 @@ import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.ConstantScoreQuery; +import org.apache.lucene.search.FieldExistsQuery; import org.apache.lucene.search.IndexOrDocValuesQuery; +import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.PointInSetQuery; import org.apache.lucene.search.Query; @@ -137,6 +139,8 @@ protected void doAssertLuceneQuery(TermsQueryBuilder queryBuilder, Query query, .or(instanceOf(ConstantScoreQuery.class)) .or(instanceOf(MatchNoDocsQuery.class)) .or(instanceOf(IndexOrDocValuesQuery.class)) + .or(instanceOf(MatchAllDocsQuery.class)) + .or(instanceOf(FieldExistsQuery.class)) ); if (query instanceof ConstantScoreQuery) { assertThat(((ConstantScoreQuery) query).getQuery(), instanceOf(BooleanQuery.class)); From 79957ebc03c4af746a31cdeb47158bad96553505 Mon Sep 17 00:00:00 2001 From: Chenyang Ji Date: Thu, 2 May 2024 14:48:23 -0700 Subject: [PATCH 6/7] Add support for deep copying SearchRequest (#12295) Signed-off-by: Chenyang Ji --- CHANGELOG.md | 1 + .../action/search/SearchRequest.java | 13 ++++++++ .../action/search/SearchRequestTests.java | 30 +++++++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ac0d64558e75..ba55a80f5d1d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Remote Store] Add settings for remote path type and hash algorithm ([#13225](https://github.com/opensearch-project/OpenSearch/pull/13225)) - [Remote Store] Upload remote paths during remote enabled index creation ([#13386](https://github.com/opensearch-project/OpenSearch/pull/13386)) - [Search Pipeline] Handle default pipeline for multiple indices ([#13276](https://github.com/opensearch-project/OpenSearch/pull/13276)) +- Add support for deep copying SearchRequest ([#12295](https://github.com/opensearch-project/OpenSearch/pull/12295)) ### Dependencies - Bump `org.apache.commons:commons-configuration2` from 2.10.0 to 2.10.1 ([#12896](https://github.com/opensearch-project/OpenSearch/pull/12896)) diff --git a/server/src/main/java/org/opensearch/action/search/SearchRequest.java b/server/src/main/java/org/opensearch/action/search/SearchRequest.java index f3d9f77e2394c..4d3bb868b779a 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchRequest.java +++ b/server/src/main/java/org/opensearch/action/search/SearchRequest.java @@ -40,6 +40,7 @@ import org.opensearch.action.support.IndicesOptions; import org.opensearch.common.Nullable; import org.opensearch.common.annotation.PublicApi; +import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.Strings; import org.opensearch.core.common.io.stream.StreamInput; @@ -161,6 +162,18 @@ public SearchRequest(String[] indices, SearchSourceBuilder source) { this.source = source; } + /** + * Deep clone a SearchRequest + * + * @return a copy of the current SearchRequest + */ + public SearchRequest deepCopy() throws IOException { + BytesStreamOutput out = new BytesStreamOutput(); + this.writeTo(out); + StreamInput in = out.bytes().streamInput(); + return new SearchRequest(in); + } + /** * Creates a new sub-search request starting from the original search request that is provided. * For internal use only, allows to fork a search request into multiple search requests that will be executed independently. diff --git a/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java b/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java index 9ee314e77ca7e..40514c526f190 100644 --- a/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java +++ b/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java @@ -46,6 +46,7 @@ import org.opensearch.search.Scroll; import org.opensearch.search.builder.PointInTimeBuilder; import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.fetch.subphase.FetchSourceContext; import org.opensearch.search.rescore.QueryRescorerBuilder; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.VersionUtils; @@ -76,6 +77,35 @@ protected SearchRequest createSearchRequest() throws IOException { ); } + public void testClone() throws IOException { + SearchRequest searchRequest = new SearchRequest(); + SearchRequest clonedRequest = searchRequest.deepCopy(); + assertEquals(searchRequest.hashCode(), clonedRequest.hashCode()); + assertNotSame(searchRequest, clonedRequest); + + String[] includes = new String[] { "field1.*" }; + String[] excludes = new String[] { "field2.*" }; + FetchSourceContext fetchSourceContext = new FetchSourceContext(true, includes, excludes); + SearchSourceBuilder source = new SearchSourceBuilder().fetchSource(fetchSourceContext); + SearchRequest complexSearchRequest = createSearchRequest().source(source); + complexSearchRequest.requestCache(false); + complexSearchRequest.scroll(new TimeValue(1000)); + SearchRequest clonedComplexRequest = complexSearchRequest.deepCopy(); + assertEquals(complexSearchRequest.hashCode(), clonedComplexRequest.hashCode()); + assertNotSame(complexSearchRequest, clonedComplexRequest); + assertEquals(fetchSourceContext, clonedComplexRequest.source().fetchSource()); + assertNotSame(fetchSourceContext, clonedComplexRequest.source().fetchSource()); + // Change the value of the original includes array and excludes array + includes[0] = "new_field1.*"; + excludes[0] = "new_field2.*"; + // Values in the original fetchSource object should be updated + assertEquals("new_field1.*", complexSearchRequest.source().fetchSource().includes()[0]); + assertEquals("new_field2.*", complexSearchRequest.source().fetchSource().excludes()[0]); + // Values in the cloned fetchSource object should not be updated + assertEquals("field1.*", clonedComplexRequest.source().fetchSource().includes()[0]); + assertEquals("field2.*", clonedComplexRequest.source().fetchSource().excludes()[0]); + } + public void testWithLocalReduction() { expectThrows(NullPointerException.class, () -> SearchRequest.subSearchRequest(null, Strings.EMPTY_ARRAY, "", 0, randomBoolean())); SearchRequest request = new SearchRequest(); From 37df569367c5f57ee80785dbd446bd7ed7e49faa Mon Sep 17 00:00:00 2001 From: Harsha Vamsi Kalluri Date: Thu, 2 May 2024 15:57:33 -0700 Subject: [PATCH 7/7] Update TermsQueryBuilderTests to include TermQuery (#13517) Signed-off-by: Harsha Vamsi Kalluri --- .../java/org/opensearch/index/query/TermsQueryBuilderTests.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/test/java/org/opensearch/index/query/TermsQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/TermsQueryBuilderTests.java index 61f7dc5facebe..97f372dc04a1b 100644 --- a/server/src/test/java/org/opensearch/index/query/TermsQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/TermsQueryBuilderTests.java @@ -41,6 +41,7 @@ import org.apache.lucene.search.PointInSetQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermInSetQuery; +import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRef; import org.opensearch.OpenSearchException; import org.opensearch.action.get.GetRequest; @@ -141,6 +142,7 @@ protected void doAssertLuceneQuery(TermsQueryBuilder queryBuilder, Query query, .or(instanceOf(IndexOrDocValuesQuery.class)) .or(instanceOf(MatchAllDocsQuery.class)) .or(instanceOf(FieldExistsQuery.class)) + .or(instanceOf(TermQuery.class)) ); if (query instanceof ConstantScoreQuery) { assertThat(((ConstantScoreQuery) query).getQuery(), instanceOf(BooleanQuery.class));