Merge branch 'main' into main

Signed-off-by: Tommy Shao <[email protected]>
opensearch-project · Jan 24, 2025 · 5a836e3 · 5a836e3
2 parents accd092 + 13ab4ec
commit 5a836e3
Show file tree

Hide file tree

Showing 564 changed files with 6,833 additions and 4,348 deletions.
diff --git a/.github/workflows/assemble.yml b/.github/workflows/assemble.yml
@@ -7,35 +7,21 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        java: [ 11, 17, 21 ]
-        os: [ubuntu-latest, windows-latest, macos-13]
+        java: [ 21, 23 ]
+        os: [ubuntu-latest, windows-latest, macos-13, ubuntu-24.04-arm]
     steps:
       - uses: actions/checkout@v4
       - name: Set up JDK ${{ matrix.java }}
         uses: actions/setup-java@v4
         with:
           java-version: ${{ matrix.java }}
           distribution: temurin
-      - name: Set up JDK 17
-        # See please https://docs.gradle.org/8.10/userguide/upgrading_version_8.html#minimum_daemon_jvm_version
-        if: matrix.java == 11
-        uses: actions/setup-java@v4
-        with:
-          java-version: 17
-          distribution: temurin
-      - name: Set JAVA${{ matrix.java }}_HOME
-        shell: bash
-        run: |
-          echo "JAVA${{ matrix.java }}_HOME=$JAVA_HOME_${{ matrix.java }}_${{ runner.arch }}" >> $GITHUB_ENV
       - name: Setup docker (missing on MacOS)
         id: setup_docker
         if: runner.os == 'macos'
         continue-on-error: true
         run: |
-          # Force QEMU 9.0.2 usage
-          curl https://raw.githubusercontent.com/Homebrew/homebrew-core/f1a9cf104a9a51779c7a532b658c490f69974839/Formula/q/qemu.rb > qemu.rb
-          brew install qemu.rb
-          HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 HOMEBREW_NO_AUTO_UPDATE=1 brew install docker colima coreutils
+          brew install docker colima coreutils
           gtimeout 15m colima start
         shell: bash
       - name: Run Gradle (assemble)
@@ -47,8 +33,8 @@ jobs:
         shell: bash
         if: runner.os != 'macos'
         run: |
-          ./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE -Druntime.java=${{ matrix.java }} 
+          ./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE 
       - name: Run Gradle (assemble)
         if: runner.os == 'macos' && steps.setup_docker.outcome == 'success'
         run: |
-          ./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE -Druntime.java=${{ matrix.java }}
+          ./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE
diff --git a/.github/workflows/precommit.yml b/.github/workflows/precommit.yml
@@ -7,8 +7,8 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        java: [ 11, 17, 21, 23 ]
-        os: [ubuntu-latest, windows-latest, macos-latest, macos-13]
+        java: [ 21, 23 ]
+        os: [ubuntu-latest, windows-latest, macos-latest, macos-13, ubuntu-24.04-arm]
     steps:
       - uses: actions/checkout@v4
       - name: Set up JDK ${{ matrix.java }}
@@ -17,18 +17,7 @@ jobs:
           java-version: ${{ matrix.java }}
           distribution: temurin
           cache: gradle
-      - name: Set up JDK 17
-        # See please https://docs.gradle.org/8.10/userguide/upgrading_version_8.html#minimum_daemon_jvm_version
-        if: matrix.java == 11
-        uses: actions/setup-java@v4
-        with:
-          java-version: 17
-          distribution: temurin
-      - name: Set JAVA${{ matrix.java }}_HOME
-        shell: bash
-        run: |
-          echo "JAVA${{ matrix.java }}_HOME=$JAVA_HOME_${{ matrix.java }}_${{ runner.arch }}" >> $GITHUB_ENV
       - name: Run Gradle (precommit)
         shell: bash
         run: |
-          ./gradlew javadoc precommit --parallel -Druntime.java=${{ matrix.java }}
+          ./gradlew javadoc precommit --parallel
diff --git a/.github/workflows/publish-maven-snapshots.yml b/.github/workflows/publish-maven-snapshots.yml
@@ -19,11 +19,11 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-      - name: Set up JDK 17
+      - name: Set up JDK 21
         uses: actions/setup-java@v4
         with:
-          distribution: adopt
-          java-version: 17
+          distribution: temurin
+          java-version: 21
 
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4
@@ -37,4 +37,4 @@ jobs:
           export SONATYPE_PASSWORD=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-password --query SecretString --output text)
           echo "::add-mask::$SONATYPE_USERNAME"
           echo "::add-mask::$SONATYPE_PASSWORD"
-          ./gradlew publishNebulaPublicationToSnapshotsRepository
+          ./gradlew publishNebulaPublicationToSnapshotsRepository -Dbuild.version_qualifier=alpha1
diff --git a/CHANGELOG-3.0.md b/CHANGELOG-3.0.md
@@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Views, simplify data access and manipulation by providing a virtual layer over one or more indices ([#11957](https://github.com/opensearch-project/OpenSearch/pull/11957))
 
 ### Dependencies
+- Update Apache Lucene to 10.1.0 ([#16366](https://github.com/opensearch-project/OpenSearch/pull/16366))
 - Bump Apache HttpCore5/HttpClient5 dependencies from 5.2.5/5.3.1 to 5.3.1/5.4.1 to support ExtendedSocketOption in HttpAsyncClient ([#16757](https://github.com/opensearch-project/OpenSearch/pull/16757))
 
 ### Changed

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -24,6 +24,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add stats for remote publication failure and move download failure stats to remote methods([#16682](https://github.com/opensearch-project/OpenSearch/pull/16682/))
 - Update script supports java.lang.String.sha1() and java.lang.String.sha256() methods ([#16923](https://github.com/opensearch-project/OpenSearch/pull/16923))
 - Added a precaution to handle extreme date values during sorting to prevent `arithmetic_exception: long overflow` ([#16812](https://github.com/opensearch-project/OpenSearch/pull/16812)).
+- Add `verbose_pipeline` parameter to output each processor's execution details ([#16843](https://github.com/opensearch-project/OpenSearch/pull/16843)).
 - Add search replica stats to segment replication stats API ([#16678](https://github.com/opensearch-project/OpenSearch/pull/16678))
 - Introduce a setting to disable download of full cluster state from remote on term mismatch([#16798](https://github.com/opensearch-project/OpenSearch/pull/16798/))
 - Added ability to retrieve value from DocValues in a flat_object filed([#16802](https://github.com/opensearch-project/OpenSearch/pull/16802))
@@ -32,8 +33,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Support object fields in star-tree index([#16728](https://github.com/opensearch-project/OpenSearch/pull/16728/))
 - Support searching from doc_value using termQueryCaseInsensitive/termQuery in flat_object/keyword field([#16974](https://github.com/opensearch-project/OpenSearch/pull/16974/))
 - Added a new `time` field to replace the deprecated `getTime` field in `GetStats`. ([#17009](https://github.com/opensearch-project/OpenSearch/pull/17009))
+- Improve flat_object field parsing performance by reducing two passes to a single pass ([#16297](https://github.com/opensearch-project/OpenSearch/pull/16297))
 - Improve performance of the bitmap filtering([#16936](https://github.com/opensearch-project/OpenSearch/pull/16936/))
 - Added new Setting property UnmodifiableOnRestore to prevent updating settings on restore snapshot ([#16957](https://github.com/opensearch-project/OpenSearch/pull/16957))
+- Introduce Template query ([#16818](https://github.com/opensearch-project/OpenSearch/pull/16818))
 
 ### Dependencies
 - Bump `com.google.cloud:google-cloud-core-http` from 2.23.0 to 2.47.0 ([#16504](https://github.com/opensearch-project/OpenSearch/pull/16504))
@@ -68,6 +71,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `opentelemetry-semconv` from 1.27.0-alpha to 1.29.0-alpha ([#16700](https://github.com/opensearch-project/OpenSearch/pull/16700))
 - Bump `com.google.re2j:re2j` from 1.7 to 1.8 ([#17012](https://github.com/opensearch-project/OpenSearch/pull/17012))
 - Bump `com.squareup.okio:okio` from 3.9.1 to 3.10.2 ([#17060](https://github.com/opensearch-project/OpenSearch/pull/17060))
+- Bump `org.jruby.jcodings:jcodings` from 1.0.58 to 1.0.61 ([#17061](https://github.com/opensearch-project/OpenSearch/pull/17061))
 
 ### Changed
 - Indexed IP field supports `terms_query` with more than 1025 IP masks [#16391](https://github.com/opensearch-project/OpenSearch/pull/16391)
@@ -76,6 +80,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Allow extended plugins to be optional ([#16909](https://github.com/opensearch-project/OpenSearch/pull/16909))
 - Use the correct type to widen the sort fields when merging top docs ([#16881](https://github.com/opensearch-project/OpenSearch/pull/16881))
 - Limit reader writer separation to remote store enabled clusters [#16760](https://github.com/opensearch-project/OpenSearch/pull/16760)
+- Optimize innerhits query performance [#16937](https://github.com/opensearch-project/OpenSearch/pull/16937)
 
 ### Deprecated
 - Performing update operation with default pipeline or final pipeline is deprecated ([#16712](https://github.com/opensearch-project/OpenSearch/pull/16712))
@@ -106,6 +111,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix multi-value sort for unsigned long ([#16732](https://github.com/opensearch-project/OpenSearch/pull/16732))
 - The `phone-search` analyzer no longer emits the tel/sip prefix, international calling code, extension numbers and unformatted input as a token ([#16993](https://github.com/opensearch-project/OpenSearch/pull/16993))
 - Fix GRPC AUX_TRANSPORT_PORT and SETTING_GRPC_PORT settings and remove lingering HTTP terminology ([#17037](https://github.com/opensearch-project/OpenSearch/pull/17037))
+- Fix exists queries on nested flat_object fields throws exception ([#16803](https://github.com/opensearch-project/OpenSearch/pull/16803))
 
 ### Security
 

diff --git a/benchmarks/build.gradle b/benchmarks/build.gradle
@@ -85,44 +85,5 @@ spotless {
   }
 }
 
-if (BuildParams.runtimeJavaVersion >= JavaVersion.VERSION_20) {
-  // Add support for incubator modules on supported Java versions.
-  run.jvmArgs += ['--add-modules=jdk.incubator.vector']
-  run.classpath += files(jar.archiveFile)
-  run.classpath -= sourceSets.main.output
-  evaluationDependsOn(':libs:opensearch-common')
-
-  sourceSets {
-    java20 {
-      java {
-        srcDirs = ['src/main/java20']
-      }
-    }
-  }
-
-  configurations {
-    java20Implementation.extendsFrom(implementation)
-  }
-
-  dependencies {
-    java20Implementation sourceSets.main.output
-    java20Implementation project(':libs:opensearch-common').sourceSets.java20.output
-    java20AnnotationProcessor "org.openjdk.jmh:jmh-generator-annprocess:$versions.jmh"
-  }
-
-  compileJava20Java {
-    targetCompatibility = JavaVersion.VERSION_20
-    options.compilerArgs.addAll(["-processor", "org.openjdk.jmh.generators.BenchmarkProcessor"])
-  }
-
-  jar {
-    metaInf {
-      into 'versions/20'
-      from sourceSets.java20.output
-    }
-    manifest.attributes('Multi-Release': 'true')
-  }
-
-  // classes generated by JMH can use all sorts of forbidden APIs but we have no influence at all and cannot exclude these classes
-  disableTasks('forbiddenApisJava20')
-}
+// Add support for incubator modules on supported Java versions.
+run.jvmArgs += ['--add-modules=jdk.incubator.vector']
diff --git a/benchmarks/src/main/java/org/opensearch/common/round/RoundableSupplier.java b/benchmarks/src/main/java/org/opensearch/common/round/RoundableSupplier.java
@@ -13,7 +13,7 @@
 public class RoundableSupplier implements Supplier<Roundable> {
     private final Supplier<Roundable> delegate;
 
-    RoundableSupplier(String type, long[] values, int size) throws ClassNotFoundException {
+    RoundableSupplier(String type, long[] values, int size) {
         switch (type) {
             case "binary":
                 delegate = () -> new BinarySearcher(values, size);
@@ -22,7 +22,8 @@ public class RoundableSupplier implements Supplier<Roundable> {
                 delegate = () -> new BidirectionalLinearSearcher(values, size);
                 break;
             case "btree":
-                throw new ClassNotFoundException("BtreeSearcher is not supported below JDK 20");
+                delegate = () -> new BtreeSearcher(values, size);
+                break;
             default:
                 throw new IllegalArgumentException("invalid type: " + type);
         }

diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
@@ -74,8 +74,8 @@ processResources {
  *         Java version                                                      *
  *****************************************************************************/
 
-if (JavaVersion.current() < JavaVersion.VERSION_11) {
-  throw new GradleException('At least Java 11 is required to build opensearch gradle tools')
+if (JavaVersion.current() < JavaVersion.VERSION_21) {
+  throw new GradleException('At least Java 21 is required to build opensearch gradle tools')
 }
 
 sourceSets {
@@ -111,7 +111,7 @@ dependencies {
   api 'org.apache.rat:apache-rat:0.15'
   api "commons-io:commons-io:${props.getProperty('commonsio')}"
   api "net.java.dev.jna:jna:5.14.0"
-  api 'com.github.johnrengelman:shadow:8.1.1'
+  api 'com.gradleup.shadow:shadow-gradle-plugin:8.3.5'
   api 'org.jdom:jdom2:2.0.6.1'
   api "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${props.getProperty('kotlin')}"
   api 'de.thetaphi:forbiddenapis:3.8'
@@ -171,8 +171,8 @@ if (project != rootProject) {
 
   allprojects {
     java {
-        targetCompatibility = JavaVersion.VERSION_11
-        sourceCompatibility = JavaVersion.VERSION_11
+        targetCompatibility = JavaVersion.VERSION_21
+        sourceCompatibility = JavaVersion.VERSION_21
     }
   }
 

diff --git a/buildSrc/reaper/build.gradle b/buildSrc/reaper/build.gradle
@@ -12,8 +12,8 @@
 apply plugin: 'java'
 
 java {
-  targetCompatibility = JavaVersion.VERSION_11
-  sourceCompatibility = JavaVersion.VERSION_11
+  targetCompatibility = JavaVersion.VERSION_21
+  sourceCompatibility = JavaVersion.VERSION_21
 }
 
 jar {

diff --git a/...est/resources/org/opensearch/gradle/internal/fake_git/remote/.ci/java-versions.properties b/...est/resources/org/opensearch/gradle/internal/fake_git/remote/.ci/java-versions.properties
@@ -27,6 +27,6 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-OPENSEARCH_BUILD_JAVA=openjdk11
-OPENSEARCH_RUNTIME_JAVA=java11
+OPENSEARCH_BUILD_JAVA=openjdk21
+OPENSEARCH_RUNTIME_JAVA=java21
 GRADLE_TASK=build
diff --git a/buildSrc/src/main/resources/forbidden/opensearch-server-signatures.txt b/buildSrc/src/main/resources/forbidden/opensearch-server-signatures.txt
@@ -61,10 +61,6 @@ java.nio.channels.FileChannel#read(java.nio.ByteBuffer, long)
 @defaultMessage Use Lucene.parseLenient instead it strips off minor version
 org.apache.lucene.util.Version#parseLeniently(java.lang.String)
 
-@defaultMessage Spawns a new thread which is solely under lucenes control use ThreadPool#relativeTimeInMillis instead
-org.apache.lucene.search.TimeLimitingCollector#getGlobalTimerThread()
-org.apache.lucene.search.TimeLimitingCollector#getGlobalCounter()
-
 @defaultMessage Don't interrupt threads use FutureUtils#cancel(Future<T>) instead
 java.util.concurrent.Future#cancel(boolean)
 

diff --git a/buildSrc/src/main/resources/minimumCompilerVersion b/buildSrc/src/main/resources/minimumCompilerVersion
@@ -1 +1 @@
-17
+21
diff --git a/buildSrc/src/main/resources/minimumRuntimeVersion b/buildSrc/src/main/resources/minimumRuntimeVersion
@@ -1 +1 @@
-11
+21
diff --git a/buildSrc/src/testKit/thirdPartyAudit/build.gradle b/buildSrc/src/testKit/thirdPartyAudit/build.gradle
@@ -46,11 +46,11 @@ dependencies {
 }
 
 tasks.register("empty", ThirdPartyAuditTask) {
-  targetCompatibility = JavaVersion.VERSION_11
+  targetCompatibility = JavaVersion.VERSION_21
   signatureFile = file('third-party-audit-empty.txt')
 }
 
 tasks.register("absurd", ThirdPartyAuditTask) {
-  targetCompatibility = JavaVersion.VERSION_11
+  targetCompatibility = JavaVersion.VERSION_21
   signatureFile = file('third-party-audit-absurd.txt')
 }
diff --git a/...rest-high-level/src/test/java/org/opensearch/client/HighLevelRestClientCompressionIT.java b/...rest-high-level/src/test/java/org/opensearch/client/HighLevelRestClientCompressionIT.java
@@ -67,7 +67,7 @@ public void testCompressesResponseIfRequested() throws IOException {
         SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync, requestOptions);
 
         assertThat(searchResponse.status().getStatus(), equalTo(200));
-        assertEquals(1L, searchResponse.getHits().getTotalHits().value);
+        assertEquals(1L, searchResponse.getHits().getTotalHits().value());
         assertEquals(SAMPLE_DOCUMENT, searchResponse.getHits().getHits()[0].getSourceAsString());
     }
 

diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/ReindexIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/ReindexIT.java
@@ -236,7 +236,7 @@ public void testDeleteByQuery() throws Exception {
             assertEquals(0, bulkResponse.getSearchFailures().size());
             assertEquals(
                 2,
-                highLevelClient().search(new SearchRequest(sourceIndex), RequestOptions.DEFAULT).getHits().getTotalHits().value
+                highLevelClient().search(new SearchRequest(sourceIndex), RequestOptions.DEFAULT).getHits().getTotalHits().value()
             );
         }
         {

diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/RestHighLevelClientTests.java
@@ -237,7 +237,7 @@ public void testSearchScroll() throws IOException {
             RequestOptions.DEFAULT
         );
         assertEquals(mockSearchResponse.getScrollId(), searchResponse.getScrollId());
-        assertEquals(0, searchResponse.getHits().getTotalHits().value);
+        assertEquals(0, searchResponse.getHits().getTotalHits().value());
         assertEquals(5, searchResponse.getTotalShards());
         assertEquals(5, searchResponse.getSuccessfulShards());
         assertEquals(100, searchResponse.getTook().getMillis());