Merge branch 'main' into kp/integration-test-stalekeys

Signed-off-by: Sagar <[email protected]>

.github/CODEOWNERS

@@ -11,7 +11,7 @@
 #   3. Use the command palette to run the CODEOWNERS: Show owners of current file command, which will display all code owners for the current file.
 
 # Default ownership for all repo files
-* @abbashus @adnapibar @anasalkouz @andrross @Bukhtawar @CEHENKLE @dblock @dbwiddis @dreamer-89 @gbbafna @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @ryanbogan @sachinpkale @saratvemulapalli @setiah @shwetathareja @sohami @tlfeng @VachaShah
+* @anasalkouz @andrross @Bukhtawar @CEHENKLE @dblock @dbwiddis @dreamer-89 @gbbafna @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @tlfeng @VachaShah
 
 /modules/transport-netty4/ @peternied
 
@@ -24,4 +24,4 @@
 
 /.github/ @peternied
 
-/MAINTAINERS.md @abbashus @adnapibar @anasalkouz @andrross @Bukhtawar @CEHENKLE @dblock @dbwiddis @dreamer-89 @gbbafna @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @peternied @reta @Rishikesh1159 @ryanbogan @sachinpkale @saratvemulapalli @setiah @shwetathareja @sohami @tlfeng @VachaShah
+/MAINTAINERS.md @anasalkouz @andrross @Bukhtawar @CEHENKLE @dblock @dbwiddis @dreamer-89 @gbbafna @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @peternied @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @tlfeng @VachaShah

.github/workflows/changelog_verifier.yml

@@ -26,6 +26,19 @@ jobs:
           changeLogPath: 'CHANGELOG.md'
         continue-on-error: true
       - run: |
+          # The check was possibly skipped leading to success for both the jobs
+          if [[ ${{ steps.verify-changelog-3x.outcome }} == 'success' && ${{ steps.verify-changelog.outcome }} == 'success' ]]; then
+            exit 0
+          fi
+          
           if [[ ${{ steps.verify-changelog-3x.outcome }} == 'failure' && ${{ steps.verify-changelog.outcome }} == 'failure' ]]; then
+            echo "error: Please ensure a changelog entry exists in CHANGELOG.md or CHANGELOG-3.0.md"
+            exit 1
+          fi
+          
+          # Concatenates the labels and checks if the string contains "backport"
+          has_backport_label=${{ contains(join(github.event.pull_request.labels.*.name, ', '), 'backport')}}
+          if [[ ${{ steps.verify-changelog.outcome }} == 'success' && $has_backport_label == false ]]; then
+            echo "error: Please make sure that the PR has a backport label associated with it when making an entry to the CHANGELOG.md file"
             exit 1
           fi

.github/workflows/wrapper.yml

@@ -8,4 +8,4 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: gradle/wrapper-validation-action@v2
+      - uses: gradle/wrapper-validation-action@v3

CHANGELOG.md

@@ -19,7 +19,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Remote Store] Make translog transfer timeout configurable ([#12704](https://github.com/opensearch-project/OpenSearch/pull/12704))
 - Reject Resize index requests (i.e, split, shrink and clone), While DocRep to SegRep migration is in progress.([#12686](https://github.com/opensearch-project/OpenSearch/pull/12686))
 - Add support for more than one protocol for transport ([#12967](https://github.com/opensearch-project/OpenSearch/pull/12967))
+- [Tiered Caching] Add dimension-based stats to ICache implementations. ([#12531](https://github.com/opensearch-project/OpenSearch/pull/12531))
 - Add changes for overriding remote store and replication settings during snapshot restore. ([#11868](https://github.com/opensearch-project/OpenSearch/pull/11868))
+- Add an individual setting of rate limiter for segment replication ([#12959](https://github.com/opensearch-project/OpenSearch/pull/12959))
+- [Streaming Indexing] Ensure support of the new transport by security plugin ([#13174](https://github.com/opensearch-project/OpenSearch/pull/13174))
 
 ### Dependencies
 - Bump `org.apache.commons:commons-configuration2` from 2.10.0 to 2.10.1 ([#12896](https://github.com/opensearch-project/OpenSearch/pull/12896))
@@ -31,12 +34,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `org.apache.commons:commonscodec` from 1.15 to 1.16.1 ([#12627](https://github.com/opensearch-project/OpenSearch/pull/12627))
 - Bump `org.apache.commons:commonslang` from 3.13.0 to 3.14.0 ([#12627](https://github.com/opensearch-project/OpenSearch/pull/12627))
 - Bump Apache Tika from 2.6.0 to 2.9.2 ([#12627](https://github.com/opensearch-project/OpenSearch/pull/12627))
-- Bump `com.gradle.enterprise` from 3.16.2 to 3.17 ([#13116](https://github.com/opensearch-project/OpenSearch/pull/13116))
+- Bump `com.gradle.enterprise` from 3.16.2 to 3.17.1 ([#13116](https://github.com/opensearch-project/OpenSearch/pull/13116), [#13191](https://github.com/opensearch-project/OpenSearch/pull/13191))
+- Bump `gradle/wrapper-validation-action` from 2 to 3 ([#13192](https://github.com/opensearch-project/OpenSearch/pull/13192))
 
 ### Changed
 - [BWC and API enforcement] Enforcing the presence of API annotations at build time ([#12872](https://github.com/opensearch-project/OpenSearch/pull/12872))
 - Improve built-in secure transports support ([#12907](https://github.com/opensearch-project/OpenSearch/pull/12907))
 - Update links to documentation in rest-api-spec ([#13043](https://github.com/opensearch-project/OpenSearch/pull/13043))
+- Refactoring globMatch using simpleMatchWithNormalizedStrings from Regex ([#13104](https://github.com/opensearch-project/OpenSearch/pull/13104))
 
 ### Deprecated
 

MAINTAINERS.md

@@ -5,8 +5,7 @@ This document contains a list of maintainers in this repo. See [opensearch-proje
 ## Current Maintainers
 
 | Maintainer               | GitHub ID                                               | Affiliation |
-|--------------------------| ------------------------------------------------------- | ----------- |
-| Abbas Hussain            | [abbashus](https://github.com/abbashus)                 | Meta        |
+| ------------------------ | ------------------------------------------------------- | ----------- |
 | Anas Alkouz              | [anasalkouz](https://github.com/anasalkouz)             | Amazon      |
 | Andrew Ross              | [andrross](https://github.com/andrross)                 | Amazon      |
 | Andriy Redko             | [reta](https://github.com/reta)                         | Aiven       |
@@ -15,16 +14,13 @@ This document contains a list of maintainers in this repo. See [opensearch-proje
 | Dan Widdis               | [dbwiddis](https://github.com/dbwiddis)                 | Amazon      |
 | Daniel "dB." Doubrovkine | [dblock](https://github.com/dblock)                     | Amazon      |
 | Gaurav Bafna             | [gbbafna](https://github.com/gbbafna)                   | Amazon      |
-| Himanshu Setia           | [setiah](https://github.com/setiah)                     | Amazon      |
 | Kunal Kotwani            | [kotwanikunal](https://github.com/kotwanikunal)         | Amazon      |
 | Marc Handalian           | [mch2](https://github.com/mch2)                         | Amazon      |
 | Michael Froh             | [msfroh](https://github.com/msfroh)                     | Amazon      |
 | Nick Knize               | [nknize](https://github.com/nknize)                     | Amazon      |
 | Owais Kazi               | [owaiskazi19](https://github.com/owaiskazi19)           | Amazon      |
 | Peter Nied               | [peternied](https://github.com/peternied)               | Amazon      |
-| Rabi Panda               | [adnapibar](https://github.com/adnapibar)               | Independent |
 | Rishikesh Pasham         | [Rishikesh1159](https://github.com/Rishikesh1159)       | Amazon      |
-| Ryan Bogan               | [ryanbogan](https://github.com/ryanbogan)               | Amazon      |
 | Sachin Kale              | [sachinpkale](https://github.com/sachinpkale)           | Amazon      |
 | Sarat Vemulapalli        | [saratvemulapalli](https://github.com/saratvemulapalli) | Amazon      |
 | Shweta Thareja           | [shwetathareja](https://github.com/shwetathareja)       | Amazon      |
@@ -35,8 +31,12 @@ This document contains a list of maintainers in this repo. See [opensearch-proje
 
 ## Emeritus
 
-| Maintainer              | GitHub ID                                   | Affiliation |
-|-------------------------|---------------------------------------------|-------------|
-| Megha Sai Kavikondala   | [meghasaik](https://github.com/meghasaik)   | Amazon      |
-| Xue Zhou                | [xuezhou25](https://github.com/xuezhou25)   | Amazon      |
-| Kartik Ganesh           | [kartg](https://github.com/kartg)           | Amazon      |
+| Maintainer            | GitHub ID                                 | Affiliation |
+| --------------------- | ----------------------------------------- | ----------- |
+| Megha Sai Kavikondala | [meghasaik](https://github.com/meghasaik) | Amazon      |
+| Xue Zhou              | [xuezhou25](https://github.com/xuezhou25) | Amazon      |
+| Kartik Ganesh         | [kartg](https://github.com/kartg)         | Amazon      |
+| Abbas Hussain         | [abbashus](https://github.com/abbashus)   | Meta        |
+| Himanshu Setia        | [setiah](https://github.com/setiah)       | Amazon      |
+| Ryan Bogan            | [ryanbogan](https://github.com/ryanbogan) | Amazon      |
+| Rabi Panda            | [adnapibar](https://github.com/adnapibar) | Independent |

buildSrc/version.properties

@@ -1,5 +1,5 @@
 opensearch        = 3.0.0
-lucene            = 9.11.0-snapshot-8a555eb
+lucene            = 9.11.0-snapshot-fb97840
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 21.0.2+13

doc-tools/missing-doclet/src/main/java/org/opensearch/missingdoclet/MissingDoclet.java

@@ -332,13 +332,20 @@ private void checkComment(Element element) {
 
     // Ignore classes annotated with @Generated and all enclosed elements in them.
     private boolean isGenerated(Element element) {
-        return element
+        final boolean isGenerated = element
             .getAnnotationMirrors()
             .stream()
             .anyMatch(m -> m
                 .getAnnotationType()
                 .toString() /* ClassSymbol.toString() returns class name */
                 .equalsIgnoreCase("javax.annotation.Generated"));
+
+        if (!isGenerated && element.getEnclosingElement() != null) {
+            // check if enclosing element is generated
+            return isGenerated(element.getEnclosingElement());
+        } 
+
+        return isGenerated;
     }
 
     private boolean hasInheritedJavadocs(Element element) {

libs/common/src/main/java/org/opensearch/common/Glob.java

@@ -52,34 +52,35 @@ public static boolean globMatch(String pattern, String str) {
         if (pattern == null || str == null) {
             return false;
         }
-        int firstIndex = pattern.indexOf('*');
-        if (firstIndex == -1) {
-            return pattern.equals(str);
-        }
-        if (firstIndex == 0) {
-            if (pattern.length() == 1) {
-                return true;
-            }
-            int nextIndex = pattern.indexOf('*', firstIndex + 1);
-            if (nextIndex == -1) {
-                return str.endsWith(pattern.substring(1));
-            } else if (nextIndex == 1) {
-                // Double wildcard "**" - skipping the first "*"
-                return globMatch(pattern.substring(1), str);
+        int sIdx = 0, pIdx = 0, match = 0, wildcardIdx = -1;
+        while (sIdx < str.length()) {
+            // both chars matching, incrementing both pointers
+            if (pIdx < pattern.length() && str.charAt(sIdx) == pattern.charAt(pIdx)) {
+                sIdx++;
+                pIdx++;
+            } else if (pIdx < pattern.length() && pattern.charAt(pIdx) == '*') {
+                // wildcard found, only incrementing pattern pointer
+                wildcardIdx = pIdx;
+                match = sIdx;
+                pIdx++;
+            } else if (wildcardIdx != -1) {
+                // last pattern pointer was a wildcard, incrementing string pointer
+                pIdx = wildcardIdx + 1;
+                match++;
+                sIdx = match;
+            } else {
+                // current pattern pointer is not a wildcard, last pattern pointer was also not a wildcard
+                // characters do not match
+                return false;
             }
-            String part = pattern.substring(1, nextIndex);
-            int partIndex = str.indexOf(part);
-            while (partIndex != -1) {
-                if (globMatch(pattern.substring(nextIndex), str.substring(partIndex + part.length()))) {
-                    return true;
-                }
-                partIndex = str.indexOf(part, partIndex + 1);
-            }
-            return false;
         }
-        return (str.length() >= firstIndex
-            && pattern.substring(0, firstIndex).equals(str.substring(0, firstIndex))
-            && globMatch(pattern.substring(firstIndex), str.substring(firstIndex)));
+
+        // check for remaining characters in pattern
+        while (pIdx < pattern.length() && pattern.charAt(pIdx) == '*') {
+            pIdx++;
+        }
+
+        return pIdx == pattern.length();
     }
 
 }

libs/core/licenses/lucene-core-9.11.0-snapshot-fb97840.jar.sha1

@@ -0,0 +1 @@
+53a828e3e88f55c83979cd3df0704617cc9edb9a

modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCache.java

@@ -12,11 +12,13 @@
 import org.opensearch.common.annotation.ExperimentalApi;
 import org.opensearch.common.cache.CacheType;
 import org.opensearch.common.cache.ICache;
+import org.opensearch.common.cache.ICacheKey;
 import org.opensearch.common.cache.LoadAwareCacheLoader;
 import org.opensearch.common.cache.RemovalListener;
 import org.opensearch.common.cache.RemovalNotification;
 import org.opensearch.common.cache.RemovalReason;
 import org.opensearch.common.cache.policy.CachedQueryResult;
+import org.opensearch.common.cache.stats.ImmutableCacheStatsHolder;
 import org.opensearch.common.cache.store.config.CacheConfig;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
@@ -54,7 +56,11 @@ public class TieredSpilloverCache<K, V> implements ICache<K, V> {
 
     private final ICache<K, V> diskCache;
     private final ICache<K, V> onHeapCache;
-    private final RemovalListener<K, V> removalListener;
+
+    // The listener for removals from the spillover cache as a whole
+    // TODO: In TSC stats PR, each tier will have its own separate removal listener.
+    private final RemovalListener<ICacheKey<K>, V> removalListener;
+    private final List<String> dimensionNames;
     ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
     ReleasableLock readLock = new ReleasableLock(readWriteLock.readLock());
     ReleasableLock writeLock = new ReleasableLock(readWriteLock.writeLock());
@@ -70,9 +76,9 @@ public class TieredSpilloverCache<K, V> implements ICache<K, V> {
         this.removalListener = Objects.requireNonNull(builder.removalListener, "Removal listener can't be null");
 
         this.onHeapCache = builder.onHeapCacheFactory.create(
-            new CacheConfig.Builder<K, V>().setRemovalListener(new RemovalListener<K, V>() {
+            new CacheConfig.Builder<K, V>().setRemovalListener(new RemovalListener<ICacheKey<K>, V>() {
                 @Override
-                public void onRemoval(RemovalNotification<K, V> notification) {
+                public void onRemoval(RemovalNotification<ICacheKey<K>, V> notification) {
                     try (ReleasableLock ignore = writeLock.acquire()) {
                         if (SPILLOVER_REMOVAL_REASONS.contains(notification.getRemovalReason())
                             && evaluatePolicies(notification.getValue())) {
@@ -87,6 +93,7 @@ && evaluatePolicies(notification.getValue())) {
                 .setValueType(builder.cacheConfig.getValueType())
                 .setSettings(builder.cacheConfig.getSettings())
                 .setWeigher(builder.cacheConfig.getWeigher())
+                .setDimensionNames(builder.cacheConfig.getDimensionNames())
                 .setMaxSizeInBytes(builder.cacheConfig.getMaxSizeInBytes())
                 .setExpireAfterAccess(builder.cacheConfig.getExpireAfterAccess())
                 .setClusterSettings(builder.cacheConfig.getClusterSettings())
@@ -97,7 +104,7 @@ && evaluatePolicies(notification.getValue())) {
         );
         this.diskCache = builder.diskCacheFactory.create(builder.cacheConfig, builder.cacheType, builder.cacheFactories);
         this.cacheList = Arrays.asList(onHeapCache, diskCache);
-
+        this.dimensionNames = builder.cacheConfig.getDimensionNames();
         this.policies = builder.policies; // Will never be null; builder initializes it to an empty list
     }
 
@@ -112,19 +119,19 @@ ICache<K, V> getDiskCache() {
     }
 
     @Override
-    public V get(K key) {
+    public V get(ICacheKey<K> key) {
         return getValueFromTieredCache().apply(key);
     }
 
     @Override
-    public void put(K key, V value) {
+    public void put(ICacheKey<K> key, V value) {
         try (ReleasableLock ignore = writeLock.acquire()) {
             onHeapCache.put(key, value);
         }
     }
 
     @Override
-    public V computeIfAbsent(K key, LoadAwareCacheLoader<K, V> loader) throws Exception {
+    public V computeIfAbsent(ICacheKey<K> key, LoadAwareCacheLoader<ICacheKey<K>, V> loader) throws Exception {
 
         V cacheValue = getValueFromTieredCache().apply(key);
         if (cacheValue == null) {
@@ -141,7 +148,7 @@ public V computeIfAbsent(K key, LoadAwareCacheLoader<K, V> loader) throws Except
     }
 
     @Override
-    public void invalidate(K key) {
+    public void invalidate(ICacheKey<K> key) {
         // We are trying to invalidate the key from all caches though it would be present in only of them.
         // Doing this as we don't know where it is located. We could do a get from both and check that, but what will
         // also trigger a hit/miss listener event, so ignoring it for now.
@@ -167,9 +174,9 @@ public void invalidateAll() {
      */
     @SuppressWarnings({ "unchecked" })
     @Override
-    public Iterable<K> keys() {
-        Iterable<K>[] iterables = (Iterable<K>[]) new Iterable<?>[] { onHeapCache.keys(), diskCache.keys() };
-        return new ConcatenatedIterables<K>(iterables);
+    public Iterable<ICacheKey<K>> keys() {
+        Iterable<ICacheKey<K>>[] iterables = (Iterable<ICacheKey<K>>[]) new Iterable<?>[] { onHeapCache.keys(), diskCache.keys() };
+        return new ConcatenatedIterables<ICacheKey<K>>(iterables);
     }
 
     @Override
@@ -197,7 +204,12 @@ public void close() throws IOException {
         }
     }
 
-    private Function<K, V> getValueFromTieredCache() {
+    @Override
+    public ImmutableCacheStatsHolder stats() {
+        return null; // TODO: in TSC stats PR
+    }
+
+    private Function<ICacheKey<K>, V> getValueFromTieredCache() {
         return key -> {
             try (ReleasableLock ignore = readLock.acquire()) {
                 for (ICache<K, V> cache : cacheList) {
@@ -354,7 +366,7 @@ public String getCacheName() {
     public static class Builder<K, V> {
         private ICache.Factory onHeapCacheFactory;
         private ICache.Factory diskCacheFactory;
-        private RemovalListener<K, V> removalListener;
+        private RemovalListener<ICacheKey<K>, V> removalListener;
         private CacheConfig<K, V> cacheConfig;
         private CacheType cacheType;
         private Map<String, ICache.Factory> cacheFactories;
@@ -390,7 +402,7 @@ public Builder<K, V> setDiskCacheFactory(ICache.Factory diskCacheFactory) {
          * @param removalListener Removal listener
          * @return builder
          */
-        public Builder<K, V> setRemovalListener(RemovalListener<K, V> removalListener) {
+        public Builder<K, V> setRemovalListener(RemovalListener<ICacheKey<K>, V> removalListener) {
             this.removalListener = removalListener;
             return this;
         }