Merge branch 'main' into feature/synthtic-source-recovery-default

salvatore-campagna · Jan 8, 2025 · f21e8d4 · f21e8d4
2 parents 1f4bfed + 12eb1cf
commit f21e8d4
Show file tree

Hide file tree

Showing 78 changed files with 7,390 additions and 102 deletions.
diff --git a/docs/changelog/118652.yaml b/docs/changelog/118652.yaml
@@ -0,0 +1,5 @@
+pr: 118652
+summary: Add Jina AI API to do inference for Embedding and Rerank models
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/119067.yaml b/docs/changelog/119067.yaml
@@ -0,0 +1,5 @@
+pr: 119067
+summary: Metrics for indexing failures due to version conflicts
+area: CRUD
+type: feature
+issues: []
diff --git a/docs/changelog/119504.yaml b/docs/changelog/119504.yaml
@@ -0,0 +1,5 @@
+pr: 119504
+summary: Optimized index sorting for OTel logs
+area: Data streams
+type: enhancement
+issues: []
diff --git a/docs/reference/cat/nodes.asciidoc b/docs/reference/cat/nodes.asciidoc
@@ -239,6 +239,9 @@ Number of indexing operations, such as `1`.
 `indexing.index_failed`, `iif`, `indexingIndexFailed`::
 Number of failed indexing operations, such as `0`.
 
+`indexing.index_failed_due_to_version_conflict`, `iifvc`, `indexingIndexFailedDueToVersionConflict`::
+Number of failed indexing operations due to version conflict, such as `0`.
+
 `merges.current`, `mc`, `mergesCurrent`::
 Number of current merge operations, such as `0`.
 

diff --git a/docs/reference/cat/shards.asciidoc b/docs/reference/cat/shards.asciidoc
@@ -162,6 +162,9 @@ Number of indexing operations, such as `1`.
 `indexing.index_failed`, `iif`, `indexingIndexFailed`::
 Number of failed indexing operations, such as `0`.
 
+`indexing.index_failed_due_to_version_conflict`, `iifvc`, `indexingIndexFailedDueToVersionConflict`::
+Number of failed indexing operations due to version conflict, such as `0`.
+
 `merges.current`, `mc`, `mergesCurrent`::
 Number of current merge operations, such as `0`.
 

diff --git a/.../src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamAutoshardingIT.java b/.../src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamAutoshardingIT.java
@@ -493,7 +493,7 @@ private static ShardStats getShardStats(IndexMetadata indexMeta, int shardIndex,
         CommonStats stats = new CommonStats();
         stats.docs = new DocsStats(100, 0, randomByteSizeValue().getBytes());
         stats.store = new StoreStats();
-        stats.indexing = new IndexingStats(new IndexingStats.Stats(1, 1, 1, 1, 1, 1, 1, 1, false, 1, targetWriteLoad, 1));
+        stats.indexing = new IndexingStats(new IndexingStats.Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, false, 1, targetWriteLoad, 1));
         return new ShardStats(shardRouting, new ShardPath(false, path, path, shardId), stats, null, null, null, false, 0);
     }
 

diff --git a/muted-tests.yml b/muted-tests.yml
@@ -236,38 +236,14 @@ tests:
 - class: org.elasticsearch.smoketest.MlWithSecurityIT
   method: test {yaml=ml/sparse_vector_search/Test sparse_vector search with query vector and pruning config}
   issue: https://github.com/elastic/elasticsearch/issues/119548
-- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT
-  method: testSearchableSnapshotUpgrade {p0=[9.0.0, 8.18.0, 8.18.0]}
-  issue: https://github.com/elastic/elasticsearch/issues/119549
-- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT
-  method: testMountSearchableSnapshot {p0=[9.0.0, 8.18.0, 8.18.0]}
-  issue: https://github.com/elastic/elasticsearch/issues/119550
-- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT
-  method: testMountSearchableSnapshot {p0=[9.0.0, 9.0.0, 8.18.0]}
-  issue: https://github.com/elastic/elasticsearch/issues/119551
 - class: org.elasticsearch.index.engine.LuceneSyntheticSourceChangesSnapshotTests
   method: testSkipNonRootOfNestedDocuments
   issue: https://github.com/elastic/elasticsearch/issues/119553
-- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT
-  method: testSearchableSnapshotUpgrade {p0=[9.0.0, 9.0.0, 8.18.0]}
-  issue: https://github.com/elastic/elasticsearch/issues/119560
-- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT
-  method: testMountSearchableSnapshot {p0=[9.0.0, 9.0.0, 9.0.0]}
-  issue: https://github.com/elastic/elasticsearch/issues/119561
-- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT
-  method: testSearchableSnapshotUpgrade {p0=[9.0.0, 9.0.0, 9.0.0]}
-  issue: https://github.com/elastic/elasticsearch/issues/119562
 - class: org.elasticsearch.xpack.ml.integration.ForecastIT
   method: testOverflowToDisk
   issue: https://github.com/elastic/elasticsearch/issues/117740
 - class: org.elasticsearch.xpack.security.authc.ldap.MultiGroupMappingIT
   issue: https://github.com/elastic/elasticsearch/issues/119599
-- class: org.elasticsearch.lucene.FullClusterRestartSearchableSnapshotIndexCompatibilityIT
-  method: testSearchableSnapshotUpgrade {p0=8.18.0}
-  issue: https://github.com/elastic/elasticsearch/issues/119631
-- class: org.elasticsearch.lucene.FullClusterRestartSearchableSnapshotIndexCompatibilityIT
-  method: testSearchableSnapshotUpgrade {p0=9.0.0}
-  issue: https://github.com/elastic/elasticsearch/issues/119632
 - class: org.elasticsearch.search.profile.dfs.DfsProfilerIT
   method: testProfileDfs
   issue: https://github.com/elastic/elasticsearch/issues/119711

diff --git a/qa/lucene-index-compatibility/build.gradle b/qa/lucene-index-compatibility/build.gradle
@@ -14,12 +14,13 @@ buildParams.bwcVersions.withLatestReadOnlyIndexCompatible { bwcVersion ->
   tasks.named("javaRestTest").configure {
     systemProperty("tests.minimum.index.compatible", bwcVersion)
     usesBwcDistribution(bwcVersion)
-    enabled = true
+
+    // Tests rely on unreleased code in 8.18 branch
+    enabled = buildParams.isSnapshotBuild()
   }
 }
 
 tasks.withType(Test).configureEach {
   // CI doesn't like it when there's multiple clusters running at once
   maxParallelForks = 1
 }
-
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.shards/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.shards/10_basic.yml
@@ -45,6 +45,7 @@
              indexing.index_time              .+   \n
              indexing.index_total             .+   \n
              indexing.index_failed            .+   \n
+             indexing.index_failed_due_to_version_conflict .+   \n
              merges.current                   .+   \n
              merges.current_docs              .+   \n
              merges.current_size              .+   \n

diff --git a/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/IndicesMetricsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/IndicesMetricsIT.java
@@ -30,6 +30,7 @@
 
 import java.io.IOException;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -76,6 +77,7 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
     static final String STANDARD_INDEXING_COUNT = "es.indices.standard.indexing.total";
     static final String STANDARD_INDEXING_TIME = "es.indices.standard.indexing.time";
     static final String STANDARD_INDEXING_FAILURE = "es.indices.standard.indexing.failure.total";
+    static final String STANDARD_INDEXING_FAILURE_DUE_TO_VERSION_CONFLICT = "es.indices.standard.indexing.failure.version_conflict.total";
 
     static final String TIME_SERIES_INDEX_COUNT = "es.indices.time_series.total";
     static final String TIME_SERIES_BYTES_SIZE = "es.indices.time_series.size";
@@ -89,6 +91,8 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
     static final String TIME_SERIES_INDEXING_COUNT = "es.indices.time_series.indexing.total";
     static final String TIME_SERIES_INDEXING_TIME = "es.indices.time_series.indexing.time";
     static final String TIME_SERIES_INDEXING_FAILURE = "es.indices.time_series.indexing.failure.total";
+    static final String TIME_SERIES_INDEXING_FAILURE_DUE_TO_VERSION_CONFLICT =
+        "es.indices.time_series.indexing.failure.version_conflict.total";
 
     static final String LOGSDB_INDEX_COUNT = "es.indices.logsdb.total";
     static final String LOGSDB_BYTES_SIZE = "es.indices.logsdb.size";
@@ -102,6 +106,7 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
     static final String LOGSDB_INDEXING_COUNT = "es.indices.logsdb.indexing.total";
     static final String LOGSDB_INDEXING_TIME = "es.indices.logsdb.indexing.time";
     static final String LOGSDB_INDEXING_FAILURE = "es.indices.logsdb.indexing.failure.total";
+    static final String LOGSDB_INDEXING_FAILURE_DUE_TO_VERSION_CONFLICT = "es.indices.logsdb.indexing.failure.version_conflict.total";
 
     public void testIndicesMetrics() {
         String indexNode = internalCluster().startNode();
@@ -132,7 +137,9 @@ public void testIndicesMetrics() {
                 STANDARD_INDEXING_TIME,
                 greaterThanOrEqualTo(0L),
                 STANDARD_INDEXING_FAILURE,
-                equalTo(indexing1.getIndexFailedCount() - indexing0.getIndexCount())
+                equalTo(indexing1.getIndexFailedCount() - indexing0.getIndexFailedCount()),
+                STANDARD_INDEXING_FAILURE_DUE_TO_VERSION_CONFLICT,
+                equalTo(indexing1.getIndexFailedDueToVersionConflictCount() - indexing0.getIndexFailedDueToVersionConflictCount())
             )
         );
 
@@ -155,7 +162,9 @@ public void testIndicesMetrics() {
                 TIME_SERIES_INDEXING_TIME,
                 greaterThanOrEqualTo(0L),
                 TIME_SERIES_INDEXING_FAILURE,
-                equalTo(indexing2.getIndexFailedCount() - indexing1.getIndexFailedCount())
+                equalTo(indexing1.getIndexFailedCount() - indexing0.getIndexFailedCount()),
+                TIME_SERIES_INDEXING_FAILURE_DUE_TO_VERSION_CONFLICT,
+                equalTo(indexing1.getIndexFailedDueToVersionConflictCount() - indexing0.getIndexFailedDueToVersionConflictCount())
             )
         );
 
@@ -177,36 +186,50 @@ public void testIndicesMetrics() {
                 LOGSDB_INDEXING_TIME,
                 greaterThanOrEqualTo(0L),
                 LOGSDB_INDEXING_FAILURE,
-                equalTo(indexing3.getIndexFailedCount() - indexing2.getIndexFailedCount())
+                equalTo(indexing3.getIndexFailedCount() - indexing2.getIndexFailedCount()),
+                LOGSDB_INDEXING_FAILURE_DUE_TO_VERSION_CONFLICT,
+                equalTo(indexing3.getIndexFailedDueToVersionConflictCount() - indexing2.getIndexFailedDueToVersionConflictCount())
             )
         );
         // already collected indexing stats
-        collectThenAssertMetrics(
-            telemetry,
-            4,
+        Map<String, Matcher<Long>> zeroMatchers = new HashMap<>();
+        zeroMatchers.putAll(
             Map.of(
                 STANDARD_INDEXING_COUNT,
                 equalTo(0L),
                 STANDARD_INDEXING_TIME,
                 equalTo(0L),
                 STANDARD_INDEXING_FAILURE,
                 equalTo(0L),
-
+                STANDARD_INDEXING_FAILURE_DUE_TO_VERSION_CONFLICT,
+                equalTo(0L)
+            )
+        );
+        zeroMatchers.putAll(
+            Map.of(
                 TIME_SERIES_INDEXING_COUNT,
                 equalTo(0L),
                 TIME_SERIES_INDEXING_TIME,
                 equalTo(0L),
                 TIME_SERIES_INDEXING_FAILURE,
                 equalTo(0L),
-
+                TIME_SERIES_INDEXING_FAILURE_DUE_TO_VERSION_CONFLICT,
+                equalTo(0L)
+            )
+        );
+        zeroMatchers.putAll(
+            Map.of(
                 LOGSDB_INDEXING_COUNT,
                 equalTo(0L),
                 LOGSDB_INDEXING_TIME,
                 equalTo(0L),
                 LOGSDB_INDEXING_FAILURE,
+                equalTo(0L),
+                LOGSDB_INDEXING_FAILURE_DUE_TO_VERSION_CONFLICT,
                 equalTo(0L)
             )
         );
+        collectThenAssertMetrics(telemetry, 4, zeroMatchers);
         String searchNode = internalCluster().startDataOnlyNode();
         indicesService = internalCluster().getInstance(IndicesService.class, searchNode);
         telemetry = internalCluster().getInstance(PluginsService.class, searchNode)