Merge remote-tracking branch 'upstream/2.x' into backport-doctest-fix…

…-to-2.x

async-query/build.gradle

@@ -28,7 +28,7 @@ dependencies {
     implementation group: 'org.json', name: 'json', version: '20231013'
     api group: 'com.amazonaws', name: 'aws-java-sdk-emr', version: "${aws_java_sdk_version}"
     api group: 'com.amazonaws', name: 'aws-java-sdk-emrserverless', version: "${aws_java_sdk_version}"
-    implementation group: 'commons-io', name: 'commons-io', version: '2.8.0'
+    implementation group: 'commons-io', name: 'commons-io', version: '2.14.0'
 
     testImplementation(platform("org.junit:junit-bom:5.9.3"))
 

async-query/src/main/java/org/opensearch/sql/spark/config/OpenSearchAsyncQuerySchedulerConfigComposer.java

@@ -9,6 +9,7 @@
 import static org.opensearch.sql.spark.data.constants.SparkConstants.FLINT_JOB_EXTERNAL_SCHEDULER_INTERVAL;
 
 import lombok.RequiredArgsConstructor;
+import org.opensearch.core.common.Strings;
 import org.opensearch.sql.common.setting.Settings;
 import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext;
 import org.opensearch.sql.spark.dispatcher.model.DispatchQueryRequest;
@@ -30,7 +31,11 @@ public void compose(
         settings.getSettingValue(Settings.Key.ASYNC_QUERY_EXTERNAL_SCHEDULER_INTERVAL);
     sparkSubmitParameters.setConfigItem(
         FLINT_JOB_EXTERNAL_SCHEDULER_ENABLED, String.valueOf(externalSchedulerEnabled));
-    sparkSubmitParameters.setConfigItem(
-        FLINT_JOB_EXTERNAL_SCHEDULER_INTERVAL, externalSchedulerInterval);
+    if (!Strings.isNullOrEmpty(externalSchedulerInterval)) {
+      externalSchedulerInterval =
+          "\"" + externalSchedulerInterval + "\""; // Wrap the value with double quotes
+      sparkSubmitParameters.setConfigItem(
+          FLINT_JOB_EXTERNAL_SCHEDULER_INTERVAL, externalSchedulerInterval);
+    }
   }
 }

async-query/src/test/java/org/opensearch/sql/spark/config/OpenSearchAsyncQuerySchedulerConfigComposerTest.java

@@ -1,5 +1,6 @@
 package org.opensearch.sql.spark.config;
 
+import static org.junit.Assert.assertNull;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
@@ -40,7 +41,7 @@ public void testCompose() {
     verify(sparkSubmitParameters)
         .setConfigItem("spark.flint.job.externalScheduler.enabled", "true");
     verify(sparkSubmitParameters)
-        .setConfigItem("spark.flint.job.externalScheduler.interval", "10 minutes");
+        .setConfigItem("spark.flint.job.externalScheduler.interval", "\"10 minutes\"");
   }
 
   @Test
@@ -63,6 +64,6 @@ public void testComposeWithMissingInterval() {
 
     composer.compose(sparkSubmitParameters, dispatchQueryRequest, context);
 
-    verify(sparkSubmitParameters).setConfigItem("spark.flint.job.externalScheduler.interval", "");
+    assertNull(sparkSubmitParameters.getConfigItem("spark.flint.job.externalScheduler.interval"));
   }
 }

common/src/main/java/org/opensearch/sql/common/grok/GrokCompiler.java

@@ -29,7 +29,7 @@
 public class GrokCompiler implements Serializable {
 
   // We don't want \n and commented line
-  private static final Pattern patternLinePattern = Pattern.compile("^([A-z0-9_]+)\\s+(.*)$");
+  private static final Pattern patternLinePattern = Pattern.compile("^([a-zA-Z0-9_]+)\\s+(.*)$");
 
   /** {@code Grok} patterns definitions. */
   private final Map<String, String> grokPatternDefinitions = new HashMap<>();

common/src/main/java/org/opensearch/sql/common/grok/GrokUtils.java

@@ -24,8 +24,8 @@ public class GrokUtils {
       Pattern.compile(
           "%\\{"
               + "(?<name>"
-              + "(?<pattern>[A-z0-9]+)"
-              + "(?::(?<subname>[A-z0-9_:;,\\-\\/\\s\\.']+))?"
+              + "(?<pattern>[a-zA-Z0-9_]+)"
+              + "(?::(?<subname>[a-zA-Z0-9_:;,\\-\\/\\s\\.']+))?"
               + ")"
               + "(?:=(?<definition>"
               + "(?:"

datasources/build.gradle

@@ -21,7 +21,7 @@ dependencies {
     implementation group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}"
     implementation group: 'org.opensearch', name: 'opensearch-x-content', version: "${opensearch_version}"
     implementation group: 'org.opensearch', name: 'common-utils', version: "${opensearch_build}"
-    implementation group: 'commons-io', name: 'commons-io', version: '2.8.0'
+    implementation group: 'commons-io', name: 'commons-io', version: '2.14.0'
     // FIXME. upgrade aws-encryption-sdk-java once the bouncycastle dependency update to 1.78.
     implementation ('com.amazonaws:aws-encryption-sdk-java:2.4.1') {
         exclude group: 'org.bouncycastle', module: 'bcprov-ext-jdk18on'

integ-test/src/test/java/org/opensearch/sql/jdbc/CursorIT.java

@@ -22,6 +22,7 @@
 import java.sql.ResultSet;
 import java.sql.Statement;
 import java.util.List;
+import java.util.Map;
 import javax.annotation.Nullable;
 import lombok.SneakyThrows;
 import org.json.JSONObject;
@@ -116,6 +117,8 @@ public void select_all_no_cursor() {
 
       var restResponse = executeRestQuery(query, null);
       assertEquals(rows, restResponse.getInt("total"));
+      var restPrettyResponse = executeRestQuery(query, null, Map.of("pretty", "true"));
+      assertEquals(rows, restPrettyResponse.getInt("total"));
     }
   }
 
@@ -134,6 +137,8 @@ public void select_count_all_no_cursor() {
 
       var restResponse = executeRestQuery(query, null);
       assertEquals(rows, restResponse.getInt("total"));
+      var restPrettyResponse = executeRestQuery(query, null, Map.of("pretty", "true"));
+      assertEquals(rows, restPrettyResponse.getInt("total"));
     }
   }
 
@@ -152,6 +157,8 @@ public void select_all_small_table_big_cursor() {
 
       var restResponse = executeRestQuery(query, null);
       assertEquals(rows, restResponse.getInt("total"));
+      var restPrettyResponse = executeRestQuery(query, null, Map.of("pretty", "true"));
+      assertEquals(rows, restPrettyResponse.getInt("total"));
     }
   }
 
@@ -170,6 +177,8 @@ public void select_all_small_table_small_cursor() {
 
       var restResponse = executeRestQuery(query, null);
       assertEquals(rows, restResponse.getInt("total"));
+      var restPrettyResponse = executeRestQuery(query, null, Map.of("pretty", "true"));
+      assertEquals(rows, restPrettyResponse.getInt("total"));
     }
   }
 
@@ -188,6 +197,8 @@ public void select_all_big_table_small_cursor() {
 
       var restResponse = executeRestQuery(query, null);
       assertEquals(rows, restResponse.getInt("total"));
+      var restPrettyResponse = executeRestQuery(query, null, Map.of("pretty", "true"));
+      assertEquals(rows, restPrettyResponse.getInt("total"));
     }
   }
 
@@ -206,6 +217,8 @@ public void select_all_big_table_big_cursor() {
 
       var restResponse = executeRestQuery(query, null);
       assertEquals(rows, restResponse.getInt("total"));
+      var restPrettyResponse = executeRestQuery(query, null, Map.of("pretty", "true"));
+      assertEquals(rows, restPrettyResponse.getInt("total"));
     }
   }
 
@@ -218,13 +231,20 @@ private static String getConnectionString() {
 
   @SneakyThrows
   protected JSONObject executeRestQuery(String query, @Nullable Integer fetch_size) {
+    return executeRestQuery(query, fetch_size, Map.of());
+  }
+
+  @SneakyThrows
+  protected JSONObject executeRestQuery(
+      String query, @Nullable Integer fetch_size, Map<String, String> params) {
     Request request = new Request("POST", QUERY_API_ENDPOINT);
     if (fetch_size != null) {
       request.setJsonEntity(
           String.format("{ \"query\": \"%s\", \"fetch_size\": %d }", query, fetch_size));
     } else {
       request.setJsonEntity(String.format("{ \"query\": \"%s\" }", query));
     }
+    request.addParameters(params);
 
     RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder();
     restOptionsBuilder.addHeader("Content-Type", "application/json");

integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java

@@ -182,7 +182,7 @@ public void validTotalResultWithAndWithoutPaginationOrderBy() throws IOException
     String selectQuery =
         StringUtils.format(
             "SELECT firstname, state FROM %s ORDER BY balance DESC ", TEST_INDEX_ACCOUNT);
-    verifyWithAndWithoutPaginationResponse(selectQuery + " LIMIT 2000", selectQuery, 26, false);
+    verifyWithAndWithoutPaginationResponse(selectQuery + " LIMIT 2000", selectQuery, 25, false);
   }
 
   @Test
@@ -423,6 +423,17 @@ public void noPaginationWithNonJDBCFormat() throws IOException {
     assertThat(rows.length, equalTo(1000));
   }
 
+  @Test
+  public void testMalformedCursorGracefullyHandled() throws IOException {
+    ResponseException result =
+        assertThrows(
+            "Expected query with malformed cursor to raise error, but didn't",
+            ResponseException.class,
+            () -> executeCursorQuery("d:a11b4db33f"));
+    assertTrue(result.getMessage().contains("Malformed cursor"));
+    assertEquals(result.getResponse().getStatusLine().getStatusCode(), 400);
+  }
+
   public void verifyWithAndWithoutPaginationResponse(
       String sqlQuery, String cursorQuery, int fetch_size, boolean shouldFallBackToV1)
       throws IOException {

integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java

@@ -468,6 +468,12 @@ protected String makeRequest(String query, int fetch_size) {
         "{\n" + "  \"fetch_size\": \"%s\",\n" + "  \"query\": \"%s\"\n" + "}", fetch_size, query);
   }
 
+  protected String makeRequest(String query, int fetch_size, String filterQuery) {
+    return String.format(
+        "{ \"fetch_size\": \"%s\", \"query\": \"%s\", \"filter\" :  %s }",
+        fetch_size, query, filterQuery);
+  }
+
   protected String makeFetchLessRequest(String query) {
     return String.format("{\n" + "  \"query\": \"%s\"\n" + "}", query);
   }

integ-test/src/test/java/org/opensearch/sql/ppl/ResourceMonitorIT.java

@@ -34,7 +34,7 @@ public void queryExceedResourceLimitShouldFail() throws IOException {
     assertEquals(500, exception.getResponse().getStatusLine().getStatusCode());
     assertThat(
         exception.getMessage(),
-        Matchers.containsString("resource is not enough to run the" + " query, quit."));
+        Matchers.containsString("insufficient resources to run the query, quit."));
 
     // update plugins.ppl.query.memory_limit to default value 85%
     updateClusterSettings(

integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java

@@ -149,8 +149,11 @@ public void onFailure(Exception e) {
 
   private Settings defaultSettings() {
     return new Settings() {
-      private final Map<Key, Integer> defaultSettings =
-          new ImmutableMap.Builder<Key, Integer>().put(Key.QUERY_SIZE_LIMIT, 200).build();
+      private final Map<Key, Object> defaultSettings =
+          new ImmutableMap.Builder<Key, Object>()
+              .put(Key.QUERY_SIZE_LIMIT, 200)
+              .put(Key.SQL_PAGINATION_API_SEARCH_AFTER, true)
+              .build();
 
       @Override
       public <T> T getSettingValue(Key key) {

integ-test/src/test/java/org/opensearch/sql/sql/PaginationFilterIT.java

@@ -34,25 +34,30 @@ public class PaginationFilterIT extends SQLIntegTestCase {
    */
   private static final Map<String, Integer> STATEMENT_TO_NUM_OF_PAGES =
       Map.of(
-          "SELECT * FROM " + TestsConstants.TEST_INDEX_ACCOUNT, 1000,
+          "SELECT * FROM " + TestsConstants.TEST_INDEX_ACCOUNT,
+          1000,
           "SELECT * FROM " + TestsConstants.TEST_INDEX_ACCOUNT + " WHERE match(address, 'street')",
-              385,
+          385,
           "SELECT * FROM "
-                  + TestsConstants.TEST_INDEX_ACCOUNT
-                  + " WHERE match(address, 'street') AND match(city, 'Ola')",
-              1,
+              + TestsConstants.TEST_INDEX_ACCOUNT
+              + " WHERE match(address, 'street') AND match(city, 'Ola')",
+          1,
           "SELECT firstname, lastname, highlight(address) FROM "
-                  + TestsConstants.TEST_INDEX_ACCOUNT
-                  + " WHERE match(address, 'street') AND match(state, 'OH')",
-              5,
+              + TestsConstants.TEST_INDEX_ACCOUNT
+              + " WHERE match(address, 'street') AND match(state, 'OH')",
+          5,
           "SELECT firstname, lastname, highlight('*') FROM "
-                  + TestsConstants.TEST_INDEX_ACCOUNT
-                  + " WHERE match(address, 'street') AND match(state, 'OH')",
-              5,
-          "SELECT * FROM " + TestsConstants.TEST_INDEX_BEER + " WHERE true", 60,
-          "SELECT * FROM " + TestsConstants.TEST_INDEX_BEER + " WHERE Id=10", 1,
-          "SELECT * FROM " + TestsConstants.TEST_INDEX_BEER + " WHERE Id + 5=15", 1,
-          "SELECT * FROM " + TestsConstants.TEST_INDEX_BANK, 7);
+              + TestsConstants.TEST_INDEX_ACCOUNT
+              + " WHERE match(address, 'street') AND match(state, 'OH')",
+          5,
+          "SELECT * FROM " + TestsConstants.TEST_INDEX_BEER + " WHERE true",
+          60,
+          "SELECT * FROM " + TestsConstants.TEST_INDEX_BEER + " WHERE Id=10",
+          1,
+          "SELECT * FROM " + TestsConstants.TEST_INDEX_BEER + " WHERE Id + 5=15",
+          1,
+          "SELECT * FROM " + TestsConstants.TEST_INDEX_BANK,
+          7);
 
   private final String sqlStatement;
 

integ-test/src/test/java/org/opensearch/sql/sql/PaginationIT.java

@@ -7,6 +7,7 @@
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
+import static org.opensearch.sql.legacy.TestUtils.getResponseBody;
 import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_CALCS;
 import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ONLINE;
 
@@ -18,6 +19,7 @@
 import org.junit.Test;
 import org.opensearch.client.Request;
 import org.opensearch.client.RequestOptions;
+import org.opensearch.client.Response;
 import org.opensearch.client.ResponseException;
 import org.opensearch.sql.common.setting.Settings;
 import org.opensearch.sql.legacy.SQLIntegTestCase;
@@ -215,4 +217,52 @@ public void testQueryWithoutFrom() {
     assertEquals(1, response.getInt("total"));
     assertEquals(1, response.getJSONArray("datarows").getJSONArray(0).getInt(0));
   }
+
+  @Test
+  public void testAlias() throws Exception {
+    String indexName = Index.ONLINE.getName();
+    String aliasName = "alias_ONLINE";
+    String filterQuery = "{\n" + "  \"term\": {\n" + "    \"107\": 72 \n" + "  }\n" + "}";
+
+    // Execute the SQL query with filter
+    String selectQuery = "SELECT * FROM " + TEST_INDEX_ONLINE;
+    JSONObject initialResponse =
+        new JSONObject(executeFetchQuery(selectQuery, 10, "jdbc", filterQuery));
+    assertEquals(initialResponse.getInt("size"), 10);
+
+    // Create an alias
+    String createAliasQuery =
+        String.format(
+            "{ \"actions\": [ { \"add\": { \"index\": \"%s\", \"alias\": \"%s\" } } ] }",
+            indexName, aliasName);
+    Request createAliasRequest = new Request("POST", "/_aliases");
+    createAliasRequest.setJsonEntity(createAliasQuery);
+    JSONObject aliasResponse = new JSONObject(executeRequest(createAliasRequest));
+
+    // Assert that alias creation was acknowledged
+    assertTrue(aliasResponse.getBoolean("acknowledged"));
+
+    // Query using the alias
+    String aliasSelectQuery = String.format("SELECT * FROM %s", aliasName);
+    JSONObject aliasQueryResponse = new JSONObject(executeFetchQuery(aliasSelectQuery, 4, "jdbc"));
+    assertEquals(4, aliasQueryResponse.getInt("size"));
+
+    // Query using the alias with filter
+    JSONObject aliasFilteredResponse =
+        new JSONObject(executeFetchQuery(aliasSelectQuery, 4, "jdbc", filterQuery));
+    assertEquals(aliasFilteredResponse.getInt("size"), 4);
+  }
+
+  private String executeFetchQuery(String query, int fetchSize, String requestType, String filter)
+      throws IOException {
+    String endpoint = "/_plugins/_sql?format=" + requestType;
+    String requestBody = makeRequest(query, fetchSize, filter);
+
+    Request sqlRequest = new Request("POST", endpoint);
+    sqlRequest.setJsonEntity(requestBody);
+
+    Response response = client().performRequest(sqlRequest);
+    String responseString = getResponseBody(response, true);
+    return responseString;
+  }
 }

integ-test/src/test/java/org/opensearch/sql/sql/StandalonePaginationIT.java

@@ -166,6 +166,7 @@ private Settings defaultSettings() {
           new ImmutableMap.Builder<Key, Object>()
               .put(Key.QUERY_SIZE_LIMIT, 200)
               .put(Key.SQL_CURSOR_KEEP_ALIVE, TimeValue.timeValueMinutes(1))
+              .put(Key.SQL_PAGINATION_API_SEARCH_AFTER, true)
               .build();
 
       @Override

integ-test/src/test/resources/expectedOutput/ppl/explain_filter_agg_push.json

@@ -8,7 +8,7 @@
       {
         "name": "OpenSearchIndexScan",
         "description": {
-          "request": "OpenSearchQueryRequest(indexName\u003dopensearch-sql_test_index_account, sourceBuilder\u003d{\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, searchDone\u003dfalse)"
+          "request": "OpenSearchQueryRequest(indexName\u003dopensearch-sql_test_index_account, sourceBuilder\u003d{\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, needClean\u003dtrue, searchDone\u003dfalse, pitId\u003dnull, cursorKeepAlive\u003dnull, searchAfter\u003dnull, searchResponse\u003dnull)"
         },
         "children": []
       }

integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push.json

@@ -8,7 +8,7 @@
       {
         "name": "OpenSearchIndexScan",
         "description": {
-          "request": "OpenSearchQueryRequest(indexName\u003dopensearch-sql_test_index_account, sourceBuilder\u003d{\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"bool\":{\"filter\":[{\"range\":{\"balance\":{\"from\":10000,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},{\"range\":{\"age\":{\"from\":null,\"to\":40,\"include_lower\":true,\"include_upper\":false,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, searchDone\u003dfalse)"
+          "request": "OpenSearchQueryRequest(indexName\u003dopensearch-sql_test_index_account, sourceBuilder\u003d{\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"bool\":{\"filter\":[{\"range\":{\"balance\":{\"from\":10000,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},{\"range\":{\"age\":{\"from\":null,\"to\":40,\"include_lower\":true,\"include_upper\":false,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, needClean\u003dtrue, searchDone\u003dfalse, pitId\u003dnull, cursorKeepAlive\u003dnull, searchAfter\u003dnull, searchResponse\u003dnull)"
         },
         "children": []
       }

integ-test/src/test/resources/expectedOutput/ppl/explain_limit_push.json

@@ -16,7 +16,7 @@
           {
             "name": "OpenSearchIndexScan",
             "description": {
-              "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\"}, searchDone=false)"
+              "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\"}, needClean\u003dtrue, searchDone\u003dfalse, pitId\u003dnull, cursorKeepAlive\u003dnull, searchAfter\u003dnull, searchResponse\u003dnull)"
             },
             "children": []
           }