diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index c820c97196..1ed2a64e76 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -1443,7 +1443,6 @@ public void testTimeFormat( private static Stream getInvalidTestDataForTimeFormat() { return Stream.of( Arguments.of(DSL.literal("asdfasdf"), DSL.literal("%f")), - Arguments.of(DSL.literal("12345"), DSL.literal("%h")), Arguments.of(DSL.literal("10:11:61"), DSL.literal("%h")), Arguments.of(DSL.literal("10:61:12"), DSL.literal("%h")), Arguments.of(DSL.literal("61:11:12"), DSL.literal("%h"))); diff --git a/integ-test/src/test/java/org/opensearch/sql/correctness/runner/connection/JDBCConnection.java b/integ-test/src/test/java/org/opensearch/sql/correctness/runner/connection/JDBCConnection.java index 7a67022117..0e60201112 100644 --- a/integ-test/src/test/java/org/opensearch/sql/correctness/runner/connection/JDBCConnection.java +++ b/integ-test/src/test/java/org/opensearch/sql/correctness/runner/connection/JDBCConnection.java @@ -8,15 +8,18 @@ import static java.util.stream.Collectors.joining; import com.google.common.base.Strings; +import com.google.common.collect.ImmutableSet; import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Statement; -import java.util.Arrays; -import java.util.List; -import java.util.Properties; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.time.temporal.ChronoField; +import java.util.*; import org.json.JSONObject; import org.opensearch.sql.correctness.runner.resultset.DBResult; import org.opensearch.sql.correctness.runner.resultset.Row; @@ -29,6 +32,20 @@ public class JDBCConnection implements DBConnection { private static final String DOUBLE_QUOTE = "''"; private static final String BACKTICK = "`"; + /** Possible types for date field* */ + private static final Set DATE_TIME_TYPES = ImmutableSet.of("DATE", "TIMESTAMP"); + + /** Formatter used to convert date time from h2 and sqlite to date * */ + private static final DateTimeFormatter DATE_OPTIONAL_TIME_NANO_FORMATTER = + new DateTimeFormatterBuilder() + .appendPattern("yyyy-MM-dd") + .optionalStart() + .appendPattern(" HH:mm:ss") + .optionalStart() + .appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true) + .optionalEnd() + .toFormatter(); + /** Database name for display */ private final String databaseName; @@ -170,16 +187,35 @@ private void populateMetaData(ResultSet resultSet, DBResult result) throws SQLEx if (Strings.isNullOrEmpty(colName)) { colName = metaData.getColumnName(i); } - result.addColumn(colName, metaData.getColumnTypeName(i)); + + String type = metaData.getColumnTypeName(i); + // OpenSearch database returns datetime as DATE + if (type.equals("DATE")) { + result.addColumn(colName, mapToJDBCType(type)); + } else { + result.addColumn(colName, type); + } } } private void populateData(ResultSet resultSet, DBResult result) throws SQLException { + while (resultSet.next()) { Row row = new Row(); + ResultSetMetaData metaData = resultSet.getMetaData(); + for (int i = 1; i <= result.columnSize(); i++) { Object value = resultSet.getObject(i); - row.add(resultSet.wasNull() ? null : value); + + // Even though the OpenSearch database returns the full date and time, + // the retrieved object from the result set contains only the date. + // We convert date-time values from other databases to a date format for precise comparison. + if (value != null && DATE_TIME_TYPES.contains(metaData.getColumnTypeName(i))) { + LocalDate dateTime = LocalDate.parse(value.toString(), DATE_OPTIONAL_TIME_NANO_FORMATTER); + row.add(dateTime.format(DateTimeFormatter.ofPattern("yyyy-MM-dd"))); + } else { + row.add(resultSet.wasNull() ? null : value); + } } result.addRow(row); } diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationExpressionIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationExpressionIT.java index 37398220ff..20f04f825e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationExpressionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationExpressionIT.java @@ -204,7 +204,7 @@ public void groupByDateShouldPass() { Index.BANK.getName())); verifySchema( - response, schema("birthdate", null, "timestamp"), schema("count(*)", "count", "integer")); + response, schema("birthdate", null, "date"), schema("count(*)", "count", "integer")); verifyDataRows(response, rows("2018-06-23 00:00:00", 1)); } @@ -220,9 +220,7 @@ public void groupByDateWithAliasShouldPass() { Index.BANK.getName())); verifySchema( - response, - schema("birthdate", "birth", "timestamp"), - schema("count(*)", "count", "integer")); + response, schema("birthdate", "birth", "date"), schema("count(*)", "count", "integer")); verifyDataRows(response, rows("2018-06-23 00:00:00", 1)); } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java index fe5c2ff270..967e51c722 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java @@ -48,8 +48,8 @@ public void test_nonnumeric_data_types() throws IOException { schema("keyword_value", "string"), schema("text_value", "string"), schema("binary_value", "binary"), - schema("date_value", "timestamp"), - schema("date_nanos_value", "timestamp"), + schema("date_value", "date"), + schema("date_nanos_value", "date"), schema("ip_value", "ip"), schema("object_value", "struct"), schema("nested_value", "array"), diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/FieldsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/FieldsCommandIT.java index e8a287c80e..bb0b2d097a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/FieldsCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/FieldsCommandIT.java @@ -56,7 +56,7 @@ public void testFieldsWildCard() throws IOException { public void testSelectDateTypeField() throws IOException { JSONObject result = executeQuery(String.format("source=%s | fields birthdate", TEST_INDEX_BANK)); - verifySchema(result, schema("birthdate", null, "timestamp")); + verifySchema(result, schema("birthdate", null, "date")); verifyDataRows( result, diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java index 2d1cd709e1..f184d0855d 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java @@ -175,9 +175,7 @@ public void testStatsTimeSpan() throws IOException { executeQuery( String.format("source=%s | stats count() by span(birthdate,1y)", TEST_INDEX_BANK)); verifySchema( - response, - schema("count()", null, "integer"), - schema("span(birthdate,1y)", null, "timestamp")); + response, schema("count()", null, "integer"), schema("span(birthdate,1y)", null, "date")); verifyDataRows(response, rows(2, "2017-01-01 00:00:00"), rows(5, "2018-01-01 00:00:00")); } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java index c1356ce838..1aa5e6b3b5 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java @@ -82,15 +82,6 @@ public void typeof_opensearch_types() throws IOException { TEST_INDEX_DATATYPE_NONNUMERIC)); verifyDataRows( response, - rows( - "TEXT", - "TIMESTAMP", - "TIMESTAMP", - "BOOLEAN", - "OBJECT", - "KEYWORD", - "IP", - "BINARY", - "GEO_POINT")); + rows("TEXT", "DATE", "DATE", "BOOLEAN", "OBJECT", "KEYWORD", "IP", "BINARY", "GEO_POINT")); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java index 13c2eecd56..48798d4ac5 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java @@ -44,8 +44,8 @@ public void testReadingDateFormats() throws IOException { verifySchema( result, schema("weekyear_week_day", null, "date"), - schema("hour_minute_second_millis", null, "time"), - schema("strict_ordinal_date_time", null, "timestamp")); + schema("hour_minute_second_millis", null, "date"), + schema("strict_ordinal_date_time", null, "date")); verifyDataRows(result, rows("1984-04-12", "09:07:42", "1984-04-12 09:07:42.000123456")); } @@ -68,11 +68,11 @@ public void testCustomFormats() { JSONObject result = executeQuery(query); verifySchema( result, - schema("custom_time", null, "time"), - schema("custom_timestamp", null, "timestamp"), + schema("custom_time", null, "date"), + schema("custom_timestamp", null, "date"), schema("custom_date_or_date", null, "date"), - schema("custom_date_or_custom_time", null, "timestamp"), - schema("custom_time_parser_check", null, "time")); + schema("custom_date_or_custom_time", null, "date"), + schema("custom_time_parser_check", null, "date")); verifyDataRows( result, rows( @@ -97,8 +97,8 @@ public void testCustomFormats2() { verifySchema( result, schema("custom_no_delimiter_date", null, "date"), - schema("custom_no_delimiter_time", null, "time"), - schema("custom_no_delimiter_ts", null, "timestamp")); + schema("custom_no_delimiter_time", null, "date"), + schema("custom_no_delimiter_ts", null, "date")); verifyDataRows( result, rows("1984-10-20", "10:20:30", "1984-10-20 15:35:48"), @@ -116,10 +116,10 @@ public void testIncompleteFormats() { JSONObject result = executeQuery(query); verifySchema( result, - schema("incomplete_1", null, "timestamp"), + schema("incomplete_1", null, "date"), schema("incomplete_2", null, "date"), - schema("incorrect", null, "timestamp"), - schema("incomplete_custom_time", null, "time"), + schema("incorrect", null, "date"), + schema("incomplete_custom_time", null, "date"), schema("incomplete_custom_date", null, "date")); verifyDataRows( result, @@ -133,8 +133,7 @@ public void testNumericFormats() { String query = String.format("SELECT epoch_sec, epoch_milli" + " FROM %s", TEST_INDEX_DATE_FORMATS); JSONObject result = executeQuery(query); - verifySchema( - result, schema("epoch_sec", null, "timestamp"), schema("epoch_milli", null, "timestamp")); + verifySchema(result, schema("epoch_sec", null, "date"), schema("epoch_milli", null, "date")); verifyDataRows( result, rows("1970-01-01 00:00:42", "1970-01-01 00:00:00.042"), @@ -147,7 +146,7 @@ public void testDateNanosWithFormats() { String query = String.format("SELECT hour_minute_second_OR_t_time" + " FROM %s", TEST_INDEX_DATE_FORMATS); JSONObject result = executeQuery(query); - verifySchema(result, schema("hour_minute_second_OR_t_time", null, "time")); + verifySchema(result, schema("hour_minute_second_OR_t_time", null, "date")); verifyDataRows(result, rows("09:07:42"), rows("07:07:42.123456789")); } @@ -182,7 +181,7 @@ public void testDateNanosWithFunctions() { + " FROM %s WHERE hour_minute_second_OR_t_time > TIME '08:07:00'", TEST_INDEX_DATE_FORMATS); result = executeQuery(query); - verifySchema(result, schema("hour_minute_second_OR_t_time", null, "time")); + verifySchema(result, schema("hour_minute_second_OR_t_time", null, "date")); verifyDataRows(result, rows("09:07:42")); query = String.format( @@ -190,7 +189,7 @@ public void testDateNanosWithFunctions() { + " FROM %s WHERE hour_minute_second_OR_t_time < TIME '08:07:00'", TEST_INDEX_DATE_FORMATS); result = executeQuery(query); - verifySchema(result, schema("hour_minute_second_OR_t_time", null, "time")); + verifySchema(result, schema("hour_minute_second_OR_t_time", null, "date")); verifyDataRows(result, rows("07:07:42.123456789")); } @@ -203,7 +202,7 @@ public void testDateNanosOrderBy() { + " FROM %s ORDER BY hour_minute_second_OR_t_time ASC", TEST_INDEX_DATE_FORMATS); JSONObject result = executeQuery(query); - verifySchema(result, schema("hour_minute_second_OR_t_time", null, "time")); + verifySchema(result, schema("hour_minute_second_OR_t_time", null, "date")); verifyDataRows(result, rows("07:07:42.123456789"), rows("09:07:42")); } @@ -225,7 +224,7 @@ public void testDateNanosWithNanos() { String query = String.format("SELECT date_nanos_value" + " FROM %s", TEST_INDEX_DATATYPE_NONNUMERIC); JSONObject result = executeQuery(query); - verifySchema(result, schema("date_nanos_value", null, "timestamp")); + verifySchema(result, schema("date_nanos_value", null, "date")); verifyDataRows(result, rows("2019-03-24 01:34:46.123456789")); } diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/JdbcFormatIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/JdbcFormatIT.java index f36992b1d0..4f8ee17de7 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/JdbcFormatIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/JdbcFormatIT.java @@ -35,7 +35,7 @@ public void testSimpleDataTypesInSchema() { schema("account_number", "long"), schema("address", "text"), schema("age", "integer"), - schema("birthdate", "timestamp"), + schema("birthdate", "date"), schema("city", "keyword"), schema("male", "boolean"), schema("state", "text")); diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java index 7129d058c0..cd70c5e020 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java @@ -65,15 +65,6 @@ public void typeof_opensearch_types() { TEST_INDEX_DATATYPE_NONNUMERIC)); verifyDataRows( response, - rows( - "TEXT", - "TIMESTAMP", - "TIMESTAMP", - "BOOLEAN", - "OBJECT", - "KEYWORD", - "IP", - "BINARY", - "GEO_POINT")); + rows("TEXT", "DATE", "DATE", "BOOLEAN", "OBJECT", "KEYWORD", "IP", "BINARY", "GEO_POINT")); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java index ddbba61260..7c9513afef 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java @@ -62,19 +62,23 @@ public String toString() { @EqualsAndHashCode.Exclude @Getter protected MappingType mappingType; // resolved ExprCoreType - protected ExprCoreType exprCoreType; + @Getter protected ExprCoreType exprCoreType; /** * Get a simplified type {@link ExprCoreType} if possible. To avoid returning `UNKNOWN` for - * `OpenSearch*Type`s, e.g. for IP, returns itself. + * `OpenSearch*Type`s, e.g. for IP, returns itself. If the `exprCoreType` is {@link + * ExprCoreType#DATE}, {@link ExprCoreType#TIMESTAMP}, {@link ExprCoreType#TIME}, or {@link + * ExprCoreType#UNKNOWN}, it returns the current instance; otherwise, it returns `exprCoreType`. * * @return An {@link ExprType}. */ public ExprType getExprType() { - if (exprCoreType != ExprCoreType.UNKNOWN) { - return exprCoreType; - } - return this; + return (exprCoreType == ExprCoreType.DATE + || exprCoreType == ExprCoreType.TIMESTAMP + || exprCoreType == ExprCoreType.TIME + || exprCoreType == ExprCoreType.UNKNOWN) + ? this + : exprCoreType; } /** diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java index 7e6bee77c2..215c0b0690 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java @@ -11,11 +11,17 @@ import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.temporal.TemporalAccessor; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; import lombok.EqualsAndHashCode; import org.opensearch.common.time.DateFormatter; +import org.opensearch.common.time.DateFormatters; import org.opensearch.common.time.FormatNames; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; @@ -26,6 +32,9 @@ public class OpenSearchDateType extends OpenSearchDataType { private static final OpenSearchDateType instance = new OpenSearchDateType(); + /** Could be user defined custom or OpenSearch named formatter * */ + private DateFormatter formatter; + /** Numeric formats which support full datetime. */ public static final List SUPPORTED_NAMED_NUMERIC_FORMATS = List.of(FormatNames.EPOCH_MILLIS, FormatNames.EPOCH_SECOND); @@ -137,6 +146,9 @@ public class OpenSearchDateType extends OpenSearchDataType { private static final String CUSTOM_FORMAT_DATE_SYMBOLS = "FecEWwYqQgdMLDyuG"; + private static final List OPENSEARCH_DEFAULT_FORMATS = + Arrays.asList("strict_date_time_no_millis", "strict_date_optional_time", "epoch_millis"); + @EqualsAndHashCode.Exclude private final List formats; private OpenSearchDateType() { @@ -235,6 +247,71 @@ public List getAllCustomFormatters() { .collect(Collectors.toList()); } + /** + * Retrieves a list of custom formatters and OpenSearch named formatters defined by the user, and + * attempts to parse the given date/time string using these formatters. + * + * @param dateTime The date/time string to parse. + * @return A ZonedDateTime representing the parsed date/time in UTC, or null if parsing fails. + */ + public ZonedDateTime getParsedDateTime(String dateTime) { + List dateFormatters = this.getAllNamedFormatters(); + dateFormatters.addAll(this.getAllCustomFormatters()); + ZonedDateTime zonedDateTime = null; + + // check if dateFormatters are empty, then set default ones + if (dateFormatters.isEmpty()) { + dateFormatters = initializeDateFormatters(); + } + // parse using OpenSearch DateFormatters + for (DateFormatter formatter : dateFormatters) { + try { + TemporalAccessor accessor = formatter.parse(dateTime); + zonedDateTime = DateFormatters.from(accessor).withZoneSameLocal(ZoneOffset.UTC); + this.formatter = formatter; + break; + } catch (IllegalArgumentException ignored) { + // nothing to do, try another format + } + } + return zonedDateTime; + } + + /** + * Returns a formatted date string using the internal formatter, if available. + * + * @param accessor The TemporalAccessor object containing the date/time information. + * @return A formatted date string if a formatter is available, otherwise null. + */ + public String getFormattedDate(TemporalAccessor accessor) { + if (!hasNoFormatter()) { + return this.formatter.format(accessor); + } + return null; + } + + /** + * Checks if the formatter is not initialized. + * + * @return True if the formatter is not set, otherwise false. + */ + public boolean hasNoFormatter() { + return this.formatter == null; + } + + /** + * Initializes and returns a list of default OpenSearch date formatters. + * + * @return A list of DateFormatter objects initialized with default patterns. + */ + private static List initializeDateFormatters() { + List dateFormatters = new ArrayList<>(); + for (String pattern : OPENSEARCH_DEFAULT_FORMATS) { + dateFormatters.add(DateFormatter.forPattern(pattern)); + } + return dateFormatters; + } + /** * Retrieves a list of named formatters that format for dates. * diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java index 3341e01ab2..3cb182de5b 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java @@ -230,7 +230,7 @@ private Optional type(String field) { private static ExprValue parseDateTimeString(String value, OpenSearchDateType dataType) { List formatters = dataType.getAllNamedFormatters(); formatters.addAll(dataType.getAllCustomFormatters()); - ExprCoreType returnFormat = (ExprCoreType) dataType.getExprType(); + ExprCoreType returnFormat = dataType.getExprCoreType(); for (DateFormatter formatter : formatters) { try { @@ -273,8 +273,7 @@ private static ExprValue parseDateTimeString(String value, OpenSearchDateType da private static ExprValue createOpenSearchDateType(Content value, ExprType type) { OpenSearchDateType dt = (OpenSearchDateType) type; - ExprType returnFormat = dt.getExprType(); - + ExprCoreType returnFormat = dt.getExprCoreType(); if (value.isNumber()) { // isNumber var numFormatters = dt.getNumericNamedFormatters(); if (numFormatters.size() > 0 || !dt.hasFormats()) { @@ -287,7 +286,7 @@ private static ExprValue createOpenSearchDateType(Content value, ExprType type) epochMillis = value.longValue(); } Instant instant = Instant.ofEpochMilli(epochMillis); - switch ((ExprCoreType) returnFormat) { + switch (returnFormat) { case TIME: return new ExprTimeValue(LocalTime.from(instant.atZone(ZoneOffset.UTC))); case DATE: diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilder.java index ff66ec425a..4488128b97 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilder.java @@ -23,6 +23,7 @@ import org.opensearch.sql.ast.expression.SpanUnit; import org.opensearch.sql.expression.NamedExpression; import org.opensearch.sql.expression.span.SpanExpression; +import org.opensearch.sql.opensearch.data.type.OpenSearchDateType; import org.opensearch.sql.opensearch.storage.serialization.ExpressionSerializer; /** Bucket Aggregation Builder. */ @@ -65,7 +66,10 @@ private CompositeValuesSourceBuilder buildCompositeValuesSourceBuilder( .missingOrder(missingOrder) .order(sortOrder); // Time types values are converted to LONG in ExpressionAggregationScript::execute - if (List.of(TIMESTAMP, TIME, DATE).contains(expr.getDelegated().type())) { + if ((expr.getDelegated().type() instanceof OpenSearchDateType + && List.of(TIMESTAMP, TIME, DATE) + .contains(((OpenSearchDateType) expr.getDelegated().type()).getExprCoreType())) + || List.of(TIMESTAMP, TIME, DATE).contains(expr.getDelegated().type())) { sourceBuilder.userValuetypeHint(ValueType.LONG); } return helper.build(expr.getDelegated(), sourceBuilder::field, sourceBuilder::script); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQuery.java index 11533c754e..6aebee561c 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQuery.java @@ -8,6 +8,7 @@ import static org.opensearch.sql.analysis.NestedAnalyzer.isNestedFunction; import com.google.common.collect.ImmutableMap; +import java.time.ZonedDateTime; import java.util.Map; import java.util.function.Function; import org.opensearch.index.query.QueryBuilder; @@ -32,10 +33,13 @@ import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.FunctionName; +import org.opensearch.sql.opensearch.data.type.OpenSearchDateType; /** Lucene query abstraction that builds Lucene query from function expression. */ public abstract class LuceneQuery { + private ReferenceExpression ref; + /** * Check if function expression supported by current Lucene query. Default behavior is that report * supported if: @@ -102,10 +106,11 @@ private boolean literalExpressionWrappedByCast(FunctionExpression func) { * @return query */ public QueryBuilder build(FunctionExpression func) { - ReferenceExpression ref = (ReferenceExpression) func.getArguments().get(0); + this.ref = (ReferenceExpression) func.getArguments().get(0); Expression expr = func.getArguments().get(1); ExprValue literalValue = expr instanceof LiteralExpression ? expr.valueOf() : cast((FunctionExpression) expr); + return doBuild(ref.getAttr(), ref.type(), literalValue); } @@ -120,7 +125,7 @@ private ExprValue cast(FunctionExpression castFunction) { ImmutableMap.>builder() .put( BuiltinFunctionName.CAST_TO_STRING.getName(), - expr -> { + (expr) -> { if (!expr.type().equals(ExprCoreType.STRING)) { return new ExprStringValue(String.valueOf(expr.valueOf().value())); } else { @@ -209,7 +214,16 @@ private ExprValue cast(FunctionExpression castFunction) { .put( BuiltinFunctionName.CAST_TO_DATE.getName(), expr -> { - if (expr.type().equals(ExprCoreType.STRING)) { + if (expr.type().equals(ExprCoreType.STRING) + && this.ref.type() instanceof OpenSearchDateType) { + ZonedDateTime zonedDateTime = + ((OpenSearchDateType) this.ref.type()) + .getParsedDateTime(expr.valueOf().stringValue()); + if (zonedDateTime != null) { + return new ExprDateValue(zonedDateTime.toLocalDate()); + } + return new ExprDateValue(expr.valueOf().stringValue()); + } else if (expr.type().equals(ExprCoreType.STRING)) { return new ExprDateValue(expr.valueOf().stringValue()); } else { return new ExprDateValue(expr.valueOf().dateValue()); @@ -218,7 +232,16 @@ private ExprValue cast(FunctionExpression castFunction) { .put( BuiltinFunctionName.CAST_TO_TIME.getName(), expr -> { - if (expr.type().equals(ExprCoreType.STRING)) { + if (expr.type().equals(ExprCoreType.STRING) + && this.ref.type() instanceof OpenSearchDateType) { + ZonedDateTime zonedDateTime = + ((OpenSearchDateType) this.ref.type()) + .getParsedDateTime(expr.valueOf().stringValue()); + if (zonedDateTime != null) { + return new ExprTimeValue(zonedDateTime.toLocalTime()); + } + return new ExprTimeValue(expr.valueOf().stringValue()); + } else if (expr.type().equals(ExprCoreType.STRING)) { return new ExprTimeValue(expr.valueOf().stringValue()); } else { return new ExprTimeValue(expr.valueOf().timeValue()); @@ -227,7 +250,16 @@ private ExprValue cast(FunctionExpression castFunction) { .put( BuiltinFunctionName.CAST_TO_TIMESTAMP.getName(), expr -> { - if (expr.type().equals(ExprCoreType.STRING)) { + if (expr.type().equals(ExprCoreType.STRING) + && this.ref.type() instanceof OpenSearchDateType) { + ZonedDateTime zonedDateTime = + ((OpenSearchDateType) this.ref.type()) + .getParsedDateTime(expr.valueOf().stringValue()); + if (zonedDateTime != null) { + return new ExprTimestampValue(zonedDateTime.toInstant()); + } + return new ExprTimestampValue(expr.valueOf().stringValue()); + } else if (expr.type().equals(ExprCoreType.STRING)) { return new ExprTimestampValue(expr.valueOf().stringValue()); } else { return new ExprTimestampValue(expr.valueOf().timestampValue()); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/RangeQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/RangeQuery.java index 2e33e3cc7c..aba9bc6fd2 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/RangeQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/RangeQuery.java @@ -12,6 +12,7 @@ import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.data.type.OpenSearchDateType; /** Lucene query that builds range query for non-quality comparison. */ @RequiredArgsConstructor @@ -30,7 +31,8 @@ public enum Comparison { @Override protected QueryBuilder doBuild(String fieldName, ExprType fieldType, ExprValue literal) { - Object value = value(literal); + + Object value = value(literal, fieldType); RangeQueryBuilder query = QueryBuilders.rangeQuery(fieldName); switch (comparison) { @@ -47,11 +49,23 @@ protected QueryBuilder doBuild(String fieldName, ExprType fieldType, ExprValue l } } - private Object value(ExprValue literal) { - if (literal.type().equals(ExprCoreType.TIMESTAMP)) { - return literal.timestampValue().toEpochMilli(); - } else { - return literal.value(); + private Object value(ExprValue literal, ExprType fieldType) { + if (fieldType instanceof OpenSearchDateType) { + OpenSearchDateType openSearchDateType = (OpenSearchDateType) fieldType; + if (literal.type().equals(ExprCoreType.TIMESTAMP)) { + return openSearchDateType.hasNoFormatter() + ? literal.timestampValue().toEpochMilli() + : openSearchDateType.getFormattedDate(literal.timestampValue()); + } else if (literal.type().equals(ExprCoreType.DATE)) { + return openSearchDateType.hasNoFormatter() + ? literal.value() + : openSearchDateType.getFormattedDate(literal.dateValue()); + } else if (literal.type().equals(ExprCoreType.TIME)) { + return openSearchDateType.hasNoFormatter() + ? literal.value() + : openSearchDateType.getFormattedDate(literal.timeValue()); + } } + return literal.value(); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQuery.java index cd506898d7..fddeb13a0b 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQuery.java @@ -10,6 +10,7 @@ import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.data.type.OpenSearchDateType; import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; /** Lucene query that build term query for equality comparison. */ @@ -18,14 +19,26 @@ public class TermQuery extends LuceneQuery { @Override protected QueryBuilder doBuild(String fieldName, ExprType fieldType, ExprValue literal) { fieldName = OpenSearchTextType.convertTextToKeyword(fieldName, fieldType); - return QueryBuilders.termQuery(fieldName, value(literal)); + return QueryBuilders.termQuery(fieldName, value(literal, fieldType)); } - private Object value(ExprValue literal) { - if (literal.type().equals(ExprCoreType.TIMESTAMP)) { - return literal.timestampValue().toEpochMilli(); - } else { - return literal.value(); + private Object value(ExprValue literal, ExprType fieldType) { + if (fieldType instanceof OpenSearchDateType) { + OpenSearchDateType openSearchDateType = (OpenSearchDateType) fieldType; + if (literal.type().equals(ExprCoreType.TIMESTAMP)) { + return openSearchDateType.hasNoFormatter() + ? literal.timestampValue().toEpochMilli() + : openSearchDateType.getFormattedDate(literal.timestampValue()); + } else if (literal.type().equals(ExprCoreType.DATE)) { + return openSearchDateType.hasNoFormatter() + ? literal.value() + : openSearchDateType.getFormattedDate(literal.dateValue()); + } else if (literal.type().equals(ExprCoreType.TIME)) { + return openSearchDateType.hasNoFormatter() + ? literal.value() + : openSearchDateType.getFormattedDate(literal.timeValue()); + } } + return literal.value(); } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java index 82e6222dc4..3bbf8f19fb 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java @@ -124,7 +124,15 @@ public void of_MappingType(MappingType mappingType, String name, ExprType dataTy assertAll( () -> assertEquals(nameForPPL, type.typeName()), () -> assertEquals(nameForSQL, type.legacyTypeName()), - () -> assertEquals(dataType, type.getExprType())); + () -> { + if (dataType == ExprCoreType.TIMESTAMP + || dataType == ExprCoreType.DATE + || dataType == ExprCoreType.TIME) { + assertEquals(dataType, type.getExprCoreType()); + } else { + assertEquals(dataType, type.getExprType()); + } + }); } @ParameterizedTest(name = "{0}") @@ -133,7 +141,7 @@ public void of_ExprCoreType(ExprCoreType coreType) { assumeFalse(coreType == UNKNOWN); var type = OpenSearchDataType.of(coreType); if (type instanceof OpenSearchDateType) { - assertEquals(coreType, type.getExprType()); + assertEquals(coreType, type.getExprCoreType()); } else { assertEquals(coreType.toString(), type.typeName()); assertEquals(coreType.toString(), type.legacyTypeName()); @@ -416,7 +424,7 @@ public void test_getExprType() { assertEquals(FLOAT, OpenSearchDataType.of(MappingType.HalfFloat).getExprType()); assertEquals(DOUBLE, OpenSearchDataType.of(MappingType.Double).getExprType()); assertEquals(DOUBLE, OpenSearchDataType.of(MappingType.ScaledFloat).getExprType()); - assertEquals(TIMESTAMP, OpenSearchDataType.of(MappingType.Date).getExprType()); + assertEquals(TIMESTAMP, OpenSearchDataType.of(MappingType.Date).getExprCoreType()); } @Test diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateTypeTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateTypeTest.java index c6885c8ffe..4728b454f9 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateTypeTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateTypeTest.java @@ -5,12 +5,7 @@ package org.opensearch.sql.opensearch.data.type; -import static org.junit.jupiter.api.Assertions.assertAll; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertSame; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.Assertions.*; import static org.opensearch.sql.data.type.ExprCoreType.DATE; import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; @@ -22,6 +17,9 @@ import static org.opensearch.sql.opensearch.data.type.OpenSearchDateType.isDateTypeCompatible; import com.google.common.collect.Lists; +import java.time.*; +import java.time.format.DateTimeFormatter; +import java.util.Arrays; import java.util.EnumSet; import java.util.List; import java.util.stream.Stream; @@ -48,8 +46,6 @@ class OpenSearchDateTypeTest { OpenSearchDateType.of(defaultFormatString); private static final OpenSearchDateType dateDateType = OpenSearchDateType.of(dateFormatString); private static final OpenSearchDateType timeDateType = OpenSearchDateType.of(timeFormatString); - private static final OpenSearchDateType datetimeDateType = - OpenSearchDateType.of(timestampFormatString); @Test public void isCompatible() { @@ -94,9 +90,9 @@ public void check_legacyTypeName() { public void check_exprTypeName() { assertAll( // exprType changes based on type (no datetime): - () -> assertEquals(TIMESTAMP, defaultDateType.getExprType()), - () -> assertEquals(TIME, timeDateType.getExprType()), - () -> assertEquals(DATE, dateDateType.getExprType())); + () -> assertEquals(TIMESTAMP, defaultDateType.getExprCoreType()), + () -> assertEquals(TIME, timeDateType.getExprCoreType()), + () -> assertEquals(DATE, dateDateType.getExprCoreType())); } private static Stream getAllSupportedFormats() { @@ -129,22 +125,22 @@ public void check_datetime_format_names(FormatNames datetimeFormat) { if (camelCaseName != null && !camelCaseName.isEmpty()) { OpenSearchDateType dateType = OpenSearchDateType.of(camelCaseName); assertSame( - dateType.getExprType(), + dateType.getExprCoreType(), TIMESTAMP, camelCaseName + " does not format to a TIMESTAMP type, instead got " - + dateType.getExprType()); + + dateType.getExprCoreType()); } String snakeCaseName = datetimeFormat.getSnakeCaseName(); if (snakeCaseName != null && !snakeCaseName.isEmpty()) { OpenSearchDateType dateType = OpenSearchDateType.of(snakeCaseName); assertSame( - dateType.getExprType(), + dateType.getExprCoreType(), TIMESTAMP, snakeCaseName + " does not format to a TIMESTAMP type, instead got " - + dateType.getExprType()); + + dateType.getExprCoreType()); } else { fail(); } @@ -161,18 +157,22 @@ public void check_date_format_names(FormatNames dateFormat) { if (camelCaseName != null && !camelCaseName.isEmpty()) { OpenSearchDateType dateType = OpenSearchDateType.of(camelCaseName); assertSame( - dateType.getExprType(), + dateType.getExprCoreType(), DATE, - camelCaseName + " does not format to a DATE type, instead got " + dateType.getExprType()); + camelCaseName + + " does not format to a DATE type, instead got " + + dateType.getExprCoreType()); } String snakeCaseName = dateFormat.getSnakeCaseName(); if (snakeCaseName != null && !snakeCaseName.isEmpty()) { OpenSearchDateType dateType = OpenSearchDateType.of(snakeCaseName); assertSame( - dateType.getExprType(), + dateType.getExprCoreType(), DATE, - snakeCaseName + " does not format to a DATE type, instead got " + dateType.getExprType()); + snakeCaseName + + " does not format to a DATE type, instead got " + + dateType.getExprCoreType()); } else { fail(); } @@ -189,18 +189,22 @@ public void check_time_format_names(FormatNames timeFormat) { if (camelCaseName != null && !camelCaseName.isEmpty()) { OpenSearchDateType dateType = OpenSearchDateType.of(camelCaseName); assertSame( - dateType.getExprType(), + dateType.getExprCoreType(), TIME, - camelCaseName + " does not format to a TIME type, instead got " + dateType.getExprType()); + camelCaseName + + " does not format to a TIME type, instead got " + + dateType.getExprCoreType()); } String snakeCaseName = timeFormat.getSnakeCaseName(); if (snakeCaseName != null && !snakeCaseName.isEmpty()) { OpenSearchDateType dateType = OpenSearchDateType.of(snakeCaseName); assertSame( - dateType.getExprType(), + dateType.getExprCoreType(), TIME, - snakeCaseName + " does not format to a TIME type, instead got " + dateType.getExprType()); + snakeCaseName + + " does not format to a TIME type, instead got " + + dateType.getExprCoreType()); } else { fail(); } @@ -244,9 +248,9 @@ private static Stream get_format_combinations_for_test() { @MethodSource("get_format_combinations_for_test") public void check_ExprCoreType_of_combinations_of_custom_and_predefined_formats( ExprCoreType expected, List formats, String testName) { - assertEquals(expected, OpenSearchDateType.of(String.join(" || ", formats)).getExprType()); + assertEquals(expected, OpenSearchDateType.of(String.join(" || ", formats)).getExprCoreType()); formats = Lists.reverse(formats); - assertEquals(expected, OpenSearchDateType.of(String.join(" || ", formats)).getExprType()); + assertEquals(expected, OpenSearchDateType.of(String.join(" || ", formats)).getExprCoreType()); } @Test @@ -259,4 +263,138 @@ public void check_if_date_type_compatible() { assertTrue(isDateTypeCompatible(DATE)); assertFalse(isDateTypeCompatible(OpenSearchDataType.of(OpenSearchDataType.MappingType.Text))); } + + @Test + void testValidTimestampWithCustomFormat() { + String timestamp = "2021-11-08T17:00:00Z"; + String format = "strict_date_time_no_millis"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + ZonedDateTime zonedDateTime = dateType.getParsedDateTime(timestamp); + + assertEquals("2021-11-08T17:00:00Z", dateType.getFormattedDate(zonedDateTime.toInstant())); + assertEquals(LocalDate.parse("2021-11-08"), zonedDateTime.toLocalDate()); + assertFalse(dateType.hasNoFormatter()); + } + + @Test + void testValidTimestampWithMultipleFormats() { + String timestamp = "2021-11-08T17:00:00Z"; + String timestamp2 = "2021/11/08T17:00:00Z"; + + List formats = Arrays.asList("strict_date_time_no_millis", "yyyy/MM/dd'T'HH:mm:ssX"); + OpenSearchDateType dateType = OpenSearchDateType.of(String.join(" || ", formats)); + + // Testing with the first timestamp + ZonedDateTime zonedDateTime1 = dateType.getParsedDateTime(timestamp); + + assertEquals("2021-11-08T17:00:00Z", dateType.getFormattedDate(zonedDateTime1.toInstant())); + assertEquals(LocalDate.parse("2021-11-08"), zonedDateTime1.toLocalDate()); + assertFalse(dateType.hasNoFormatter()); + + // Testing with the second timestamp + ZonedDateTime zonedDateTime2 = dateType.getParsedDateTime(timestamp2); + + assertEquals("2021/11/08T17:00:00Z", dateType.getFormattedDate(zonedDateTime2.toInstant())); + assertEquals(LocalDate.parse("2021-11-08"), zonedDateTime2.toLocalDate()); + assertFalse(dateType.hasNoFormatter()); + } + + @Test + void testOpenSearchDateTimeNamedFormatter() { + String timestamp = "2019-03-23T21:34:46"; + String format = "strict_date_hour_minute_second"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + ZonedDateTime zonedDateTime = dateType.getParsedDateTime(timestamp); + + assertEquals("2019-03-23T21:34:46", dateType.getFormattedDate(zonedDateTime.toInstant())); + assertEquals(LocalDate.parse("2019-03-23"), zonedDateTime.toLocalDate()); + assertEquals(LocalTime.parse("21:34:46"), zonedDateTime.toLocalTime()); + assertFalse(dateType.hasNoFormatter()); + } + + @Test + void testInvalidTimestamp() { + String timestamp = "invalid-timestamp"; + List formats = Arrays.asList("yyyy/MM/dd'T'HH:mm:ssX", "yyyy-MM-dd'T'HH:mm:ssX"); + OpenSearchDateType dateType = OpenSearchDateType.of(String.join(" || ", formats)); + ZonedDateTime zonedDateTime = dateType.getParsedDateTime(timestamp); + assertNull(dateType.getFormattedDate(zonedDateTime)); + assertNull(zonedDateTime); + assertTrue(dateType.hasNoFormatter()); + } + + @Test + void testEpochDateTimeFormatter() { + long epochTimestamp = 1636390800000L; // Corresponds to "2021-11-08T17:00:00Z" + String format = "epoch_millis"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + ZonedDateTime zonedDateTime = dateType.getParsedDateTime(String.valueOf(epochTimestamp)); + + assertEquals(Long.toString(epochTimestamp), dateType.getFormattedDate(zonedDateTime)); + assertEquals(LocalDate.parse("2021-11-08"), zonedDateTime.toLocalDate()); + assertEquals(LocalTime.parse("17:00:00"), zonedDateTime.toLocalTime()); + assertFalse(dateType.hasNoFormatter()); + } + + @Test + void testCustomTimeStampFormatWithDefaultFormatters() { + String timestamp = "2021-11-08 17:00:00"; + String format = "strict_date_optional_time || epoch_millis"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + ZonedDateTime zonedDateTime = dateType.getParsedDateTime(timestamp); + assertNull(dateType.getParsedDateTime(timestamp)); + assertNull(dateType.getFormattedDate(zonedDateTime)); + } + + @Test + void testValidDateWithCustomFormatter() { + String dateString = "2021-11-08"; + String format = "yyyy-MM-dd"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + + LocalDate expectedDate = LocalDate.parse(dateString, DateTimeFormatter.ISO_DATE); + LocalDate parsedDate = dateType.getParsedDateTime(dateString).toLocalDate(); + + assertEquals(expectedDate, parsedDate); + assertEquals("2021-11-08", dateType.getFormattedDate(parsedDate)); + } + + @Test + void testValidDateWithMultipleFormatters() { + String dateString = "2021-11-08"; + String format = "yyyy/MM/dd || yyyy-MM-dd"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + + LocalDate expectedDate = LocalDate.parse(dateString, DateTimeFormatter.ofPattern("yyyy-MM-dd")); + LocalDate parsedDate = dateType.getParsedDateTime(dateString).toLocalDate(); + + assertEquals(expectedDate, parsedDate); + assertEquals("2021-11-08", dateType.getFormattedDate(parsedDate)); + } + + @Test + void testValidTimeWithCustomFormatter() { + String timeString = "12:10:30.000"; + String format = "HH:mm:ss.SSS"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + + LocalTime expectedTime = LocalTime.parse(timeString, DateTimeFormatter.ofPattern(format)); + LocalTime parsedTime = dateType.getParsedDateTime(timeString).toLocalTime(); + + assertEquals(expectedTime, parsedTime); + assertEquals("12:10:30.000", dateType.getFormattedDate(parsedTime)); + } + + @Test + void testValidTimeWithMultipleFormatters() { + String timeString = "12:10:30"; + String format = "HH:mm:ss.SSS || HH:mm:ss"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + + LocalTime expectedTime = LocalTime.parse(timeString, DateTimeFormatter.ofPattern("HH:mm:ss")); + LocalTime parsedTime = dateType.getParsedDateTime(timeString).toLocalTime(); + + assertEquals(expectedTime, parsedTime); + assertEquals("12:10:30", dateType.getFormattedDate(parsedTime)); + } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java index 3ddb07d86a..3ca566fac6 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java @@ -148,7 +148,7 @@ void getFieldTypes() { hasEntry("gender", ExprCoreType.BOOLEAN), hasEntry("family", ExprCoreType.ARRAY), hasEntry("employer", ExprCoreType.STRUCT), - hasEntry("birthday", ExprCoreType.TIMESTAMP), + hasEntry("birthday", (ExprType) OpenSearchDataType.of(MappingType.Date)), hasEntry("id1", ExprCoreType.BYTE), hasEntry("id2", ExprCoreType.SHORT), hasEntry("blob", (ExprType) OpenSearchDataType.of(MappingType.Binary)))); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilderTest.java index 4250b3297f..08c4017f1d 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilderTest.java @@ -40,6 +40,7 @@ import org.opensearch.sql.expression.NamedExpression; import org.opensearch.sql.expression.parse.ParseExpression; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; +import org.opensearch.sql.opensearch.data.type.OpenSearchDateType; import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; import org.opensearch.sql.opensearch.storage.serialization.ExpressionSerializer; @@ -134,6 +135,39 @@ void should_build_bucket_with_parse_expression() { buildQuery(Arrays.asList(asc(named("name", parseExpression))))); } + @Test + void terms_bucket_for_opensearchdate_type_uses_long() { + OpenSearchDateType dataType = OpenSearchDateType.of(ExprCoreType.TIMESTAMP); + + assertEquals( + "{\n" + + " \"terms\" : {\n" + + " \"field\" : \"date\",\n" + + " \"missing_bucket\" : true,\n" + + " \"value_type\" : \"long\",\n" + + " \"missing_order\" : \"first\",\n" + + " \"order\" : \"asc\"\n" + + " }\n" + + "}", + buildQuery(Arrays.asList(asc(named("date", ref("date", dataType)))))); + } + + @Test + void terms_bucket_for_opensearchdate_type_uses_long_false() { + OpenSearchDateType dataType = OpenSearchDateType.of(STRING); + + assertEquals( + "{\n" + + " \"terms\" : {\n" + + " \"field\" : \"date\",\n" + + " \"missing_bucket\" : true,\n" + + " \"missing_order\" : \"first\",\n" + + " \"order\" : \"asc\"\n" + + " }\n" + + "}", + buildQuery(Arrays.asList(asc(named("date", ref("date", dataType)))))); + } + @ParameterizedTest(name = "{0}") @EnumSource( value = ExprCoreType.class, diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java index 90b982e017..bd2a9901ed 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java @@ -1772,9 +1772,9 @@ void cast_to_date_in_filter() { + " }\n" + " }\n" + "}"; - assertJsonEquals( json, buildQuery(DSL.equal(ref("date_value", DATE), DSL.castDate(literal("2021-11-08"))))); + assertJsonEquals( json, buildQuery( @@ -1821,7 +1821,7 @@ void cast_to_timestamp_in_filter() { "{\n" + " \"term\" : {\n" + " \"timestamp_value\" : {\n" - + " \"value\" : 1636390800000,\n" + + " \"value\" : \"2021-11-08 17:00:00\",\n" + " \"boost\" : 1.0\n" + " }\n" + " }\n" @@ -1847,7 +1847,7 @@ void cast_in_range_query() { "{\n" + " \"range\" : {\n" + " \"timestamp_value\" : {\n" - + " \"from\" : 1636390800000,\n" + + " \"from\" : \"2021-11-08 17:00:00\",\n" + " \"to\" : null," + " \"include_lower\" : false," + " \"include_upper\" : true,"