From 620edeb77dfd633b19be3b8d5b29c88f4fddbe11 Mon Sep 17 00:00:00 2001 From: Lantao Jin Date: Fri, 18 Oct 2024 21:10:39 +0800 Subject: [PATCH 1/4] Parse datetime string with specific time format Signed-off-by: Lantao Jin --- .../sql/planner/physical/collector/Rounding.java | 6 +++--- .../data/value/OpenSearchExprValueFactory.java | 11 ++++++++++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/collector/Rounding.java b/core/src/main/java/org/opensearch/sql/planner/physical/collector/Rounding.java index 82c8af52cd..7645213c67 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/collector/Rounding.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/collector/Rounding.java @@ -46,13 +46,13 @@ public static Rounding createRounding(SpanExpression span) { if (DOUBLE.isCompatible(type)) { return new DoubleRounding(interval); } - if (type.equals(TIMESTAMP)) { + if (type.equals(TIMESTAMP) || type.typeName().equalsIgnoreCase(TIMESTAMP.typeName())) { return new TimestampRounding(interval, span.getUnit().getName()); } - if (type.equals(DATE)) { + if (type.equals(DATE) || type.typeName().equalsIgnoreCase(DATE.typeName())) { return new DateRounding(interval, span.getUnit().getName()); } - if (type.equals(TIME)) { + if (type.equals(TIME) || type.typeName().equalsIgnoreCase(TIME.typeName())) { return new TimeRounding(interval, span.getUnit().getName()); } return new UnknownRounding(); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java index 417aaddaee..e70613278c 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java @@ -58,6 +58,7 @@ import org.opensearch.sql.data.model.ExprTimestampValue; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.opensearch.data.type.OpenSearchBinaryType; @@ -227,6 +228,14 @@ private Optional type(String field) { * @return Parsed value */ private static ExprValue parseDateTimeString(String value, OpenSearchDateType dataType) { + try { + String customFormat = dataType.getAllCustomFormatters().toString(); + OpenSearchDateType dateType = OpenSearchDateType.of(customFormat); + LocalDate parsedDate = dateType.getParsedDateTime(value).toLocalDate(); + return ExprValueUtils.dateValue(parsedDate); + } catch (Exception ignored) { + // nothing to do, try another old parser + } List formatters = dataType.getAllNamedFormatters(); formatters.addAll(dataType.getAllCustomFormatters()); ExprCoreType returnFormat = dataType.getExprCoreType(); @@ -295,7 +304,7 @@ private static ExprValue createOpenSearchDateType(Content value, ExprType type) } } else { // custom format - return parseDateTimeString(value.stringValue(), dt); + return parseDateTimeString(value.objectValue().toString(), dt); } } if (value.isString()) { From 70ee912a94cf21972269ce8a239ff582e8f62766 Mon Sep 17 00:00:00 2001 From: Lantao Jin Date: Thu, 24 Oct 2024 16:12:49 +0800 Subject: [PATCH 2/4] fix IT Signed-off-by: Lantao Jin --- .../org/opensearch/sql/sql/DateTimeFormatsIT.java | 15 +++++++++++---- .../data/value/OpenSearchExprValueFactory.java | 9 --------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java index 13c2eecd56..4b71280990 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java @@ -63,7 +63,7 @@ public void testCustomFormats() { String query = String.format( "SELECT custom_time, custom_timestamp, custom_date_or_date," - + "custom_date_or_custom_time, custom_time_parser_check FROM %s", + + "custom_date_or_custom_time, custom_time_parser_check, yyyy-MM-dd FROM %s", TEST_INDEX_DATE_FORMATS); JSONObject result = executeQuery(query); verifySchema( @@ -72,17 +72,24 @@ public void testCustomFormats() { schema("custom_timestamp", null, "timestamp"), schema("custom_date_or_date", null, "date"), schema("custom_date_or_custom_time", null, "timestamp"), - schema("custom_time_parser_check", null, "time")); + schema("custom_time_parser_check", null, "time"), + schema("yyyy-MM-dd", null, "date")); verifyDataRows( result, rows( - "09:07:42", "1984-04-12 09:07:42", "1984-04-12", "1961-04-12 00:00:00", "23:44:36.321"), + "09:07:42", + "1984-04-12 09:07:42", + "1984-04-12", + "1961-04-12 00:00:00", + "23:44:36.321", + "1984-04-12"), rows( "21:07:42", "1984-04-12 22:07:42", "1984-04-12", "1970-01-01 09:07:00", - "09:01:16.542")); + "09:01:16.542", + "1984-04-12")); } @Test diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java index e70613278c..2f93775290 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java @@ -58,7 +58,6 @@ import org.opensearch.sql.data.model.ExprTimestampValue; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.opensearch.data.type.OpenSearchBinaryType; @@ -228,14 +227,6 @@ private Optional type(String field) { * @return Parsed value */ private static ExprValue parseDateTimeString(String value, OpenSearchDateType dataType) { - try { - String customFormat = dataType.getAllCustomFormatters().toString(); - OpenSearchDateType dateType = OpenSearchDateType.of(customFormat); - LocalDate parsedDate = dateType.getParsedDateTime(value).toLocalDate(); - return ExprValueUtils.dateValue(parsedDate); - } catch (Exception ignored) { - // nothing to do, try another old parser - } List formatters = dataType.getAllNamedFormatters(); formatters.addAll(dataType.getAllCustomFormatters()); ExprCoreType returnFormat = dataType.getExprCoreType(); From 549bd1ac552c85f7cd3be59d51d9dd55a1ae1e98 Mon Sep 17 00:00:00 2001 From: Lantao Jin Date: Thu, 24 Oct 2024 17:02:54 +0800 Subject: [PATCH 3/4] correct IT Signed-off-by: Lantao Jin --- .../sql/ppl/DateTimeImplementationIT.java | 14 ++++++++++++++ .../org/opensearch/sql/sql/DateTimeFormatsIT.java | 15 ++++----------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeImplementationIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeImplementationIT.java index f9dc7d8027..ab5e2e2b66 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeImplementationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeImplementationIT.java @@ -6,8 +6,10 @@ package org.opensearch.sql.ppl; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_FORMATS; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; import static org.opensearch.sql.util.MatcherUtils.verifySchema; import static org.opensearch.sql.util.MatcherUtils.verifySome; @@ -20,6 +22,7 @@ public class DateTimeImplementationIT extends PPLIntegTestCase { @Override public void init() throws IOException { loadIndex(Index.DATE); + loadIndex(Index.DATE_FORMATS); } @Test @@ -176,4 +179,15 @@ public void nullDateTimeInvalidDateValueMonth() throws IOException { verifySchema(result, schema("f", null, "timestamp")); verifySome(result.getJSONArray("datarows"), rows(new Object[] {null})); } + + @Test + public void testSpanDatetimeWithCustomFormat() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | eval a = 1 | stats count() as cnt by span(yyyy-MM-dd, 1d) as span", + TEST_INDEX_DATE_FORMATS)); + verifySchema(result, schema("cnt", null, "integer"), schema("span", null, "date")); + verifyDataRows(result, rows(2, "1984-04-12")); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java index 4b71280990..13c2eecd56 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java @@ -63,7 +63,7 @@ public void testCustomFormats() { String query = String.format( "SELECT custom_time, custom_timestamp, custom_date_or_date," - + "custom_date_or_custom_time, custom_time_parser_check, yyyy-MM-dd FROM %s", + + "custom_date_or_custom_time, custom_time_parser_check FROM %s", TEST_INDEX_DATE_FORMATS); JSONObject result = executeQuery(query); verifySchema( @@ -72,24 +72,17 @@ public void testCustomFormats() { schema("custom_timestamp", null, "timestamp"), schema("custom_date_or_date", null, "date"), schema("custom_date_or_custom_time", null, "timestamp"), - schema("custom_time_parser_check", null, "time"), - schema("yyyy-MM-dd", null, "date")); + schema("custom_time_parser_check", null, "time")); verifyDataRows( result, rows( - "09:07:42", - "1984-04-12 09:07:42", - "1984-04-12", - "1961-04-12 00:00:00", - "23:44:36.321", - "1984-04-12"), + "09:07:42", "1984-04-12 09:07:42", "1984-04-12", "1961-04-12 00:00:00", "23:44:36.321"), rows( "21:07:42", "1984-04-12 22:07:42", "1984-04-12", "1970-01-01 09:07:00", - "09:01:16.542", - "1984-04-12")); + "09:01:16.542")); } @Test From df64ccd56d55446c62573e3686cd5e55851a8919 Mon Sep 17 00:00:00 2001 From: Lantao Jin Date: Fri, 25 Oct 2024 12:19:05 +0800 Subject: [PATCH 4/4] fix coverage Signed-off-by: Lantao Jin --- .../physical/collector/RoundingTest.java | 54 +++++++++++++++++++ .../sql/ppl/DateTimeImplementationIT.java | 23 ++++++++ 2 files changed, 77 insertions(+) diff --git a/core/src/test/java/org/opensearch/sql/planner/physical/collector/RoundingTest.java b/core/src/test/java/org/opensearch/sql/planner/physical/collector/RoundingTest.java index 3a2601a874..4f6d51c901 100644 --- a/core/src/test/java/org/opensearch/sql/planner/physical/collector/RoundingTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/physical/collector/RoundingTest.java @@ -5,14 +5,18 @@ package org.opensearch.sql.planner.physical.collector; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.opensearch.sql.data.type.ExprCoreType.DATE; import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.data.type.ExprCoreType.TIME; +import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; import org.junit.jupiter.api.Test; import org.opensearch.sql.data.model.ExprTimeValue; import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.span.SpanExpression; @@ -26,6 +30,35 @@ void time_rounding_illegal_span() { ExpressionEvaluationException.class, () -> rounding.round(new ExprTimeValue("23:30:00"))); } + @Test + void datetime_rounding_span() { + SpanExpression dateSpan = DSL.span(DSL.ref("date", DATE), DSL.literal(1), "d"); + Rounding rounding = Rounding.createRounding(dateSpan); + assertInstanceOf(Rounding.DateRounding.class, rounding); + SpanExpression timeSpan = DSL.span(DSL.ref("time", TIME), DSL.literal(1), "h"); + rounding = Rounding.createRounding(timeSpan); + assertInstanceOf(Rounding.TimeRounding.class, rounding); + SpanExpression timestampSpan = DSL.span(DSL.ref("timestamp", TIMESTAMP), DSL.literal(1), "h"); + rounding = Rounding.createRounding(timestampSpan); + assertInstanceOf(Rounding.TimestampRounding.class, rounding); + } + + @Test + void datetime_rounding_non_core_type_span() { + SpanExpression dateSpan = + DSL.span(DSL.ref("date", new MockDateExprType()), DSL.literal(1), "d"); + Rounding rounding = Rounding.createRounding(dateSpan); + assertInstanceOf(Rounding.DateRounding.class, rounding); + SpanExpression timeSpan = + DSL.span(DSL.ref("time", new MockTimeExprType()), DSL.literal(1), "h"); + rounding = Rounding.createRounding(timeSpan); + assertInstanceOf(Rounding.TimeRounding.class, rounding); + SpanExpression timestampSpan = + DSL.span(DSL.ref("timestamp", new MockTimestampExprType()), DSL.literal(1), "h"); + rounding = Rounding.createRounding(timestampSpan); + assertInstanceOf(Rounding.TimestampRounding.class, rounding); + } + @Test void round_unknown_type() { SpanExpression span = DSL.span(DSL.ref("unknown", STRING), DSL.literal(1), ""); @@ -41,4 +74,25 @@ void resolve() { () -> Rounding.DateTimeUnit.resolve(illegalUnit), "Unable to resolve unit " + illegalUnit); } + + static class MockDateExprType implements ExprType { + @Override + public String typeName() { + return "DATE"; + } + } + + static class MockTimeExprType implements ExprType { + @Override + public String typeName() { + return "TIME"; + } + } + + static class MockTimestampExprType implements ExprType { + @Override + public String typeName() { + return "TIMESTAMP"; + } + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeImplementationIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeImplementationIT.java index ab5e2e2b66..e777a4f454 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeImplementationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeImplementationIT.java @@ -190,4 +190,27 @@ public void testSpanDatetimeWithCustomFormat() throws IOException { verifySchema(result, schema("cnt", null, "integer"), schema("span", null, "date")); verifyDataRows(result, rows(2, "1984-04-12")); } + + @Test + public void testSpanDatetimeWithEpochMillisFormat() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | eval a = 1 | stats count() as cnt by span(epoch_millis, 1d) as span", + TEST_INDEX_DATE_FORMATS)); + verifySchema(result, schema("cnt", null, "integer"), schema("span", null, "timestamp")); + verifyDataRows(result, rows(2, "1984-04-12 00:00:00")); + } + + @Test + public void testSpanDatetimeWithDisjunctiveDifferentFormats() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | eval a = 1 | stats count() as cnt by span(yyyy-MM-dd_OR_epoch_millis," + + " 1d) as span", + TEST_INDEX_DATE_FORMATS)); + verifySchema(result, schema("cnt", null, "integer"), schema("span", null, "timestamp")); + verifyDataRows(result, rows(2, "1984-04-12 00:00:00")); + } }