From d2b378d8ef3d6f2958ca13815df2e2cbe3c0f04f Mon Sep 17 00:00:00 2001 From: Andy Kwok Date: Tue, 19 Nov 2024 16:14:26 -0800 Subject: [PATCH 1/5] Test cases Signed-off-by: Andy Kwok --- .../executor/format/DateFieldFormatter.java | 32 +++++++++++++------ .../format/DateFieldFormatterTest.java | 18 +++++++++++ 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DateFieldFormatter.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DateFieldFormatter.java index dc239abd84..a80e24d347 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DateFieldFormatter.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DateFieldFormatter.java @@ -21,6 +21,7 @@ import org.apache.logging.log4j.Logger; import org.opensearch.sql.legacy.esdomain.LocalClusterState; import org.opensearch.sql.legacy.esdomain.mapping.FieldMappings; +import org.opensearch.sql.legacy.utils.StringUtils; /** Formatter to transform date fields into a consistent format for consumption by clients. */ public class DateFieldFormatter { @@ -83,7 +84,7 @@ public void applyJDBCDateFormat(Map rowSource) { Date date = parseDateString(formats, columnOriginalDate.toString()); if (date != null) { rowSource.put(columnName, DateFormat.getFormattedDate(date, FORMAT_JDBC)); - break; +// break; } else { LOG.warn("Could not parse date value; returning original value"); } @@ -152,15 +153,26 @@ private Date parseDateString(List formats, String columnOriginalDate) { switch (columnFormat) { case "date_optional_time": case "strict_date_optional_time": - parsedDate = - DateUtils.parseDate( - columnOriginalDate, - FORMAT_DOT_OPENSEARCH_DASHBOARDS_SAMPLE_DATA_LOGS_EXCEPTION, - FORMAT_DOT_OPENSEARCH_DASHBOARDS_SAMPLE_DATA_FLIGHTS_EXCEPTION, - FORMAT_DOT_OPENSEARCH_DASHBOARDS_SAMPLE_DATA_FLIGHTS_EXCEPTION_NO_TIME, - FORMAT_DOT_OPENSEARCH_DASHBOARDS_SAMPLE_DATA_ECOMMERCE_EXCEPTION, - FORMAT_DOT_DATE_AND_TIME, - FORMAT_DOT_DATE); + // It's possible to have date stored in second / millisecond form without explicit format hint. + // Parse it on a best-effort basis. + if (StringUtils.isNumeric(columnOriginalDate) ) { + long timestamp = Long.parseLong(columnOriginalDate); + if (timestamp > Integer.MAX_VALUE) { + parsedDate = new Date(timestamp); + } else { + parsedDate = new Date(timestamp*1000); + } + } else { + parsedDate = + DateUtils.parseDate( + columnOriginalDate, + FORMAT_DOT_OPENSEARCH_DASHBOARDS_SAMPLE_DATA_LOGS_EXCEPTION, + FORMAT_DOT_OPENSEARCH_DASHBOARDS_SAMPLE_DATA_FLIGHTS_EXCEPTION, + FORMAT_DOT_OPENSEARCH_DASHBOARDS_SAMPLE_DATA_FLIGHTS_EXCEPTION_NO_TIME, + FORMAT_DOT_OPENSEARCH_DASHBOARDS_SAMPLE_DATA_ECOMMERCE_EXCEPTION, + FORMAT_DOT_DATE_AND_TIME, + FORMAT_DOT_DATE); + } break; case "epoch_millis": parsedDate = new Date(Long.parseLong(columnOriginalDate)); diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/executor/format/DateFieldFormatterTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/executor/format/DateFieldFormatterTest.java index 1c2d1bae62..ce410a64e8 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/executor/format/DateFieldFormatterTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/executor/format/DateFieldFormatterTest.java @@ -609,6 +609,24 @@ public void testStrictDateOptionalTimeOrEpochMillsShouldPass() { verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); } + @Test + public void testDateInTimestampFormInSecondWithoutHint() { + String columnName = "date_field"; + String dateFormat = "date_optional_time"; + String originalDateValue = "1732057981"; + String expectedDateValue = "2024-11-19 23:13:01.000"; + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testDateInTimestampFormInMilliSecondWithoutHint() { + String columnName = "date_field"; + String dateFormat = "date_optional_time"; + String originalDateValue = "1732057981000"; + String expectedDateValue = "2024-11-19 23:13:01.000"; + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + private void verifyFormatting( String columnName, String dateFormatProperty, From 322d16ff78e1febf9d791d291a7c79a2ea924bda Mon Sep 17 00:00:00 2001 From: Andy Kwok Date: Tue, 19 Nov 2024 16:22:08 -0800 Subject: [PATCH 2/5] Minimise code changes Signed-off-by: Andy Kwok --- .../sql/legacy/executor/format/DateFieldFormatter.java | 1 - 1 file changed, 1 deletion(-) diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DateFieldFormatter.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DateFieldFormatter.java index a80e24d347..2177151518 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DateFieldFormatter.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DateFieldFormatter.java @@ -84,7 +84,6 @@ public void applyJDBCDateFormat(Map rowSource) { Date date = parseDateString(formats, columnOriginalDate.toString()); if (date != null) { rowSource.put(columnName, DateFormat.getFormattedDate(date, FORMAT_JDBC)); -// break; } else { LOG.warn("Could not parse date value; returning original value"); } From b85826c50764b34d0e9c1b5b8e07cd172ee687bb Mon Sep 17 00:00:00 2001 From: Andy Kwok Date: Fri, 22 Nov 2024 10:23:06 -0800 Subject: [PATCH 3/5] Format Signed-off-by: Andy Kwok --- .../executor/format/DateFieldFormatter.java | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DateFieldFormatter.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DateFieldFormatter.java index 2177151518..feba13d139 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DateFieldFormatter.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DateFieldFormatter.java @@ -152,25 +152,26 @@ private Date parseDateString(List formats, String columnOriginalDate) { switch (columnFormat) { case "date_optional_time": case "strict_date_optional_time": - // It's possible to have date stored in second / millisecond form without explicit format hint. + // It's possible to have date stored in second / millisecond form without explicit + // format hint. // Parse it on a best-effort basis. - if (StringUtils.isNumeric(columnOriginalDate) ) { + if (StringUtils.isNumeric(columnOriginalDate)) { long timestamp = Long.parseLong(columnOriginalDate); if (timestamp > Integer.MAX_VALUE) { parsedDate = new Date(timestamp); } else { - parsedDate = new Date(timestamp*1000); + parsedDate = new Date(timestamp * 1000); } } else { parsedDate = - DateUtils.parseDate( - columnOriginalDate, - FORMAT_DOT_OPENSEARCH_DASHBOARDS_SAMPLE_DATA_LOGS_EXCEPTION, - FORMAT_DOT_OPENSEARCH_DASHBOARDS_SAMPLE_DATA_FLIGHTS_EXCEPTION, - FORMAT_DOT_OPENSEARCH_DASHBOARDS_SAMPLE_DATA_FLIGHTS_EXCEPTION_NO_TIME, - FORMAT_DOT_OPENSEARCH_DASHBOARDS_SAMPLE_DATA_ECOMMERCE_EXCEPTION, - FORMAT_DOT_DATE_AND_TIME, - FORMAT_DOT_DATE); + DateUtils.parseDate( + columnOriginalDate, + FORMAT_DOT_OPENSEARCH_DASHBOARDS_SAMPLE_DATA_LOGS_EXCEPTION, + FORMAT_DOT_OPENSEARCH_DASHBOARDS_SAMPLE_DATA_FLIGHTS_EXCEPTION, + FORMAT_DOT_OPENSEARCH_DASHBOARDS_SAMPLE_DATA_FLIGHTS_EXCEPTION_NO_TIME, + FORMAT_DOT_OPENSEARCH_DASHBOARDS_SAMPLE_DATA_ECOMMERCE_EXCEPTION, + FORMAT_DOT_DATE_AND_TIME, + FORMAT_DOT_DATE); } break; case "epoch_millis": From 27af9580bd7015831dcd390537458245c2561de7 Mon Sep 17 00:00:00 2001 From: Andy Kwok Date: Tue, 26 Nov 2024 17:06:15 -0800 Subject: [PATCH 4/5] Update integration test Signed-off-by: Andy Kwok --- .../src/test/java/org/opensearch/sql/legacy/CursorIT.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java index e4ba593844..565c40b121 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java @@ -280,7 +280,9 @@ public void testRegressionOnDateFormatChange() throws IOException { Arrays.asList( "2015-01-01 00:00:00.000", "2015-01-01 12:10:30.000", - "1585882955", // by existing design, this is not formatted in MySQL standard format + // Conversion will be applied when dateTime is stored on unix timestamp, + // https://github.com/opensearch-project/sql/pull/3160 + "2020-04-03 03:02:35.000", "2020-04-08 06:10:30.000"); assertThat(actualDateList, equalTo(expectedDateList)); From 3d805cb089db6511d19d2b8be1a063f08d53ad44 Mon Sep 17 00:00:00 2001 From: Andy Kwok Date: Tue, 26 Nov 2024 17:58:05 -0800 Subject: [PATCH 5/5] Update unit test Signed-off-by: Andy Kwok --- .../sql/legacy/executor/format/DateFieldFormatterTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/executor/format/DateFieldFormatterTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/executor/format/DateFieldFormatterTest.java index ce410a64e8..7d43ea0383 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/executor/format/DateFieldFormatterTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/executor/format/DateFieldFormatterTest.java @@ -575,7 +575,7 @@ public void testIncorrectFormat() { String dateFormat = "date_optional_time"; String originalDateValue = "1581724085"; // Invalid format for date value; should return original value - String expectedDateValue = "1581724085"; + String expectedDateValue = "2020-02-14 23:48:05.000"; verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); }