Skip to content

Commit

Permalink
Feature/julian gregorian conversion fix (#100)
Browse files Browse the repository at this point in the history
  • Loading branch information
@aymeric-dispa
aymeric-dispa authored Oct 10, 2022
1 parent 4eb07c1 commit 84c6799
Show file tree
Hide file tree
Showing 7 changed files with 166 additions and 65 deletions.
19 changes: 10 additions & 9 deletions src/integrationTest/java/integration/tests/TimestampTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package integration.tests;

import static com.firebolt.jdbc.type.date.SqlDateUtil.ONE_DAY_MILLIS;
import static org.junit.jupiter.api.Assertions.assertEquals;

import java.sql.*;
Expand Down Expand Up @@ -27,9 +28,9 @@ void shouldGetTimeObjectsInEstTimezone() throws SQLException {
ZonedDateTime zonedDateTime = ZonedDateTime.of(1975, 1, 2, 4, 1, 1, 0,
TimeZone.getTimeZone("UTC").toZoneId());

Timestamp expectedTimestamp = Timestamp.valueOf(zonedDateTime.toLocalDateTime());
Time expectedTime = Time.valueOf(zonedDateTime.toLocalTime());
Date expectedDate = Date.valueOf(zonedDateTime.toLocalDate());
Timestamp expectedTimestamp = new Timestamp(zonedDateTime.toInstant().toEpochMilli());
Time expectedTime = new Time(zonedDateTime.toInstant().toEpochMilli());
Date expectedDate = new Date(zonedDateTime.toInstant().toEpochMilli());

assertEquals(expectedTimestamp, resultSet.getTimestamp(1));
assertEquals(expectedTimestamp, resultSet.getObject(1));
Expand All @@ -47,9 +48,9 @@ void shouldGetTimeObjectsInDefaultUTCTimezone() throws SQLException {
ZonedDateTime zonedDateTime = ZonedDateTime.of(1975, 1, 1, 23, 1, 1, 0,
TimeZone.getTimeZone("UTC").toZoneId());

Timestamp expectedTimestamp = Timestamp.valueOf(zonedDateTime.toLocalDateTime());
Time expectedTime = Time.valueOf(zonedDateTime.toLocalTime());
Date expectedDate = Date.valueOf(zonedDateTime.toLocalDate());
Timestamp expectedTimestamp = new Timestamp(zonedDateTime.toInstant().toEpochMilli());
Time expectedTime = new Time(zonedDateTime.toInstant().toEpochMilli());
Date expectedDate = new Date(zonedDateTime.toInstant().toEpochMilli());

assertEquals(expectedTimestamp, resultSet.getTimestamp(1));
assertEquals(expectedTimestamp, resultSet.getObject(1));
Expand All @@ -68,9 +69,9 @@ void shouldGetParsedTimeStampExtTimeObjects() throws SQLException {
ZonedDateTime zonedDateTime = ZonedDateTime.of(1111, 11, 11, 12, 0, 3, 0,
TimeZone.getTimeZone("UTC").toZoneId());

Timestamp expectedTimestamp = Timestamp.valueOf(zonedDateTime.toLocalDateTime());
Time expectedTime = Time.valueOf(zonedDateTime.toLocalTime());
Date expectedDate = Date.valueOf(zonedDateTime.toLocalDate());
Timestamp expectedTimestamp = new Timestamp(zonedDateTime.toInstant().toEpochMilli() + 7 * ONE_DAY_MILLIS);
Time expectedTime = new Time(zonedDateTime.toInstant().toEpochMilli() + 7 * ONE_DAY_MILLIS);
Date expectedDate = new Date(zonedDateTime.toInstant().toEpochMilli() + 7 * ONE_DAY_MILLIS);

assertEquals(expectedTimestamp, resultSet.getTimestamp(1));
assertEquals(expectedTimestamp, resultSet.getObject(1));
Expand Down
44 changes: 24 additions & 20 deletions src/main/java/com/firebolt/jdbc/resultset/FireboltColumn.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,7 @@

import static com.firebolt.jdbc.type.FireboltDataType.*;

import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.TimeZone;
import java.util.*;
import java.util.stream.Collectors;

import org.apache.commons.lang3.RegExUtils;
Expand All @@ -31,11 +28,13 @@ public final class FireboltColumn {
private final FireboltDataType dataType;
private final boolean nullable;
private final FireboltDataType arrayBaseDataType;
private final List<FireboltColumn> tupleBaseDateTypes;
private final List<FireboltColumn> tupleBaseDataTypes;
private final int arrayDepth;
private final int precision;
private final int scale;
private final TimeZone timeZone;
private static final Set<String> TIMEZONES = Arrays.stream(TimeZone.getAvailableIDs())
.collect(Collectors.toCollection(HashSet::new));

public static FireboltColumn of(String columnType, String columnName) {
log.debug("Creating column info for column: {} of type: {}", columnName, columnType);
Expand All @@ -49,7 +48,7 @@ public static FireboltColumn of(String columnType, String columnName) {
Optional<Pair<Optional<Integer>, Optional<Integer>>> scaleAndPrecisionPair;
FireboltDataType fireboltType;
if (typeInUpperCase.startsWith(FireboltDataType.TUPLE.getInternalName().toUpperCase())) {
tupleDataTypes = getTupleBaseDateTypes(typeInUpperCase, columnName);
tupleDataTypes = getTupleBaseDataTypes(typeInUpperCase, columnName);
}

while (typeInUpperCase.startsWith(FireboltDataType.ARRAY.getInternalName().toUpperCase(), currentIndex)) {
Expand All @@ -67,21 +66,20 @@ public static FireboltColumn of(String columnType, String columnName) {
arrayType = dataType;
if (arrayType == TUPLE) {
String tmp = columnType.substring(currentIndex, columnType.length() - arrayDepth);
tupleDataTypes = getTupleBaseDateTypes(tmp.toUpperCase(), columnName);
tupleDataTypes = getTupleBaseDataTypes(tmp.toUpperCase(), columnName);
}
fireboltType = FireboltDataType.ARRAY;
} else {
fireboltType = dataType;
}
String[] arguments = null;
if (!reachedEndOfTypeName(typeEndIndex, typeInUpperCase.length())
|| typeInUpperCase.startsWith("(", typeEndIndex)) {
if (!reachedEndOfTypeName(typeEndIndex, typeInUpperCase) || typeInUpperCase.startsWith("(", typeEndIndex)) {
arguments = splitArguments(typeInUpperCase, typeEndIndex);
scaleAndPrecisionPair = Optional.of(getsCaleAndPrecision(arguments, dataType));
} else {
scaleAndPrecisionPair = Optional.empty();
}
if (dataType.isTime() && arguments != null) {
if (dataType.isTime() && arguments != null && arguments.length != 0) {
timeZone = getTimeZoneFromArguments(arguments);
}

Expand All @@ -90,9 +88,8 @@ public static FireboltColumn of(String columnType, String columnName) {
.orElse(dataType.getDefaultScale()))
.precision(scaleAndPrecisionPair.map(Pair::getRight).filter(Optional::isPresent).map(Optional::get)
.orElse(dataType.getDefaultPrecision()))
.timeZone(timeZone)
.arrayBaseDataType(arrayType).dataType(fireboltType).nullable(isNullable).arrayDepth(arrayDepth)
.tupleBaseDateTypes(tupleDataTypes).build();
.timeZone(timeZone).arrayBaseDataType(arrayType).dataType(fireboltType).nullable(isNullable)
.arrayDepth(arrayDepth).tupleBaseDataTypes(tupleDataTypes).build();
}

private static TimeZone getTimeZoneFromArguments(String[] arguments) {
Expand All @@ -104,7 +101,13 @@ private static TimeZone getTimeZoneFromArguments(String[] arguments) {
timeZoneArgument = arguments[0];
}
if (timeZoneArgument != null) {
timeZone = TimeZone.getTimeZone(timeZoneArgument.replace("\\'", ""));
String id = timeZoneArgument.replace("\\'", "");
if (TIMEZONES.contains(id)) {
timeZone = TimeZone.getTimeZone(timeZoneArgument.replace("\\'", ""));
} else {
log.warn("Could not use the timezone returned by the server with the id {} as it is not supported.",
id);
}
}
return timeZone;
}
Expand All @@ -113,7 +116,7 @@ public static FireboltColumn of(String columnType) {
return of(columnType, null);
}

private static List<FireboltColumn> getTupleBaseDateTypes(String columnType, String columnName) {
private static List<FireboltColumn> getTupleBaseDataTypes(String columnType, String columnName) {
return Arrays.stream(getTupleTypes(columnType)).map(String::trim)
.map(type -> FireboltColumn.of(type, columnName)).collect(Collectors.toList());
}
Expand All @@ -127,8 +130,9 @@ private static String[] getTupleTypes(String columnType) {
// parenthesis
}

private static boolean reachedEndOfTypeName(int typeNameEndIndex, int type) {
return typeNameEndIndex == type;
private static boolean reachedEndOfTypeName(int typeNameEndIndex, String type) {
return typeNameEndIndex == type.length() || type.indexOf("(", typeNameEndIndex) < 0
|| type.indexOf(")", typeNameEndIndex) < 0;
}

private static int getTypeEndPosition(String type, int currentIndex) {
Expand All @@ -150,7 +154,7 @@ private static Pair<Optional<Integer>, Optional<Integer>> getsCaleAndPrecision(S
}
break;
case DECIMAL:
if (reachedEndOfTypeName(arguments.length, 2)) {
if (arguments.length == 2) {
precision = Integer.parseInt(arguments[0]);
scale = Integer.parseInt(arguments[1]);
}
Expand All @@ -170,7 +174,7 @@ public String getCompactTypeName() {
if (this.isArray()) {
return getArrayCompactTypeName();
} else if (this.isTuple()) {
return getTupleCompactTypeName(this.tupleBaseDateTypes);
return getTupleCompactTypeName(this.tupleBaseDataTypes);
} else {
Optional<String> params = getTypeArguments(columnType);
return dataType.getDisplayName() + params.orElse("");
Expand All @@ -186,7 +190,7 @@ private String getArrayCompactTypeName() {
if (this.getArrayBaseDataType() != TUPLE) {
type.append(this.getArrayBaseDataType().getDisplayName());
} else {
type.append(this.getTupleCompactTypeName(this.getTupleBaseDateTypes()));
type.append(this.getTupleCompactTypeName(this.getTupleBaseDataTypes()));
}
for (int i = 0; i < arrayDepth; i++) {
type.append(")");
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/com/firebolt/jdbc/type/array/SqlArrayUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ private static Object extractArrayFromOneDimensionalArray(String arrayContent, F

private static Object[] getArrayOfTuples(FireboltColumn fireboltColumn, List<String> tuples)
throws SQLException {
List<FireboltDataType> types = fireboltColumn.getTupleBaseDateTypes().stream().map(FireboltColumn::getDataType)
List<FireboltDataType> types = fireboltColumn.getTupleBaseDataTypes().stream().map(FireboltColumn::getDataType)
.collect(Collectors.toList());

List<Object[]> list = new ArrayList<>();
Expand Down
45 changes: 42 additions & 3 deletions src/main/java/com/firebolt/jdbc/type/date/SqlDateUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,20 +25,30 @@ public class SqlDateUtil {

private static final TimeZone DEFAULT_TZ = TimeZone.getDefault();

public static final long ONE_DAY_MILLIS = 86400000L;

// Number of milliseconds at the start of the introduction of the gregorian
// calendar(1582-10-05T00:00:00Z) from the epoch of 1970-01-01T00:00:00Z
private static final long GREGORIAN_START_DATE_IN_MILLIS = -12220156800000L;

private static final DateTimeFormatter dateFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd");

public static final Function<Date, String> transformFromDateToSQLStringFunction = value -> String.format("'%s'",
dateFormatter.format(value.toLocalDate()));
DateTimeFormatter dateTimeFormatter = new DateTimeFormatterBuilder().appendPattern("yyyy-MM-dd [HH:mm[:ss]]")
.appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true).toFormatter();
public static final BiFunction<String, TimeZone, Timestamp> transformToTimestampFunction = (value,
fromTimeZone) -> parse(value, fromTimeZone).map(t -> Timestamp.valueOf(t.toLocalDateTime())).orElse(null);
fromTimeZone) -> parse(value, fromTimeZone).map(t -> {
Timestamp ts = new Timestamp(getEpochMilli(t));
ts.setNanos(t.getNano());
return ts;
}).orElse(null);

public static final BiFunction<String, TimeZone, Date> transformToDateFunction = (value,
fromTimeZone) -> parse(value, fromTimeZone).map(t -> Date.valueOf(t.toLocalDate())).orElse(null);
fromTimeZone) -> parse(value, fromTimeZone).map(t -> new Date(getEpochMilli(t))).orElse(null);

public static final BiFunction<String, TimeZone, Time> transformToTimeFunction = (value,
fromTimeZone) -> parse(value, fromTimeZone).map(t -> Time.valueOf(t.toLocalTime())).orElse(null);
fromTimeZone) -> parse(value, fromTimeZone).map(t -> new Time(getEpochMilli(t))).orElse(null);
public static final Function<Timestamp, String> transformFromTimestampToSQLStringFunction = value -> String
.format("'%s'", dateTimeFormatter.format(value.toLocalDateTime()));

Expand All @@ -56,4 +66,33 @@ private static Optional<ZonedDateTime> parse(String value, @Nullable TimeZone fr
.atZone(zoneId).withZoneSameInstant(DEFAULT_TZ.toZoneId()));
}
}

private static long getEpochMilli(ZonedDateTime t) {
return t.toInstant().toEpochMilli() + calculateJulianToGregorianDiffMillis(t);
}

/**
* Calculates the difference in ms from Julian to Gregorian date for dates that
* are before the 5th of Oct 1582, which is before the introduction of the
* Gregorian Calendar
*
* @param zdt the date
* @return the difference in millis
*/
public static long calculateJulianToGregorianDiffMillis(ZonedDateTime zdt) {
if (zdt.toInstant().toEpochMilli() < GREGORIAN_START_DATE_IN_MILLIS) {
int year;
if (zdt.getMonthValue() == 1 || (zdt.getMonthValue() == 2 && zdt.getDayOfMonth() <= 28)) {
year = zdt.getYear() - 1;
} else {
year = zdt.getYear();
}
int hundredsOfYears = year / 100;
long daysDiff = hundredsOfYears - (hundredsOfYears / 4L) - 2L;
return daysDiff * ONE_DAY_MILLIS;
} else {
return 0;
}
}

}
40 changes: 36 additions & 4 deletions src/test/java/com/firebolt/jdbc/resultset/FireboltColumnTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,14 +26,15 @@ void shouldCreateColumDataForNullableString() {

@Test
void shouldCreateColumDataForArray() {
String type = "Array(Array(Nullable(String)))";
String type = "Array(Array(Nullable(DateTime64(4, \\'EST\\'))))";
String name = "name";
FireboltColumn column = FireboltColumn.of(type, name);
assertEquals(name, column.getColumnName());
assertEquals(type.toUpperCase(), column.getColumnType());
assertEquals(FireboltDataType.ARRAY, column.getDataType());
assertEquals(FireboltDataType.STRING, column.getArrayBaseDataType());
assertEquals("ARRAY(ARRAY(STRING))", column.getCompactTypeName());
assertEquals(FireboltDataType.DATE_TIME_64, column.getArrayBaseDataType());
assertEquals(TimeZone.getTimeZone("EST"), column.getTimeZone());
assertEquals("ARRAY(ARRAY(TIMESTAMP_EXT))", column.getCompactTypeName());
}

@Test
Expand Down Expand Up @@ -128,7 +129,7 @@ void shouldCreateColumDataForDateTime64() {

@Test
void shouldCreateColumDataForDateTime64WithoutTimeZone() {
String type = "DateTime64(6)";
String type = "Nullable(DateTime64(6))";
String name = "my_d";
FireboltColumn column = FireboltColumn.of(type, name);
assertEquals(name, column.getColumnName());
Expand All @@ -138,6 +139,16 @@ void shouldCreateColumDataForDateTime64WithoutTimeZone() {
assertEquals(25, column.getPrecision());
}

@Test
void shouldCreateColumDataForDateWithoutTimeZone() {
String type = "Nullable(Date)";
String name = "my_d";
FireboltColumn column = FireboltColumn.of(type, name);
assertEquals(name, column.getColumnName());
assertEquals(type.toUpperCase(), column.getColumnType());
assertNull(column.getTimeZone());
}

@Test
void shouldCreateColumDataForDateTimeWithTimeZone() {
String type = "DateTime(\\'EST\\')";
Expand All @@ -148,4 +159,25 @@ void shouldCreateColumDataForDateTimeWithTimeZone() {
assertEquals(FireboltDataType.DATE_TIME, column.getDataType());
assertEquals(TimeZone.getTimeZone("EST"), column.getTimeZone());
}
@Test
void shouldCreateColumDataForNullableDateTimeWithTimeZone() {
String type = "Nullable(DateTime(\\'EST\\'))";
String name = "my_d";
FireboltColumn column = FireboltColumn.of(type, name);
assertEquals(name, column.getColumnName());
assertEquals(type.toUpperCase(), column.getColumnType());
assertEquals(FireboltDataType.DATE_TIME, column.getDataType());
assertEquals(TimeZone.getTimeZone("EST"), column.getTimeZone());
}

@Test
void shouldCreateColumDataForDateTimeWithoutTimezoneWhenTheTimezoneIsInvalid() {
String type = "DateTime(\\'HelloTz\\')";
String name = "my_d";
FireboltColumn column = FireboltColumn.of(type, name);
assertEquals(name, column.getColumnName());
assertEquals(type.toUpperCase(), column.getColumnType());
assertEquals(FireboltDataType.DATE_TIME, column.getDataType());
assertNull(column.getTimeZone());
}
}
32 changes: 16 additions & 16 deletions src/test/java/com/firebolt/jdbc/resultset/FireboltResultSetTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -320,7 +320,8 @@ void shouldReturnBoolean() throws SQLException {

@Test
void shouldReturnTime() throws SQLException {
Time expectedTime = Time.valueOf(ZonedDateTime.of(2022, 5, 10, 13, 1, 2, 0, UTC_TZ.toZoneId()).toLocalTime());
Time expectedTime = new Time(
ZonedDateTime.of(2022, 5, 10, 13, 1, 2, 0, UTC_TZ.toZoneId()).toInstant().toEpochMilli());
inputStream = getInputStreamWithArray();
resultSet = new FireboltResultSet(inputStream, "array_test_table", "array_test_db", 65535);
resultSet.next();
Expand Down Expand Up @@ -459,11 +460,11 @@ void shouldGetTimeWithTimezoneFromCalendar() throws SQLException {
fireboltStatement, true);
resultSet.next();

Time firstExpectedTime = Time
.valueOf(ZonedDateTime.of(2022, 5, 10, 18, 1, 2, 0, UTC_TZ.toZoneId()).toLocalTime());
Time firstExpectedTime = new Time(
ZonedDateTime.of(2022, 5, 10, 18, 1, 2, 0, UTC_TZ.toZoneId()).toInstant().toEpochMilli());

Time secondExpectedTime = Time
.valueOf(ZonedDateTime.of(2022, 5, 10, 13, 1, 2, 0, UTC_TZ.toZoneId()).toLocalTime());
Time secondExpectedTime = new Time(
ZonedDateTime.of(2022, 5, 10, 13, 1, 2, 0, UTC_TZ.toZoneId()).toInstant().toEpochMilli());

assertEquals(firstExpectedTime, resultSet.getTime("a_timestamp", EST_CALENDAR));
assertEquals(secondExpectedTime, resultSet.getTime("a_timestamp", UTC_CALENDAR));
Expand Down Expand Up @@ -505,12 +506,11 @@ void shouldGetTimeObjectsWithTimeZoneFromResponse() throws SQLException {
resultSet.next();
ZonedDateTime zonedDateTime = ZonedDateTime.of(2022, 5, 10, 18, 1, 2, 0, UTC_TZ.toZoneId());

Timestamp expectedTimestamp = Timestamp.valueOf(zonedDateTime.toLocalDateTime());
Timestamp expectedTimestamp = new Timestamp(zonedDateTime.toInstant().toEpochMilli());

Timestamp.valueOf(zonedDateTime.toLocalDateTime());
Time expectedTime = Time.valueOf(zonedDateTime.toLocalTime());
Date expectedDate = Date.valueOf(
ZonedDateTime.of(2022, 5, 11, 4, 1, 2, 0, TimeZone.getTimeZone("UTC").toZoneId()).toLocalDate());
Time expectedTime = new Time(zonedDateTime.toInstant().toEpochMilli());
Date expectedDate = new Date(ZonedDateTime
.of(2022, 5, 11, 4, 1, 2, 0, TimeZone.getTimeZone("UTC").toZoneId()).toInstant().toEpochMilli());

// The timezone returned by the db is always used regardless of the timezone
// passed as an argument
Expand All @@ -534,12 +534,12 @@ void shouldGetDateWithTimezoneFromCalendar() throws SQLException {
resultSet = new FireboltResultSet(inputStream, "array_test_table", "array_test_db", 65535, false,
fireboltStatement, true);
resultSet.next();
Date firstExpectedDateFromEST = Date.valueOf(
ZonedDateTime.of(2022, 5, 10, 18, 1, 2, 0, TimeZone.getTimeZone("UTC").toZoneId()).toLocalDate());
Date secondExpectedDateFromEST = Date.valueOf(
ZonedDateTime.of(2022, 5, 11, 4, 1, 2, 0, TimeZone.getTimeZone("UTC").toZoneId()).toLocalDate());
Date secondExpectedDateFromUTC = Date.valueOf(
ZonedDateTime.of(2022, 5, 10, 23, 1, 2, 0, TimeZone.getTimeZone("UTC").toZoneId()).toLocalDate());
Date firstExpectedDateFromEST = new Date(ZonedDateTime
.of(2022, 5, 10, 18, 1, 2, 0, TimeZone.getTimeZone("UTC").toZoneId()).toInstant().toEpochMilli());
Date secondExpectedDateFromEST = new Date(ZonedDateTime
.of(2022, 5, 11, 4, 1, 2, 0, TimeZone.getTimeZone("UTC").toZoneId()).toInstant().toEpochMilli());
Date secondExpectedDateFromUTC = new Date(ZonedDateTime
.of(2022, 5, 10, 23, 1, 2, 0, TimeZone.getTimeZone("UTC").toZoneId()).toInstant().toEpochMilli());

assertEquals(firstExpectedDateFromEST, resultSet.getDate("a_timestamp", EST_CALENDAR));
resultSet.next();
Expand Down
Loading

0 comments on commit 84c6799

Please sign in to comment.