From 50f0febed3adedc429bebd76b005e85e107a38ed Mon Sep 17 00:00:00 2001 From: Stepan Burlakov Date: Thu, 2 Feb 2023 14:13:58 +0200 Subject: [PATCH] feat: Fir 15408 new output format stbu (#51) --- client.go | 2 +- rows.go | 104 +++++++++++++++++------------------------ rows_test.go | 128 ++++++++++++++++++++++++++++++++------------------- 3 files changed, 124 insertions(+), 110 deletions(-) diff --git a/client.go b/client.go index b5687ff..5480b5f 100644 --- a/client.go +++ b/client.go @@ -153,7 +153,7 @@ func (c *Client) GetEngineUrlByDatabase(ctx context.Context, databaseName string func (c *Client) Query(ctx context.Context, engineUrl, databaseName, query string, setStatements map[string]string) (*QueryResponse, error) { infolog.Printf("Query engine '%s' with '%s'", engineUrl, query) - params := map[string]string{"database": databaseName, "output_format": "JSONCompact"} + params := map[string]string{"database": databaseName, "output_format": "JSON_Compact"} for setKey, setValue := range setStatements { params[setKey] = setValue } diff --git a/rows.go b/rows.go index bd24328..2a2c370 100644 --- a/rows.go +++ b/rows.go @@ -10,25 +10,23 @@ import ( ) const ( - uint8Type = "UINT8" - uint16Type = "UINT16" - uint32Type = "UINT32" - uint64Type = "UINT64" - int8Type = "INT8" - int16Type = "INT16" - int32Type = "INT32" - int64Type = "INT64" - float32Type = "FLOAT32" - float64Type = "FLOAT64" - stringType = "STRING" - datetimeType = "DATETIME" - datetime64Type = "DATETIME64" - dateType = "DATE" - date32Type = "DATE32" - PGDate = "PGDATE" - TimestampNtz = "TIMESTAMPNTZ" - TimestampTz = "TIMESTAMPTZ" - boolean = "BOOLEAN" + intType = "int" + longType = "long" + floatType = "float" + doubleType = "double" + + textType = "text" + + dateType = "date" + dateExtType = "date_ext" + pgDateType = "pgdate" + + timestampType = "timestamp" + timestampExtType = "timestamp_ext" + timestampNtzType = "timestampntz" + timestampTzType = "timestamptz" + + booleanType = "boolean" ) type fireboltRows struct { @@ -93,18 +91,18 @@ func (f *fireboltRows) NextResultSet() error { // checkTypeValue checks that val type could be changed to columnType func checkTypeValue(columnType string, val interface{}) error { - switch strings.ToUpper(columnType) { - case uint8Type, int8Type, uint16Type, int16Type, uint32Type, int32Type, uint64Type, int64Type, float32Type, float64Type: + switch columnType { + case intType, longType, floatType, doubleType: if _, ok := val.(float64); !ok { return fmt.Errorf("expected to convert a value to float64, but couldn't: %v", val) } return nil - case stringType, datetimeType, dateType, date32Type, datetime64Type, PGDate, TimestampNtz, TimestampTz: + case textType, dateType, dateExtType, pgDateType, timestampType, timestampExtType, timestampNtzType, timestampTzType: if _, ok := val.(string); !ok { return fmt.Errorf("expected to convert a value to string, but couldn't: %v", val) } return nil - case boolean: + case booleanType: if _, ok := val.(bool); !ok { return fmt.Errorf("expected to convert a value to bool, but couldn't: %v", val) } @@ -130,17 +128,15 @@ func parseTimestampTz(value string) (driver.Value, error) { // parseDateTimeValue parses different date types func parseDateTimeValue(columnType string, value string) (driver.Value, error) { - switch strings.ToUpper(columnType) { - case datetimeType: + switch columnType { + case dateType, dateExtType, pgDateType: + return time.Parse("2006-01-02", value) + case timestampType: // Go doesn't use yyyy-mm-dd layout. Instead, it uses the value: Mon Jan 2 15:04:05 MST 2006 return time.Parse("2006-01-02 15:04:05", value) - case datetime64Type: - return time.Parse("2006-01-02 15:04:05.000000", value) - case dateType, date32Type, PGDate: - return time.Parse("2006-01-02", value) - case TimestampNtz: + case timestampExtType, timestampNtzType: return time.Parse("2006-01-02 15:04:05.000000", value) - case TimestampTz: + case timestampTzType: return parseTimestampTz(value) } return nil, fmt.Errorf("type not known: %s", columnType) @@ -152,33 +148,22 @@ func parseSingleValue(columnType string, val interface{}) (driver.Value, error) return nil, ConstructNestedError("error during value parsing", err) } - switch strings.ToUpper(columnType) { - case uint8Type: - return uint8(val.(float64)), nil - case int8Type: - return int8(val.(float64)), nil - case uint16Type: - return uint16(val.(float64)), nil - case int16Type: - return int16(val.(float64)), nil - case uint32Type: - return uint32(val.(float64)), nil - case int32Type: + switch columnType { + case intType: return int32(val.(float64)), nil - case uint64Type: - return uint64(val.(float64)), nil - case int64Type: + case longType: return int64(val.(float64)), nil - case float32Type: + case floatType: return float32(val.(float64)), nil - case float64Type: + case doubleType: return val.(float64), nil - case stringType: + case textType: return val.(string), nil - case datetime64Type, datetimeType, dateType, date32Type, PGDate, TimestampNtz, TimestampTz: + case dateType, dateExtType, pgDateType, timestampType, timestampExtType, timestampNtzType, timestampTzType: return parseDateTimeValue(columnType, val.(string)) - case boolean: + case booleanType: return val.(bool), nil + } return nil, fmt.Errorf("type not known: %s", columnType) @@ -188,11 +173,10 @@ func parseSingleValue(columnType string, val interface{}) (driver.Value, error) // uint8, uint32, uint64, int32, int64, float32, float64, string, Time or []driver.Value for arrays func parseValue(columnType string, val interface{}) (driver.Value, error) { const ( - nullablePrefix = "Nullable(" - arrayPrefix = "Array(" - dateTime64Prefix = "DateTime64(" - decimalPrefix = "Decimal(" - suffix = ")" + nullableSuffix = " null" + arrayPrefix = "array(" + decimalPrefix = "Decimal(" + suffix = ")" ) // No need to parse type if the value is nil @@ -208,12 +192,10 @@ func parseValue(columnType string, val interface{}) (driver.Value, error) { res[i], _ = parseValue(columnType[len(arrayPrefix):len(columnType)-len(suffix)], s.Index(i).Interface()) } return res, nil - } else if strings.HasPrefix(columnType, dateTime64Prefix) && strings.HasSuffix(columnType, suffix) { - return parseSingleValue("DateTime64", val) } else if strings.HasPrefix(columnType, decimalPrefix) && strings.HasSuffix(columnType, suffix) { - return parseSingleValue("Float64", val) - } else if strings.HasPrefix(columnType, nullablePrefix) && strings.HasSuffix(columnType, suffix) { - return parseSingleValue(columnType[len(nullablePrefix):len(columnType)-len(suffix)], val) + return parseSingleValue("double", val) + } else if strings.HasSuffix(columnType, nullableSuffix) { + return parseValue(columnType[0:len(columnType)-len(nullableSuffix)], val) } return parseSingleValue(columnType, val) diff --git a/rows_test.go b/rows_test.go index d9c63fa..a83debd 100644 --- a/rows_test.go +++ b/rows_test.go @@ -18,52 +18,62 @@ func assert(test_val interface{}, expected_val interface{}, t *testing.T, err st } func mockRows(isMultiStatement bool) driver.RowsNextResultSet { - resultJson := []string{"{" + - "\"query\":{\"query_id\":\"16FF2A0300ECA753\"}," + - "\"meta\":[" + - " {\"name\":\"int_col\",\"type\":\"Nullable(Int32)\"}," + - " {\"name\":\"bigint_col\",\"type\":\"Int64\"}," + - " {\"name\":\"float_col\",\"type\":\"Float32\"}," + - " {\"name\":\"double_col\",\"type\":\"Float64\"}," + - " {\"name\":\"text_col\",\"type\":\"String\"}," + - " {\"name\":\"date_col\",\"type\":\"Date\"}," + - " {\"name\":\"timestamp_col\",\"type\":\"DateTime\"}," + - " {\"name\":\"pgdate_col\",\"type\":\"PGDATE\"}," + - " {\"name\":\"timestampntz_col\",\"type\":\"TIMESTAMPNTZ\"}," + - " {\"name\":\"timestamptz_col\",\"type\":\"TIMESTAMPTZ\"}," + - " {\"name\":\"legacy_bool_col\",\"type\":\"UInt8\"}," + - " {\"name\":\"array_col\",\"type\":\"Array(Int32)\"}," + - " {\"name\":\"nested_array_col\",\"type\":\"Array(Array(String))\"}," + - " {\"name\":\"new_bool_col\",\"type\":\"Boolean\"}]," + - "\"data\":[" + - " [null,1,0.312321,123213.321321,\"text\", \"2080-12-31\",\"1989-04-15 01:02:03\",\"0002-01-01\",\"1989-04-15 01:02:03.123456\",\"1989-04-15 02:02:03.123456+00\",1,[1,2,3],[[]],true]," + - " [2,1,0.312321,123213.321321,\"text\",\"1970-01-01\",\"1970-01-01 00:00:00\",\"0001-01-01\",\"1989-04-15 01:02:03.123457\",\"1989-04-15 01:02:03.1234+05:30\",1,[1,2,3],[[]],true]," + - " [3,null,0.312321,123213.321321,\"text\",\"1970-01-01\",\"1970-01-01 00:00:00\",\"0001-01-01\",\"1989-04-15 01:02:03.123458\",\"1989-04-15 01:02:03+01\",1,[5,2,3,2],[[\"TEST\",\"TEST1\"],[\"TEST3\"]],false]," + - " [2,1,0.312321,123213.321321,\"text\",\"1970-01-01\",\"1970-01-01 00:00:00\",\"0001-01-01\",\"1989-04-15 01:02:03.123457\",\"1111-01-05 17:04:42.123456+05:53:28\",1,[1,2,3],[[]],false]," + - " [2,1,0.312321,123213.321321,\"text\",\"1970-01-01\",\"1970-01-01 00:00:00\",\"0001-01-01\",\"1989-04-15 01:02:03.123457\",\"1989-04-15 02:02:03.123456-01\",1,[1,2,3],[[]],null]" + - "]," + - "\"rows\":5," + - "\"statistics\":{" + - " \"elapsed\":0.001797702," + - " \"rows_read\":3," + - " \"bytes_read\":293," + - " \"time_before_execution\":0.001251613," + - " \"time_to_execute\":0.000544098," + - " \"scanned_bytes_cache\":2003," + - " \"scanned_bytes_storage\":0}}", "" + - "{" + - "\"query\":{\"query_id\":\"16FF2A0300ECA753\"}," + - "\"meta\":[{\"name\":\"int_col\",\"type\":\"Nullable(Int32)\"}]," + - "\"data\":[[3], [null]]," + - "\"rows\":2," + - "\"statistics\":{" + - " \"elapsed\":0.001797702," + - " \"rows_read\":2," + - " \"bytes_read\":293," + - " \"time_before_execution\":0.001251613," + - " \"time_to_execute\":0.000544098," + - " \"scanned_bytes_cache\":2003," + - " \"scanned_bytes_storage\":0}}"} + resultJson := []string{ + `{ + "query":{"query_id":"16FF2A0300ECA753"}, + "meta":[ + {"name":"int_col","type":"int null"}, + {"name":"bigint_col","type":"long"}, + {"name":"float_col","type":"float"}, + {"name":"double_col","type":"double"}, + {"name":"text_col","type":"text"}, + {"name":"date_col","type":"date"}, + {"name":"timestamp_col","type":"timestamp"}, + {"name":"pgdate_col","type":"pgdate"}, + {"name":"timestampntz_col","type":"timestampntz"}, + {"name":"timestamptz_col","type":"timestamptz"}, + {"name":"legacy_bool_col","type":"int"}, + {"name":"array_col","type":"array(int)"}, + {"name":"nested_array_col","type":"array(array(text))"}, + {"name":"new_bool_col","type":"boolean"}, + {"name":"decimal_col","type":"Decimal(38, 30) null"}, + {"name":"decimal_array_col","type":"array(Decimal(38, 30) null)"} + ], + "data":[ + [null,1,0.312321,123213.321321,"text", "2080-12-31","1989-04-15 01:02:03","0002-01-01","1989-04-15 01:02:03.123456","1989-04-15 02:02:03.123456+00",1,[1,2,3],[[]],true, 123.12345678, [123.12345678]], + [2,1,0.312321,123213.321321,"text","1970-01-01","1970-01-01 00:00:00","0001-01-01","1989-04-15 01:02:03.123457","1989-04-15 01:02:03.1234+05:30",1,[1,2,3],[[]],true, -123.12345678, [-123.12345678, 0.0]], + [3,null,0.312321,123213.321321,"text","1970-01-01","1970-01-01 00:00:00","0001-01-01","1989-04-15 01:02:03.123458","1989-04-15 01:02:03+01",1,[5,2,3,2],[["TEST","TEST1"],["TEST3"]],false, 0.0, [0.0]], + [2,1,0.312321,123213.321321,"text","1970-01-01","1970-01-01 00:00:00","0001-01-01","1989-04-15 01:02:03.123457","1111-01-05 17:04:42.123456+05:53:28",1,[1,2,3],[[]],false, 123456781234567812345678.123456781234567812345678, [123456781234567812345678.12345678123456781234567812345678]], + [2,1,0.312321,123213.321321,"text","1970-01-01","1970-01-01 00:00:00","0001-01-01","1989-04-15 01:02:03.123457","1989-04-15 02:02:03.123456-01",1,[1,2,3],[[]],null, null, [null]] + ], + "rows":5, + "statistics":{ + "elapsed":0.001797702, + "rows_read":3, + "bytes_read":293, + "time_before_execution":0.001251613, + "time_to_execute":0.000544098, + "scanned_bytes_cache":2003, + "scanned_bytes_storage":0 + } + }`, + + `{ + "query":{"query_id":"16FF2A0300ECA753"}, + "meta":[{"name":"int_col","type":"int null"}], + "data":[[3], [null]], + "rows":2, + "statistics":{ + "elapsed":0.001797702, + "rows_read":2, + "bytes_read":293, + "time_before_execution":0.001251613, + "time_to_execute":0.000544098, + "scanned_bytes_cache":2003, + "scanned_bytes_storage":0 + } + }`, + } var responses []QueryResponse for i := 0; i < 2; i += 1 { @@ -85,7 +95,7 @@ func mockRows(isMultiStatement bool) driver.RowsNextResultSet { func TestRowsColumns(t *testing.T) { rows := mockRows(false) - columnNames := []string{"int_col", "bigint_col", "float_col", "double_col", "text_col", "date_col", "timestamp_col", "pgdate_col", "timestampntz_col", "timestamptz_col", "legacy_bool_col", "array_col", "nested_array_col", "new_bool_col"} + columnNames := []string{"int_col", "bigint_col", "float_col", "double_col", "text_col", "date_col", "timestamp_col", "pgdate_col", "timestampntz_col", "timestamptz_col", "legacy_bool_col", "array_col", "nested_array_col", "new_bool_col", "decimal_col", "decimal_array_col"} if !reflect.DeepEqual(rows.Columns(), columnNames) { t.Errorf("column lists are not equal") } @@ -107,7 +117,7 @@ func TestRowsClose(t *testing.T) { // TestRowsNext check Next method func TestRowsNext(t *testing.T) { rows := mockRows(false) - var dest = make([]driver.Value, 14) + var dest = make([]driver.Value, 16) err := rows.Next(dest) loc, _ := time.LoadLocation("UTC") @@ -126,6 +136,10 @@ func TestRowsNext(t *testing.T) { t.Errorf("Results not equal for timestamptz Expected: %s Got %s", expected_date, tz_date_to_test.In(loc)) } assert(dest[13], true, t, "results not equal for boolean") + assert(dest[14], 123.12345678, t, "results not equal for decimal") + arr := dest[15].([]driver.Value) + assert(len(arr), 1, t, "invalid length of decimal array") + assert(arr[0], 123.12345678, t, "results not equal for decimal array") err = rows.Next(dest) assert(err, nil, t, "Next shouldn't return an error") @@ -137,6 +151,11 @@ func TestRowsNext(t *testing.T) { t.Errorf("Results not equal for timestamptz Expected: %s Got %s", expected_date, tz_date_to_test2.In(timezone)) } assert(dest[13], true, t, "results not equal for boolean") + assert(dest[14], -123.12345678, t, "results not equal for decimal") + arr = dest[15].([]driver.Value) + assert(len(arr), 2, t, "invalid length of decimal array") + assert(arr[0], -123.12345678, t, "first item not equal for decimal array") + assert(arr[1], 0.0, t, "second item not equal for decimal array") err = rows.Next(dest) assert(err, nil, t, "Next shouldn't return an error") @@ -147,6 +166,10 @@ func TestRowsNext(t *testing.T) { assert(dest[3], float64(123213.321321), t, "results not equal for float64") assert(dest[4], "text", t, "results not equal for string") assert(dest[13], false, t, "results not equal for boolean") + assert(dest[14], 0.0, t, "results not equal for decimal") + arr = dest[15].([]driver.Value) + assert(len(arr), 1, t, "invalid length of decimal array") + assert(arr[0], 0.0, t, "results not equal for decimal array") err = rows.Next(dest) assert(err, nil, t, "Next shouldn't return an error") @@ -156,6 +179,11 @@ func TestRowsNext(t *testing.T) { t.Errorf("Results not equal for timestamptz Expected: %s Got %s", expected_date, tz_date_to_test3.In(loc)) } assert(dest[13], false, t, "results not equal for boolean") + var long_double = 123456781234567812345678.12345678123456781234567812345678 + assert(dest[14], long_double, t, "results not equal for decimal") + arr = dest[15].([]driver.Value) + assert(len(arr), 1, t, "invalid length of decimal array") + assert(arr[0], long_double, t, "results not equal for decimal array") err = rows.Next(dest) assert(err, nil, t, "Next shouldn't return an error") @@ -165,6 +193,10 @@ func TestRowsNext(t *testing.T) { t.Errorf("Results not equal for timestamptz Expected: %s Got %s", expected_date, tz_date_to_test4.In(loc)) } assert(dest[13], nil, t, "results not equal for boolean") + assert(dest[14], nil, t, "results not equal for decimal") + arr = dest[15].([]driver.Value) + assert(len(arr), 1, t, "invalid length of decimal array") + assert(arr[0], nil, t, "results not equal for decimal array") assert(io.EOF, rows.Next(dest), t, "Next should return io.EOF if no data available anymore")