Skip to content

Commit

Permalink
feat: Fir 15408 new output format stbu (#51)
Browse files Browse the repository at this point in the history
  • Loading branch information
@stepansergeevitch
stepansergeevitch authored Feb 2, 2023
1 parent ffffdd9 commit 50f0feb
Show file tree
Hide file tree
Showing 3 changed files with 124 additions and 110 deletions.
2 changes: 1 addition & 1 deletion client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ func (c *Client) GetEngineUrlByDatabase(ctx context.Context, databaseName string
func (c *Client) Query(ctx context.Context, engineUrl, databaseName, query string, setStatements map[string]string) (*QueryResponse, error) {
infolog.Printf("Query engine '%s' with '%s'", engineUrl, query)

params := map[string]string{"database": databaseName, "output_format": "JSONCompact"}
params := map[string]string{"database": databaseName, "output_format": "JSON_Compact"}
for setKey, setValue := range setStatements {
params[setKey] = setValue
}
Expand Down
104 changes: 43 additions & 61 deletions rows.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,25 +10,23 @@ import (
)

const (
uint8Type = "UINT8"
uint16Type = "UINT16"
uint32Type = "UINT32"
uint64Type = "UINT64"
int8Type = "INT8"
int16Type = "INT16"
int32Type = "INT32"
int64Type = "INT64"
float32Type = "FLOAT32"
float64Type = "FLOAT64"
stringType = "STRING"
datetimeType = "DATETIME"
datetime64Type = "DATETIME64"
dateType = "DATE"
date32Type = "DATE32"
PGDate = "PGDATE"
TimestampNtz = "TIMESTAMPNTZ"
TimestampTz = "TIMESTAMPTZ"
boolean = "BOOLEAN"
intType = "int"
longType = "long"
floatType = "float"
doubleType = "double"

textType = "text"

dateType = "date"
dateExtType = "date_ext"
pgDateType = "pgdate"

timestampType = "timestamp"
timestampExtType = "timestamp_ext"
timestampNtzType = "timestampntz"
timestampTzType = "timestamptz"

booleanType = "boolean"
)

type fireboltRows struct {
Expand Down Expand Up @@ -93,18 +91,18 @@ func (f *fireboltRows) NextResultSet() error {

// checkTypeValue checks that val type could be changed to columnType
func checkTypeValue(columnType string, val interface{}) error {
switch strings.ToUpper(columnType) {
case uint8Type, int8Type, uint16Type, int16Type, uint32Type, int32Type, uint64Type, int64Type, float32Type, float64Type:
switch columnType {
case intType, longType, floatType, doubleType:
if _, ok := val.(float64); !ok {
return fmt.Errorf("expected to convert a value to float64, but couldn't: %v", val)
}
return nil
case stringType, datetimeType, dateType, date32Type, datetime64Type, PGDate, TimestampNtz, TimestampTz:
case textType, dateType, dateExtType, pgDateType, timestampType, timestampExtType, timestampNtzType, timestampTzType:
if _, ok := val.(string); !ok {
return fmt.Errorf("expected to convert a value to string, but couldn't: %v", val)
}
return nil
case boolean:
case booleanType:
if _, ok := val.(bool); !ok {
return fmt.Errorf("expected to convert a value to bool, but couldn't: %v", val)
}
Expand All @@ -130,17 +128,15 @@ func parseTimestampTz(value string) (driver.Value, error) {

// parseDateTimeValue parses different date types
func parseDateTimeValue(columnType string, value string) (driver.Value, error) {
switch strings.ToUpper(columnType) {
case datetimeType:
switch columnType {
case dateType, dateExtType, pgDateType:
return time.Parse("2006-01-02", value)
case timestampType:
// Go doesn't use yyyy-mm-dd layout. Instead, it uses the value: Mon Jan 2 15:04:05 MST 2006
return time.Parse("2006-01-02 15:04:05", value)
case datetime64Type:
return time.Parse("2006-01-02 15:04:05.000000", value)
case dateType, date32Type, PGDate:
return time.Parse("2006-01-02", value)
case TimestampNtz:
case timestampExtType, timestampNtzType:
return time.Parse("2006-01-02 15:04:05.000000", value)
case TimestampTz:
case timestampTzType:
return parseTimestampTz(value)
}
return nil, fmt.Errorf("type not known: %s", columnType)
Expand All @@ -152,33 +148,22 @@ func parseSingleValue(columnType string, val interface{}) (driver.Value, error)
return nil, ConstructNestedError("error during value parsing", err)
}

switch strings.ToUpper(columnType) {
case uint8Type:
return uint8(val.(float64)), nil
case int8Type:
return int8(val.(float64)), nil
case uint16Type:
return uint16(val.(float64)), nil
case int16Type:
return int16(val.(float64)), nil
case uint32Type:
return uint32(val.(float64)), nil
case int32Type:
switch columnType {
case intType:
return int32(val.(float64)), nil
case uint64Type:
return uint64(val.(float64)), nil
case int64Type:
case longType:
return int64(val.(float64)), nil
case float32Type:
case floatType:
return float32(val.(float64)), nil
case float64Type:
case doubleType:
return val.(float64), nil
case stringType:
case textType:
return val.(string), nil
case datetime64Type, datetimeType, dateType, date32Type, PGDate, TimestampNtz, TimestampTz:
case dateType, dateExtType, pgDateType, timestampType, timestampExtType, timestampNtzType, timestampTzType:
return parseDateTimeValue(columnType, val.(string))
case boolean:
case booleanType:
return val.(bool), nil

}

return nil, fmt.Errorf("type not known: %s", columnType)
Expand All @@ -188,11 +173,10 @@ func parseSingleValue(columnType string, val interface{}) (driver.Value, error)
// uint8, uint32, uint64, int32, int64, float32, float64, string, Time or []driver.Value for arrays
func parseValue(columnType string, val interface{}) (driver.Value, error) {
const (
nullablePrefix = "Nullable("
arrayPrefix = "Array("
dateTime64Prefix = "DateTime64("
decimalPrefix = "Decimal("
suffix = ")"
nullableSuffix = " null"
arrayPrefix = "array("
decimalPrefix = "Decimal("
suffix = ")"
)

// No need to parse type if the value is nil
Expand All @@ -208,12 +192,10 @@ func parseValue(columnType string, val interface{}) (driver.Value, error) {
res[i], _ = parseValue(columnType[len(arrayPrefix):len(columnType)-len(suffix)], s.Index(i).Interface())
}
return res, nil
} else if strings.HasPrefix(columnType, dateTime64Prefix) && strings.HasSuffix(columnType, suffix) {
return parseSingleValue("DateTime64", val)
} else if strings.HasPrefix(columnType, decimalPrefix) && strings.HasSuffix(columnType, suffix) {
return parseSingleValue("Float64", val)
} else if strings.HasPrefix(columnType, nullablePrefix) && strings.HasSuffix(columnType, suffix) {
return parseSingleValue(columnType[len(nullablePrefix):len(columnType)-len(suffix)], val)
return parseSingleValue("double", val)
} else if strings.HasSuffix(columnType, nullableSuffix) {
return parseValue(columnType[0:len(columnType)-len(nullableSuffix)], val)
}

return parseSingleValue(columnType, val)
Expand Down
128 changes: 80 additions & 48 deletions rows_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,52 +18,62 @@ func assert(test_val interface{}, expected_val interface{}, t *testing.T, err st
}

func mockRows(isMultiStatement bool) driver.RowsNextResultSet {
resultJson := []string{"{" +
"\"query\":{\"query_id\":\"16FF2A0300ECA753\"}," +
"\"meta\":[" +
" {\"name\":\"int_col\",\"type\":\"Nullable(Int32)\"}," +
" {\"name\":\"bigint_col\",\"type\":\"Int64\"}," +
" {\"name\":\"float_col\",\"type\":\"Float32\"}," +
" {\"name\":\"double_col\",\"type\":\"Float64\"}," +
" {\"name\":\"text_col\",\"type\":\"String\"}," +
" {\"name\":\"date_col\",\"type\":\"Date\"}," +
" {\"name\":\"timestamp_col\",\"type\":\"DateTime\"}," +
" {\"name\":\"pgdate_col\",\"type\":\"PGDATE\"}," +
" {\"name\":\"timestampntz_col\",\"type\":\"TIMESTAMPNTZ\"}," +
" {\"name\":\"timestamptz_col\",\"type\":\"TIMESTAMPTZ\"}," +
" {\"name\":\"legacy_bool_col\",\"type\":\"UInt8\"}," +
" {\"name\":\"array_col\",\"type\":\"Array(Int32)\"}," +
" {\"name\":\"nested_array_col\",\"type\":\"Array(Array(String))\"}," +
" {\"name\":\"new_bool_col\",\"type\":\"Boolean\"}]," +
"\"data\":[" +
" [null,1,0.312321,123213.321321,\"text\", \"2080-12-31\",\"1989-04-15 01:02:03\",\"0002-01-01\",\"1989-04-15 01:02:03.123456\",\"1989-04-15 02:02:03.123456+00\",1,[1,2,3],[[]],true]," +
" [2,1,0.312321,123213.321321,\"text\",\"1970-01-01\",\"1970-01-01 00:00:00\",\"0001-01-01\",\"1989-04-15 01:02:03.123457\",\"1989-04-15 01:02:03.1234+05:30\",1,[1,2,3],[[]],true]," +
" [3,null,0.312321,123213.321321,\"text\",\"1970-01-01\",\"1970-01-01 00:00:00\",\"0001-01-01\",\"1989-04-15 01:02:03.123458\",\"1989-04-15 01:02:03+01\",1,[5,2,3,2],[[\"TEST\",\"TEST1\"],[\"TEST3\"]],false]," +
" [2,1,0.312321,123213.321321,\"text\",\"1970-01-01\",\"1970-01-01 00:00:00\",\"0001-01-01\",\"1989-04-15 01:02:03.123457\",\"1111-01-05 17:04:42.123456+05:53:28\",1,[1,2,3],[[]],false]," +
" [2,1,0.312321,123213.321321,\"text\",\"1970-01-01\",\"1970-01-01 00:00:00\",\"0001-01-01\",\"1989-04-15 01:02:03.123457\",\"1989-04-15 02:02:03.123456-01\",1,[1,2,3],[[]],null]" +
"]," +
"\"rows\":5," +
"\"statistics\":{" +
" \"elapsed\":0.001797702," +
" \"rows_read\":3," +
" \"bytes_read\":293," +
" \"time_before_execution\":0.001251613," +
" \"time_to_execute\":0.000544098," +
" \"scanned_bytes_cache\":2003," +
" \"scanned_bytes_storage\":0}}", "" +
"{" +
"\"query\":{\"query_id\":\"16FF2A0300ECA753\"}," +
"\"meta\":[{\"name\":\"int_col\",\"type\":\"Nullable(Int32)\"}]," +
"\"data\":[[3], [null]]," +
"\"rows\":2," +
"\"statistics\":{" +
" \"elapsed\":0.001797702," +
" \"rows_read\":2," +
" \"bytes_read\":293," +
" \"time_before_execution\":0.001251613," +
" \"time_to_execute\":0.000544098," +
" \"scanned_bytes_cache\":2003," +
" \"scanned_bytes_storage\":0}}"}
resultJson := []string{
`{
"query":{"query_id":"16FF2A0300ECA753"},
"meta":[
{"name":"int_col","type":"int null"},
{"name":"bigint_col","type":"long"},
{"name":"float_col","type":"float"},
{"name":"double_col","type":"double"},
{"name":"text_col","type":"text"},
{"name":"date_col","type":"date"},
{"name":"timestamp_col","type":"timestamp"},
{"name":"pgdate_col","type":"pgdate"},
{"name":"timestampntz_col","type":"timestampntz"},
{"name":"timestamptz_col","type":"timestamptz"},
{"name":"legacy_bool_col","type":"int"},
{"name":"array_col","type":"array(int)"},
{"name":"nested_array_col","type":"array(array(text))"},
{"name":"new_bool_col","type":"boolean"},
{"name":"decimal_col","type":"Decimal(38, 30) null"},
{"name":"decimal_array_col","type":"array(Decimal(38, 30) null)"}
],
"data":[
[null,1,0.312321,123213.321321,"text", "2080-12-31","1989-04-15 01:02:03","0002-01-01","1989-04-15 01:02:03.123456","1989-04-15 02:02:03.123456+00",1,[1,2,3],[[]],true, 123.12345678, [123.12345678]],
[2,1,0.312321,123213.321321,"text","1970-01-01","1970-01-01 00:00:00","0001-01-01","1989-04-15 01:02:03.123457","1989-04-15 01:02:03.1234+05:30",1,[1,2,3],[[]],true, -123.12345678, [-123.12345678, 0.0]],
[3,null,0.312321,123213.321321,"text","1970-01-01","1970-01-01 00:00:00","0001-01-01","1989-04-15 01:02:03.123458","1989-04-15 01:02:03+01",1,[5,2,3,2],[["TEST","TEST1"],["TEST3"]],false, 0.0, [0.0]],
[2,1,0.312321,123213.321321,"text","1970-01-01","1970-01-01 00:00:00","0001-01-01","1989-04-15 01:02:03.123457","1111-01-05 17:04:42.123456+05:53:28",1,[1,2,3],[[]],false, 123456781234567812345678.123456781234567812345678, [123456781234567812345678.12345678123456781234567812345678]],
[2,1,0.312321,123213.321321,"text","1970-01-01","1970-01-01 00:00:00","0001-01-01","1989-04-15 01:02:03.123457","1989-04-15 02:02:03.123456-01",1,[1,2,3],[[]],null, null, [null]]
],
"rows":5,
"statistics":{
"elapsed":0.001797702,
"rows_read":3,
"bytes_read":293,
"time_before_execution":0.001251613,
"time_to_execute":0.000544098,
"scanned_bytes_cache":2003,
"scanned_bytes_storage":0
}
}`,

`{
"query":{"query_id":"16FF2A0300ECA753"},
"meta":[{"name":"int_col","type":"int null"}],
"data":[[3], [null]],
"rows":2,
"statistics":{
"elapsed":0.001797702,
"rows_read":2,
"bytes_read":293,
"time_before_execution":0.001251613,
"time_to_execute":0.000544098,
"scanned_bytes_cache":2003,
"scanned_bytes_storage":0
}
}`,
}

var responses []QueryResponse
for i := 0; i < 2; i += 1 {
Expand All @@ -85,7 +95,7 @@ func mockRows(isMultiStatement bool) driver.RowsNextResultSet {
func TestRowsColumns(t *testing.T) {
rows := mockRows(false)

columnNames := []string{"int_col", "bigint_col", "float_col", "double_col", "text_col", "date_col", "timestamp_col", "pgdate_col", "timestampntz_col", "timestamptz_col", "legacy_bool_col", "array_col", "nested_array_col", "new_bool_col"}
columnNames := []string{"int_col", "bigint_col", "float_col", "double_col", "text_col", "date_col", "timestamp_col", "pgdate_col", "timestampntz_col", "timestamptz_col", "legacy_bool_col", "array_col", "nested_array_col", "new_bool_col", "decimal_col", "decimal_array_col"}
if !reflect.DeepEqual(rows.Columns(), columnNames) {
t.Errorf("column lists are not equal")
}
Expand All @@ -107,7 +117,7 @@ func TestRowsClose(t *testing.T) {
// TestRowsNext check Next method
func TestRowsNext(t *testing.T) {
rows := mockRows(false)
var dest = make([]driver.Value, 14)
var dest = make([]driver.Value, 16)
err := rows.Next(dest)
loc, _ := time.LoadLocation("UTC")

Expand All @@ -126,6 +136,10 @@ func TestRowsNext(t *testing.T) {
t.Errorf("Results not equal for timestamptz Expected: %s Got %s", expected_date, tz_date_to_test.In(loc))
}
assert(dest[13], true, t, "results not equal for boolean")
assert(dest[14], 123.12345678, t, "results not equal for decimal")
arr := dest[15].([]driver.Value)
assert(len(arr), 1, t, "invalid length of decimal array")
assert(arr[0], 123.12345678, t, "results not equal for decimal array")

err = rows.Next(dest)
assert(err, nil, t, "Next shouldn't return an error")
Expand All @@ -137,6 +151,11 @@ func TestRowsNext(t *testing.T) {
t.Errorf("Results not equal for timestamptz Expected: %s Got %s", expected_date, tz_date_to_test2.In(timezone))
}
assert(dest[13], true, t, "results not equal for boolean")
assert(dest[14], -123.12345678, t, "results not equal for decimal")
arr = dest[15].([]driver.Value)
assert(len(arr), 2, t, "invalid length of decimal array")
assert(arr[0], -123.12345678, t, "first item not equal for decimal array")
assert(arr[1], 0.0, t, "second item not equal for decimal array")

err = rows.Next(dest)
assert(err, nil, t, "Next shouldn't return an error")
Expand All @@ -147,6 +166,10 @@ func TestRowsNext(t *testing.T) {
assert(dest[3], float64(123213.321321), t, "results not equal for float64")
assert(dest[4], "text", t, "results not equal for string")
assert(dest[13], false, t, "results not equal for boolean")
assert(dest[14], 0.0, t, "results not equal for decimal")
arr = dest[15].([]driver.Value)
assert(len(arr), 1, t, "invalid length of decimal array")
assert(arr[0], 0.0, t, "results not equal for decimal array")

err = rows.Next(dest)
assert(err, nil, t, "Next shouldn't return an error")
Expand All @@ -156,6 +179,11 @@ func TestRowsNext(t *testing.T) {
t.Errorf("Results not equal for timestamptz Expected: %s Got %s", expected_date, tz_date_to_test3.In(loc))
}
assert(dest[13], false, t, "results not equal for boolean")
var long_double = 123456781234567812345678.12345678123456781234567812345678
assert(dest[14], long_double, t, "results not equal for decimal")
arr = dest[15].([]driver.Value)
assert(len(arr), 1, t, "invalid length of decimal array")
assert(arr[0], long_double, t, "results not equal for decimal array")

err = rows.Next(dest)
assert(err, nil, t, "Next shouldn't return an error")
Expand All @@ -165,6 +193,10 @@ func TestRowsNext(t *testing.T) {
t.Errorf("Results not equal for timestamptz Expected: %s Got %s", expected_date, tz_date_to_test4.In(loc))
}
assert(dest[13], nil, t, "results not equal for boolean")
assert(dest[14], nil, t, "results not equal for decimal")
arr = dest[15].([]driver.Value)
assert(len(arr), 1, t, "invalid length of decimal array")
assert(arr[0], nil, t, "results not equal for decimal array")

assert(io.EOF, rows.Next(dest), t, "Next should return io.EOF if no data available anymore")

Expand Down

0 comments on commit 50f0feb

Please sign in to comment.