diff --git a/lib/go/gframer/CHANGELOG.md b/lib/go/gframer/CHANGELOG.md index 57986ef..44f4e4d 100644 --- a/lib/go/gframer/CHANGELOG.md +++ b/lib/go/gframer/CHANGELOG.md @@ -1,5 +1,9 @@ # @grafana/infinity-gframer +## 1.1.0 + +- Added support for json type fields + ## 1.0.0 - chore release diff --git a/lib/go/gframer/field.go b/lib/go/gframer/field.go index 571cef2..654030b 100644 --- a/lib/go/gframer/field.go +++ b/lib/go/gframer/field.go @@ -18,11 +18,11 @@ func anyToNullableString(input []any, fieldName string, labels data.Labels, o [] currentValue := o[i] switch cvt := currentValue.(type) { case string: - field.Set(i, ToPointer(currentValue.(string))) + field.Set(i, pointer(currentValue.(string))) case float64, float32, int, int16, int32, int64, uint, uint8, uint16, uint32, uint64: - field.Set(i, ToPointer(fmt.Sprintf("%v", currentValue))) + field.Set(i, pointer(fmt.Sprintf("%v", currentValue))) case bool: - field.Set(i, ToPointer(fmt.Sprintf("%v", currentValue.(bool)))) + field.Set(i, pointer(fmt.Sprintf("%v", currentValue.(bool)))) default: noOperation(cvt) field.Set(i, nil) @@ -40,12 +40,12 @@ func anyToNullableBool(input []any, fieldName string, labels data.Labels, o []in switch cvt := currentValue.(type) { case bool: if val, ok := (currentValue.(bool)); ok { - field.Set(i, ToPointer(val)) + field.Set(i, pointer(val)) } case string: val, ok := (currentValue.(string)) if ok && strings.ToLower(val) == "true" { - field.Set(i, ToPointer(true)) + field.Set(i, pointer(true)) } default: noOperation(cvt) @@ -64,10 +64,22 @@ func anyToNullableNumber(input []any, fieldName string, labels data.Labels, o [] switch cvt := currentValue.(type) { case string: if item, err := strconv.ParseFloat(currentValue.(string), 64); err == nil { - field.Set(i, ToPointer(item)) + field.Set(i, pointer(item)) } case float64: - field.Set(i, ToPointer(currentValue.(float64))) + field.Set(i, pointer(currentValue.(float64))) + case float32: + field.Set(i, pointer(float64(currentValue.(float32)))) + case int64: + field.Set(i, pointer(float64(currentValue.(int64)))) + case int32: + field.Set(i, pointer(float64(currentValue.(int32)))) + case int16: + field.Set(i, pointer(float64(currentValue.(int16)))) + case int8: + field.Set(i, pointer(float64(currentValue.(int8)))) + case int: + field.Set(i, pointer(float64(currentValue.(int)))) default: noOperation(cvt) field.Set(i, nil) @@ -90,7 +102,7 @@ func anyToNullableTimestamp(input []any, fieldName string, labels data.Labels, o format = timeFormat } if t, err := time.Parse(format, v); err == nil { - field.Set(i, ToPointer(t)) + field.Set(i, pointer(t)) } } case string: @@ -114,10 +126,10 @@ func anyToNullableTimestampEpoch(input []any, fieldName string, labels data.Labe switch cvt := currentValue.(type) { case string: if item, err := strconv.ParseInt(currentValue.(string), 10, 64); err == nil && currentValue.(string) != "" { - field.Set(i, ToPointer(time.UnixMilli(item))) + field.Set(i, pointer(time.UnixMilli(item))) } case float64: - field.Set(i, ToPointer(time.UnixMilli(int64(currentValue.(float64))))) + field.Set(i, pointer(time.UnixMilli(int64(currentValue.(float64))))) default: noOperation(cvt) field.Set(i, nil) @@ -135,10 +147,10 @@ func anyToNullableTimestampEpochSecond(input []any, fieldName string, labels dat switch cvt := currentValue.(type) { case string: if item, err := strconv.ParseInt(currentValue.(string), 10, 64); err == nil && currentValue.(string) != "" { - field.Set(i, ToPointer(time.Unix(item, 0))) + field.Set(i, pointer(time.Unix(item, 0))) } case float64: - field.Set(i, ToPointer(time.Unix(int64(currentValue.(float64)), 0))) + field.Set(i, pointer(time.Unix(int64(currentValue.(float64)), 0))) default: noOperation(cvt) field.Set(i, nil) diff --git a/lib/go/gframer/gframer.go b/lib/go/gframer/gframer.go index 5bb83f8..31e52d9 100644 --- a/lib/go/gframer/gframer.go +++ b/lib/go/gframer/gframer.go @@ -3,6 +3,7 @@ package gframer import ( "encoding/json" "errors" + "slices" "sort" "time" @@ -28,13 +29,17 @@ func noOperation(x interface{}) {} func ToDataFrame(input interface{}, options FramerOptions) (frame *data.Frame, err error) { switch x := input.(type) { case nil, string, float64, float32, int64, int32, int16, int, bool: - return structToFrame(options.FrameName, map[string]interface{}{options.FrameName: input}, options.ExecutedQueryString) + frame, err = structToFrame(options.FrameName, map[string]interface{}{options.FrameName: input}, options.ExecutedQueryString) case []interface{}: - return sliceToFrame(options.FrameName, input.([]interface{}), options) + frame, err = sliceToFrame(options.FrameName, input.([]interface{}), options) default: noOperation(x) - return structToFrame(options.FrameName, input, options.ExecutedQueryString) + frame, err = structToFrame(options.FrameName, input, options.ExecutedQueryString) } + if err != nil { + return frame, err + } + return convertStringFieldToJsonField(frame, options) } func structToFrame(name string, input interface{}, executedQueryString string) (frame *data.Frame, err error) { @@ -48,12 +53,12 @@ func structToFrame(name string, input interface{}, executedQueryString string) ( fields := map[string]*data.Field{} for key, value := range in { switch x := value.(type) { - case nil, string, float64, float32, int64, int32, int16, int, bool: + case nil, string, float64, float32, int64, int32, int16, int8, int, uint64, uint32, uint16, uint8, uint, bool, time.Time, json.RawMessage: noOperation(x) a, b := getFieldTypeAndValue(value) field := data.NewFieldFromFieldType(a, 1) field.Name = key - field.Set(0, ToPointer(b)) + field.Set(0, pointer(b)) fields[key] = field default: fieldType, b := getFieldTypeAndValue(value) @@ -63,7 +68,7 @@ func structToFrame(name string, input interface{}, executedQueryString string) ( field := data.NewFieldFromFieldType(fieldType, 1) field.Name = key if o, err := json.Marshal(b); err == nil { - field.Set(0, ToPointer(string(o))) + field.Set(0, pointer(string(o))) fields[key] = field } } @@ -97,7 +102,7 @@ func sliceToFrame(name string, input []interface{}, options FramerOptions) (fram field := data.NewFieldFromFieldType(a, len(input)) field.Name = name for idx, i := range input { - field.Set(idx, ToPointer(i)) + field.Set(idx, pointer(i)) } frame.Fields = append(frame.Fields, field) case []interface{}: @@ -105,7 +110,7 @@ func sliceToFrame(name string, input []interface{}, options FramerOptions) (fram field.Name = name for idx, i := range input { if o, err := json.Marshal(i); err == nil { - field.Set(idx, ToPointer(string(o))) + field.Set(idx, pointer(string(o))) } } frame.Fields = append(frame.Fields, field) @@ -146,7 +151,7 @@ func sliceToFrame(name string, input []interface{}, options FramerOptions) (fram field.Name = k for i := 0; i < len(input); i++ { if o, err := json.Marshal(o[i]); err == nil { - field.Set(i, ToPointer(string(o))) + field.Set(i, pointer(string(o))) } } frame.Fields = append(frame.Fields, field) @@ -178,7 +183,8 @@ func sliceToFrame(name string, input []interface{}, options FramerOptions) (fram field := data.NewFieldFromFieldType(fieldType, len(input)) field.Name = k for i := 0; i < len(input); i++ { - field.Set(i, ToPointer(o[i])) + _, value := getFieldTypeAndValue(o[i]) + field.Set(i, pointer(value)) } frame.Fields = append(frame.Fields, field) } @@ -189,7 +195,7 @@ func sliceToFrame(name string, input []interface{}, options FramerOptions) (fram field := data.NewFieldFromFieldType(fieldType, len(input)) field.Name = k for i := 0; i < len(input); i++ { - field.Set(i, ToPointer(o[i])) + field.Set(i, pointer(o[i])) } frame.Fields = append(frame.Fields, field) } @@ -224,10 +230,26 @@ func getFieldTypeAndValue(value interface{}) (t data.FieldType, out interface{}) return data.FieldTypeNullableFloat64, float64(value.(int32)) case int16: return data.FieldTypeNullableFloat64, float64(value.(int16)) + case int8: + return data.FieldTypeNullableFloat64, float64(value.(int8)) case int: return data.FieldTypeNullableFloat64, float64(value.(int)) + case uint64: + return data.FieldTypeNullableFloat64, float64(value.(uint64)) + case uint32: + return data.FieldTypeNullableFloat64, float64(value.(uint32)) + case uint16: + return data.FieldTypeNullableFloat64, float64(value.(uint16)) + case uint8: + return data.FieldTypeNullableFloat64, float64(value.(uint8)) + case uint: + return data.FieldTypeNullableFloat64, float64(value.(uint)) case bool: return data.FieldTypeNullableBool, value + case time.Time: + return data.FieldTypeNullableTime, value + case json.RawMessage: + return data.FieldTypeNullableJSON, value case interface{}: return data.FieldTypeJSON, value default: @@ -270,11 +292,53 @@ func sortedKeys(in interface{}) []string { return []string{} } -func ToPointer(value interface{}) interface{} { +func convertStringFieldToJsonField(frame *data.Frame, options FramerOptions) (*data.Frame, error) { + fieldRequireConversion := map[string]bool{} + for _, v := range slices.Concat(options.Columns, options.OverrideColumns) { + if v.Type == "json" { + fieldName := v.Selector + if v.Alias != "" { + fieldName = v.Alias + } + fieldRequireConversion[fieldName] = true + } + } + for i, f := range frame.Fields { + if fieldRequireConversion[f.Name] { + newField := data.NewFieldFromFieldType(data.FieldTypeNullableJSON, f.Len()) + newField.Name = f.Name + newField.Config = f.Config + newField.Labels = f.Labels + for i := 0; i < f.Len(); i++ { + fieldValue := f.At(i) + if fieldValue == nil { + continue + } + fieldValueBytes, err := json.Marshal(fieldValue) + if err != nil { + continue + } + if string(fieldValueBytes) == `"null"` { + continue + } + fieldValueJSONRawMessage := json.RawMessage(fieldValueBytes) + newField.Set(i, pointer(fieldValueJSONRawMessage)) + } + frame.Fields[i] = newField + } + } + return frame, nil +} + +func pointer(value interface{}) interface{} { if value == nil { return nil } switch v := value.(type) { + case int: + return &v + case *int: + return value case int8: return &v case *int8: @@ -327,6 +391,10 @@ func ToPointer(value interface{}) interface{} { return &v case *time.Time: return value + case json.RawMessage: + return &v + case *json.RawMessage: + return value default: return nil } diff --git a/lib/go/gframer/gframer_test.go b/lib/go/gframer/gframer_test.go index 2d7daf3..09dc9d4 100644 --- a/lib/go/gframer/gframer_test.go +++ b/lib/go/gframer/gframer_test.go @@ -135,3 +135,42 @@ func TestToDataFrameSlices(t *testing.T) { } } } + +func TestJsonFieldType(t *testing.T) { + t.Run("mixed json content without null values", func(t *testing.T) { + gotFrame, err := gframer.ToDataFrame([]any{ + map[string]any{"num": int64(1), "str": "two", "json": []int{3}, "bool": false}, + map[string]any{"num": int64(11), "str": "two-two", "json": map[string]any{"something": "else"}, "bool": true}, + }, gframer.FramerOptions{Columns: []gframer.ColumnSelector{{Selector: "num", Type: "number"}, {Selector: "str", Type: "string"}, {Selector: "json", Type: "json"}, {Selector: "bool", Type: "boolean"}}}) + require.Nil(t, err) + require.NotNil(t, gotFrame) + experimental.CheckGoldenJSONFrame(t, "testdata/jsonfield", strings.ReplaceAll(t.Name(), "TestJsonFieldType/", ""), gotFrame, true) + }) + t.Run("mixed json content without null values and override columns", func(t *testing.T) { + gotFrame, err := gframer.ToDataFrame([]any{ + map[string]any{"num": int64(1), "str": "two", "json": []int{3}, "bool": false}, + map[string]any{"num": int64(11), "str": "two-two", "json": map[string]any{"something": "else"}, "bool": true}, + }, gframer.FramerOptions{OverrideColumns: []gframer.ColumnSelector{{Selector: "num", Type: "json"}, {Selector: "str", Type: "json"}, {Selector: "json", Type: "json"}, {Selector: "bool", Type: "json"}}}) + require.Nil(t, err) + require.NotNil(t, gotFrame) + experimental.CheckGoldenJSONFrame(t, "testdata/jsonfield", strings.ReplaceAll(t.Name(), "TestJsonFieldType/", ""), gotFrame, true) + }) + t.Run("mixed json content with null values", func(t *testing.T) { + gotFrame, err := gframer.ToDataFrame([]any{ + map[string]any{"num": int64(1), "str": "two", "json": []int{3}, "bool": false}, + map[string]any{"num": nil, "str": nil, "json": nil, "bool": nil}, + }, gframer.FramerOptions{Columns: []gframer.ColumnSelector{{Selector: "num", Type: "number"}, {Selector: "str", Type: "string"}, {Selector: "json", Type: "json"}, {Selector: "bool", Type: "boolean"}}}) + require.Nil(t, err) + require.NotNil(t, gotFrame) + experimental.CheckGoldenJSONFrame(t, "testdata/jsonfield", strings.ReplaceAll(t.Name(), "TestJsonFieldType/", ""), gotFrame, true) + }) + t.Run("mixed json content with null values as first", func(t *testing.T) { + gotFrame, err := gframer.ToDataFrame([]any{ + map[string]any{"num": nil, "str": nil, "json": nil, "bool": nil}, + map[string]any{"num": int64(1), "str": "two", "json": []int{3}, "bool": false}, + }, gframer.FramerOptions{Columns: []gframer.ColumnSelector{{Selector: "num", Type: "number"}, {Selector: "str", Type: "string"}, {Selector: "json", Type: "json"}, {Selector: "bool", Type: "boolean"}}}) + require.Nil(t, err) + require.NotNil(t, gotFrame) + experimental.CheckGoldenJSONFrame(t, "testdata/jsonfield", strings.ReplaceAll(t.Name(), "TestJsonFieldType/", ""), gotFrame, true) + }) +} diff --git a/lib/go/gframer/package.json b/lib/go/gframer/package.json index 7c9ef49..9979606 100644 --- a/lib/go/gframer/package.json +++ b/lib/go/gframer/package.json @@ -1,7 +1,7 @@ { "name": "@grafana/infinity-gframer", "private": true, - "version": "1.0.0", + "version": "1.1.0", "scripts": { "tidy": "go mod tidy", "test:backend": "go test -v ./..." diff --git a/lib/go/gframer/testdata/jsonfield/mixed_json_content_with_null_values.jsonc b/lib/go/gframer/testdata/jsonfield/mixed_json_content_with_null_values.jsonc new file mode 100644 index 0000000..5ae668b --- /dev/null +++ b/lib/go/gframer/testdata/jsonfield/mixed_json_content_with_null_values.jsonc @@ -0,0 +1,79 @@ +// 🌟 This was machine generated. Do not edit. 🌟 +// +// Frame[0] +// Name: +// Dimensions: 4 Fields by 2 Rows +// +---------------+--------------------------+------------------+-----------------+ +// | Name: bool | Name: json | Name: num | Name: str | +// | Labels: | Labels: | Labels: | Labels: | +// | Type: []*bool | Type: []*json.RawMessage | Type: []*float64 | Type: []*string | +// +---------------+--------------------------+------------------+-----------------+ +// | false | "[3]" | 1 | two | +// | null | null | null | null | +// +---------------+--------------------------+------------------+-----------------+ +// +// +// 🌟 This was machine generated. Do not edit. 🌟 +{ + "status": 200, + "frames": [ + { + "schema": { + "fields": [ + { + "name": "bool", + "type": "boolean", + "typeInfo": { + "frame": "bool", + "nullable": true + } + }, + { + "name": "json", + "type": "other", + "typeInfo": { + "frame": "json.RawMessage", + "nullable": true + } + }, + { + "name": "num", + "type": "number", + "typeInfo": { + "frame": "float64", + "nullable": true + } + }, + { + "name": "str", + "type": "string", + "typeInfo": { + "frame": "string", + "nullable": true + } + } + ] + }, + "data": { + "values": [ + [ + false, + null + ], + [ + "[3]", + null + ], + [ + 1, + null + ], + [ + "two", + null + ] + ] + } + } + ] +} \ No newline at end of file diff --git a/lib/go/gframer/testdata/jsonfield/mixed_json_content_with_null_values_as_first.jsonc b/lib/go/gframer/testdata/jsonfield/mixed_json_content_with_null_values_as_first.jsonc new file mode 100644 index 0000000..6f2dd07 --- /dev/null +++ b/lib/go/gframer/testdata/jsonfield/mixed_json_content_with_null_values_as_first.jsonc @@ -0,0 +1,79 @@ +// 🌟 This was machine generated. Do not edit. 🌟 +// +// Frame[0] +// Name: +// Dimensions: 4 Fields by 2 Rows +// +---------------+--------------------------+------------------+-----------------+ +// | Name: bool | Name: json | Name: num | Name: str | +// | Labels: | Labels: | Labels: | Labels: | +// | Type: []*bool | Type: []*json.RawMessage | Type: []*float64 | Type: []*string | +// +---------------+--------------------------+------------------+-----------------+ +// | null | null | null | null | +// | false | "[3]" | 1 | two | +// +---------------+--------------------------+------------------+-----------------+ +// +// +// 🌟 This was machine generated. Do not edit. 🌟 +{ + "status": 200, + "frames": [ + { + "schema": { + "fields": [ + { + "name": "bool", + "type": "boolean", + "typeInfo": { + "frame": "bool", + "nullable": true + } + }, + { + "name": "json", + "type": "other", + "typeInfo": { + "frame": "json.RawMessage", + "nullable": true + } + }, + { + "name": "num", + "type": "number", + "typeInfo": { + "frame": "float64", + "nullable": true + } + }, + { + "name": "str", + "type": "string", + "typeInfo": { + "frame": "string", + "nullable": true + } + } + ] + }, + "data": { + "values": [ + [ + null, + false + ], + [ + null, + "[3]" + ], + [ + null, + 1 + ], + [ + null, + "two" + ] + ] + } + } + ] +} \ No newline at end of file diff --git a/lib/go/gframer/testdata/jsonfield/mixed_json_content_without_null_values.jsonc b/lib/go/gframer/testdata/jsonfield/mixed_json_content_without_null_values.jsonc new file mode 100644 index 0000000..743b7bc --- /dev/null +++ b/lib/go/gframer/testdata/jsonfield/mixed_json_content_without_null_values.jsonc @@ -0,0 +1,79 @@ +// 🌟 This was machine generated. Do not edit. 🌟 +// +// Frame[0] +// Name: +// Dimensions: 4 Fields by 2 Rows +// +---------------+----------------------------+------------------+-----------------+ +// | Name: bool | Name: json | Name: num | Name: str | +// | Labels: | Labels: | Labels: | Labels: | +// | Type: []*bool | Type: []*json.RawMessage | Type: []*float64 | Type: []*string | +// +---------------+----------------------------+------------------+-----------------+ +// | false | "[3]" | 1 | two | +// | true | "{\"something\":\"else\"}" | 11 | two-two | +// +---------------+----------------------------+------------------+-----------------+ +// +// +// 🌟 This was machine generated. Do not edit. 🌟 +{ + "status": 200, + "frames": [ + { + "schema": { + "fields": [ + { + "name": "bool", + "type": "boolean", + "typeInfo": { + "frame": "bool", + "nullable": true + } + }, + { + "name": "json", + "type": "other", + "typeInfo": { + "frame": "json.RawMessage", + "nullable": true + } + }, + { + "name": "num", + "type": "number", + "typeInfo": { + "frame": "float64", + "nullable": true + } + }, + { + "name": "str", + "type": "string", + "typeInfo": { + "frame": "string", + "nullable": true + } + } + ] + }, + "data": { + "values": [ + [ + false, + true + ], + [ + "[3]", + "{\"something\":\"else\"}" + ], + [ + 1, + 11 + ], + [ + "two", + "two-two" + ] + ] + } + } + ] +} \ No newline at end of file diff --git a/lib/go/gframer/testdata/jsonfield/mixed_json_content_without_null_values_and_override_columns.jsonc b/lib/go/gframer/testdata/jsonfield/mixed_json_content_without_null_values_and_override_columns.jsonc new file mode 100644 index 0000000..9257e4d --- /dev/null +++ b/lib/go/gframer/testdata/jsonfield/mixed_json_content_without_null_values_and_override_columns.jsonc @@ -0,0 +1,79 @@ +// 🌟 This was machine generated. Do not edit. 🌟 +// +// Frame[0] +// Name: +// Dimensions: 4 Fields by 2 Rows +// +--------------------------+----------------------------+--------------------------+--------------------------+ +// | Name: bool | Name: json | Name: num | Name: str | +// | Labels: | Labels: | Labels: | Labels: | +// | Type: []*json.RawMessage | Type: []*json.RawMessage | Type: []*json.RawMessage | Type: []*json.RawMessage | +// +--------------------------+----------------------------+--------------------------+--------------------------+ +// | false | "[3]" | 1 | "two" | +// | true | "{\"something\":\"else\"}" | 11 | "two-two" | +// +--------------------------+----------------------------+--------------------------+--------------------------+ +// +// +// 🌟 This was machine generated. Do not edit. 🌟 +{ + "status": 200, + "frames": [ + { + "schema": { + "fields": [ + { + "name": "bool", + "type": "other", + "typeInfo": { + "frame": "json.RawMessage", + "nullable": true + } + }, + { + "name": "json", + "type": "other", + "typeInfo": { + "frame": "json.RawMessage", + "nullable": true + } + }, + { + "name": "num", + "type": "other", + "typeInfo": { + "frame": "json.RawMessage", + "nullable": true + } + }, + { + "name": "str", + "type": "other", + "typeInfo": { + "frame": "json.RawMessage", + "nullable": true + } + } + ] + }, + "data": { + "values": [ + [ + false, + true + ], + [ + "[3]", + "{\"something\":\"else\"}" + ], + [ + 1, + 11 + ], + [ + "two", + "two-two" + ] + ] + } + } + ] +} \ No newline at end of file