Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

added support for json type fields #15

Merged
merged 2 commits into from
Dec 4, 2024
Merged

added support for json type fields #15

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
df55a3f
added support for json type fields
yesoreyeram Dec 4, 2024
bfb8829
update
yesoreyeram Dec 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions lib/go/gframer/CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
# @grafana/infinity-gframer

## 1.1.0

- Added support for json type fields

## 1.0.0

- chore release
Expand Down
36 changes: 24 additions & 12 deletions lib/go/gframer/field.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,11 @@ func anyToNullableString(input []any, fieldName string, labels data.Labels, o []
currentValue := o[i]
switch cvt := currentValue.(type) {
case string:
field.Set(i, ToPointer(currentValue.(string)))
field.Set(i, pointer(currentValue.(string)))
case float64, float32, int, int16, int32, int64, uint, uint8, uint16, uint32, uint64:
field.Set(i, ToPointer(fmt.Sprintf("%v", currentValue)))
field.Set(i, pointer(fmt.Sprintf("%v", currentValue)))
case bool:
field.Set(i, ToPointer(fmt.Sprintf("%v", currentValue.(bool))))
field.Set(i, pointer(fmt.Sprintf("%v", currentValue.(bool))))
default:
noOperation(cvt)
field.Set(i, nil)
Expand All @@ -40,12 +40,12 @@ func anyToNullableBool(input []any, fieldName string, labels data.Labels, o []in
switch cvt := currentValue.(type) {
case bool:
if val, ok := (currentValue.(bool)); ok {
field.Set(i, ToPointer(val))
field.Set(i, pointer(val))
}
case string:
val, ok := (currentValue.(string))
if ok && strings.ToLower(val) == "true" {
field.Set(i, ToPointer(true))
field.Set(i, pointer(true))
}
default:
noOperation(cvt)
Expand All @@ -64,10 +64,22 @@ func anyToNullableNumber(input []any, fieldName string, labels data.Labels, o []
switch cvt := currentValue.(type) {
case string:
if item, err := strconv.ParseFloat(currentValue.(string), 64); err == nil {
field.Set(i, ToPointer(item))
field.Set(i, pointer(item))
}
case float64:
field.Set(i, ToPointer(currentValue.(float64)))
field.Set(i, pointer(currentValue.(float64)))
case float32:
field.Set(i, pointer(float64(currentValue.(float32))))
case int64:
field.Set(i, pointer(float64(currentValue.(int64))))
case int32:
field.Set(i, pointer(float64(currentValue.(int32))))
case int16:
field.Set(i, pointer(float64(currentValue.(int16))))
case int8:
field.Set(i, pointer(float64(currentValue.(int8))))
case int:
field.Set(i, pointer(float64(currentValue.(int))))
default:
noOperation(cvt)
field.Set(i, nil)
Expand All @@ -90,7 +102,7 @@ func anyToNullableTimestamp(input []any, fieldName string, labels data.Labels, o
format = timeFormat
}
if t, err := time.Parse(format, v); err == nil {
field.Set(i, ToPointer(t))
field.Set(i, pointer(t))
}
}
case string:
Expand All @@ -114,10 +126,10 @@ func anyToNullableTimestampEpoch(input []any, fieldName string, labels data.Labe
switch cvt := currentValue.(type) {
case string:
if item, err := strconv.ParseInt(currentValue.(string), 10, 64); err == nil && currentValue.(string) != "" {
field.Set(i, ToPointer(time.UnixMilli(item)))
field.Set(i, pointer(time.UnixMilli(item)))
}
case float64:
field.Set(i, ToPointer(time.UnixMilli(int64(currentValue.(float64)))))
field.Set(i, pointer(time.UnixMilli(int64(currentValue.(float64)))))
default:
noOperation(cvt)
field.Set(i, nil)
Expand All @@ -135,10 +147,10 @@ func anyToNullableTimestampEpochSecond(input []any, fieldName string, labels dat
switch cvt := currentValue.(type) {
case string:
if item, err := strconv.ParseInt(currentValue.(string), 10, 64); err == nil && currentValue.(string) != "" {
field.Set(i, ToPointer(time.Unix(item, 0)))
field.Set(i, pointer(time.Unix(item, 0)))
}
case float64:
field.Set(i, ToPointer(time.Unix(int64(currentValue.(float64)), 0)))
field.Set(i, pointer(time.Unix(int64(currentValue.(float64)), 0)))
default:
noOperation(cvt)
field.Set(i, nil)
Expand Down
92 changes: 80 additions & 12 deletions lib/go/gframer/gframer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package gframer
import (
"encoding/json"
"errors"
"slices"
"sort"
"time"

Expand All @@ -28,13 +29,17 @@ func noOperation(x interface{}) {}
func ToDataFrame(input interface{}, options FramerOptions) (frame *data.Frame, err error) {
switch x := input.(type) {
case nil, string, float64, float32, int64, int32, int16, int, bool:
return structToFrame(options.FrameName, map[string]interface{}{options.FrameName: input}, options.ExecutedQueryString)
frame, err = structToFrame(options.FrameName, map[string]interface{}{options.FrameName: input}, options.ExecutedQueryString)
case []interface{}:
return sliceToFrame(options.FrameName, input.([]interface{}), options)
frame, err = sliceToFrame(options.FrameName, input.([]interface{}), options)
default:
noOperation(x)
return structToFrame(options.FrameName, input, options.ExecutedQueryString)
frame, err = structToFrame(options.FrameName, input, options.ExecutedQueryString)
}
if err != nil {
return frame, err
}
return convertStringFieldToJsonField(frame, options)
}

func structToFrame(name string, input interface{}, executedQueryString string) (frame *data.Frame, err error) {
Expand All @@ -48,12 +53,12 @@ func structToFrame(name string, input interface{}, executedQueryString string) (
fields := map[string]*data.Field{}
for key, value := range in {
switch x := value.(type) {
case nil, string, float64, float32, int64, int32, int16, int, bool:
case nil, string, float64, float32, int64, int32, int16, int8, int, uint64, uint32, uint16, uint8, uint, bool, time.Time, json.RawMessage:
noOperation(x)
a, b := getFieldTypeAndValue(value)
field := data.NewFieldFromFieldType(a, 1)
field.Name = key
field.Set(0, ToPointer(b))
field.Set(0, pointer(b))
fields[key] = field
default:
fieldType, b := getFieldTypeAndValue(value)
Expand All @@ -63,7 +68,7 @@ func structToFrame(name string, input interface{}, executedQueryString string) (
field := data.NewFieldFromFieldType(fieldType, 1)
field.Name = key
if o, err := json.Marshal(b); err == nil {
field.Set(0, ToPointer(string(o)))
field.Set(0, pointer(string(o)))
fields[key] = field
}
}
Expand Down Expand Up @@ -97,15 +102,15 @@ func sliceToFrame(name string, input []interface{}, options FramerOptions) (fram
field := data.NewFieldFromFieldType(a, len(input))
field.Name = name
for idx, i := range input {
field.Set(idx, ToPointer(i))
field.Set(idx, pointer(i))
}
frame.Fields = append(frame.Fields, field)
case []interface{}:
field := data.NewFieldFromFieldType(data.FieldTypeNullableString, len(input))
field.Name = name
for idx, i := range input {
if o, err := json.Marshal(i); err == nil {
field.Set(idx, ToPointer(string(o)))
field.Set(idx, pointer(string(o)))
}
}
frame.Fields = append(frame.Fields, field)
Expand Down Expand Up @@ -146,7 +151,7 @@ func sliceToFrame(name string, input []interface{}, options FramerOptions) (fram
field.Name = k
for i := 0; i < len(input); i++ {
if o, err := json.Marshal(o[i]); err == nil {
field.Set(i, ToPointer(string(o)))
field.Set(i, pointer(string(o)))
}
}
frame.Fields = append(frame.Fields, field)
Expand Down Expand Up @@ -178,7 +183,8 @@ func sliceToFrame(name string, input []interface{}, options FramerOptions) (fram
field := data.NewFieldFromFieldType(fieldType, len(input))
field.Name = k
for i := 0; i < len(input); i++ {
field.Set(i, ToPointer(o[i]))
_, value := getFieldTypeAndValue(o[i])
field.Set(i, pointer(value))
}
frame.Fields = append(frame.Fields, field)
}
Expand All @@ -189,7 +195,7 @@ func sliceToFrame(name string, input []interface{}, options FramerOptions) (fram
field := data.NewFieldFromFieldType(fieldType, len(input))
field.Name = k
for i := 0; i < len(input); i++ {
field.Set(i, ToPointer(o[i]))
field.Set(i, pointer(o[i]))
}
frame.Fields = append(frame.Fields, field)
}
Expand Down Expand Up @@ -224,10 +230,26 @@ func getFieldTypeAndValue(value interface{}) (t data.FieldType, out interface{})
return data.FieldTypeNullableFloat64, float64(value.(int32))
case int16:
return data.FieldTypeNullableFloat64, float64(value.(int16))
case int8:
return data.FieldTypeNullableFloat64, float64(value.(int8))
case int:
return data.FieldTypeNullableFloat64, float64(value.(int))
case uint64:
return data.FieldTypeNullableFloat64, float64(value.(uint64))
case uint32:
return data.FieldTypeNullableFloat64, float64(value.(uint32))
case uint16:
return data.FieldTypeNullableFloat64, float64(value.(uint16))
case uint8:
return data.FieldTypeNullableFloat64, float64(value.(uint8))
case uint:
return data.FieldTypeNullableFloat64, float64(value.(uint))
case bool:
return data.FieldTypeNullableBool, value
case time.Time:
return data.FieldTypeNullableTime, value
case json.RawMessage:
return data.FieldTypeNullableJSON, value
case interface{}:
return data.FieldTypeJSON, value
default:
Expand Down Expand Up @@ -270,11 +292,53 @@ func sortedKeys(in interface{}) []string {
return []string{}
}

func ToPointer(value interface{}) interface{} {
func convertStringFieldToJsonField(frame *data.Frame, options FramerOptions) (*data.Frame, error) {
fieldRequireConversion := map[string]bool{}
for _, v := range slices.Concat(options.Columns, options.OverrideColumns) {
if v.Type == "json" {
fieldName := v.Selector
if v.Alias != "" {
fieldName = v.Alias
}
fieldRequireConversion[fieldName] = true
}
}
for i, f := range frame.Fields {
if fieldRequireConversion[f.Name] {
newField := data.NewFieldFromFieldType(data.FieldTypeNullableJSON, f.Len())
newField.Name = f.Name
newField.Config = f.Config
newField.Labels = f.Labels
for i := 0; i < f.Len(); i++ {
fieldValue := f.At(i)
if fieldValue == nil {
continue
}
fieldValueBytes, err := json.Marshal(fieldValue)
if err != nil {
continue
}
if string(fieldValueBytes) == `"null"` {
continue
}
fieldValueJSONRawMessage := json.RawMessage(fieldValueBytes)
newField.Set(i, pointer(fieldValueJSONRawMessage))
}
frame.Fields[i] = newField
}
}
return frame, nil
}

func pointer(value interface{}) interface{} {
if value == nil {
return nil
}
switch v := value.(type) {
case int:
return &v
case *int:
return value
case int8:
return &v
case *int8:
Expand Down Expand Up @@ -327,6 +391,10 @@ func ToPointer(value interface{}) interface{} {
return &v
case *time.Time:
return value
case json.RawMessage:
return &v
case *json.RawMessage:
return value
default:
return nil
}
Expand Down
39 changes: 39 additions & 0 deletions lib/go/gframer/gframer_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,3 +135,42 @@ func TestToDataFrameSlices(t *testing.T) {
}
}
}

func TestJsonFieldType(t *testing.T) {
t.Run("mixed json content without null values", func(t *testing.T) {
gotFrame, err := gframer.ToDataFrame([]any{
map[string]any{"num": int64(1), "str": "two", "json": []int{3}, "bool": false},
map[string]any{"num": int64(11), "str": "two-two", "json": map[string]any{"something": "else"}, "bool": true},
}, gframer.FramerOptions{Columns: []gframer.ColumnSelector{{Selector: "num", Type: "number"}, {Selector: "str", Type: "string"}, {Selector: "json", Type: "json"}, {Selector: "bool", Type: "boolean"}}})
require.Nil(t, err)
require.NotNil(t, gotFrame)
experimental.CheckGoldenJSONFrame(t, "testdata/jsonfield", strings.ReplaceAll(t.Name(), "TestJsonFieldType/", ""), gotFrame, true)
})
t.Run("mixed json content without null values and override columns", func(t *testing.T) {
gotFrame, err := gframer.ToDataFrame([]any{
map[string]any{"num": int64(1), "str": "two", "json": []int{3}, "bool": false},
map[string]any{"num": int64(11), "str": "two-two", "json": map[string]any{"something": "else"}, "bool": true},
}, gframer.FramerOptions{OverrideColumns: []gframer.ColumnSelector{{Selector: "num", Type: "json"}, {Selector: "str", Type: "json"}, {Selector: "json", Type: "json"}, {Selector: "bool", Type: "json"}}})
require.Nil(t, err)
require.NotNil(t, gotFrame)
experimental.CheckGoldenJSONFrame(t, "testdata/jsonfield", strings.ReplaceAll(t.Name(), "TestJsonFieldType/", ""), gotFrame, true)
})
t.Run("mixed json content with null values", func(t *testing.T) {
gotFrame, err := gframer.ToDataFrame([]any{
map[string]any{"num": int64(1), "str": "two", "json": []int{3}, "bool": false},
map[string]any{"num": nil, "str": nil, "json": nil, "bool": nil},
}, gframer.FramerOptions{Columns: []gframer.ColumnSelector{{Selector: "num", Type: "number"}, {Selector: "str", Type: "string"}, {Selector: "json", Type: "json"}, {Selector: "bool", Type: "boolean"}}})
require.Nil(t, err)
require.NotNil(t, gotFrame)
experimental.CheckGoldenJSONFrame(t, "testdata/jsonfield", strings.ReplaceAll(t.Name(), "TestJsonFieldType/", ""), gotFrame, true)
})
t.Run("mixed json content with null values as first", func(t *testing.T) {
gotFrame, err := gframer.ToDataFrame([]any{
map[string]any{"num": nil, "str": nil, "json": nil, "bool": nil},
map[string]any{"num": int64(1), "str": "two", "json": []int{3}, "bool": false},
}, gframer.FramerOptions{Columns: []gframer.ColumnSelector{{Selector: "num", Type: "number"}, {Selector: "str", Type: "string"}, {Selector: "json", Type: "json"}, {Selector: "bool", Type: "boolean"}}})
require.Nil(t, err)
require.NotNil(t, gotFrame)
experimental.CheckGoldenJSONFrame(t, "testdata/jsonfield", strings.ReplaceAll(t.Name(), "TestJsonFieldType/", ""), gotFrame, true)
})
}
2 changes: 1 addition & 1 deletion lib/go/gframer/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "@grafana/infinity-gframer",
"private": true,
"version": "1.0.0",
"version": "1.1.0",
"scripts": {
"tidy": "go mod tidy",
"test:backend": "go test -v ./..."
Expand Down
Loading
Loading