From 46612fa3a01ff4479ae3aa509c6a827c90ac9c9c Mon Sep 17 00:00:00 2001 From: Lorenzo Mangani Date: Fri, 18 Oct 2024 19:49:17 +0200 Subject: [PATCH 1/6] Update helper.go --- utils/helper.go | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/utils/helper.go b/utils/helper.go index bb5f119..fea5d17 100644 --- a/utils/helper.go +++ b/utils/helper.go @@ -122,6 +122,52 @@ func rowsToJSON(rows *sql.Rows, elapsedTime time.Duration) (string, error) { return string(jsonData), nil } +// rowsToNDJSON converts the rows to NDJSON strings +func rowsToNDJSON(rows *sql.Rows) (string, error) { + defer rows.Close() + + columns, err := rows.Columns() + if err != nil { + return "", err + } + var ndjsonBuffer strings.Builder + for rows.Next() { + values := make([]interface{}, len(columns)) + for i := range columns { + values[i] = new(interface{}) + } + + err := rows.Scan(values...) + if err != nil { + return "", err + } + + rowData := make(map[string]interface{}) + for i, column := range columns { + // Convert the value to the appropriate Go type + switch v := (*(values[i].(*interface{}))).(type) { + case []byte: + rowData[column] = string(v) + default: + rowData[column] = v + } + } + + rowJSON, err := json.Marshal(rowData) + if err != nil { + return "", err + } + + ndjsonBuffer.WriteString(string(rowJSON) + "\n") + } + + if err = rows.Err(); err != nil { + return "", err + } + + return ndjsonBuffer.String(), nil +} + // rowsToTSV converts the rows to TSV string func rowsToTSV(rows *sql.Rows, cols bool) (string, error) { var result []string From 1ad7159dde11e0416c949e3c7e40cf3b823912d8 Mon Sep 17 00:00:00 2001 From: Lorenzo Mangani Date: Fri, 18 Oct 2024 19:50:29 +0200 Subject: [PATCH 2/6] JSONEachRow option --- utils/helper.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/utils/helper.go b/utils/helper.go index fea5d17..044de81 100644 --- a/utils/helper.go +++ b/utils/helper.go @@ -30,6 +30,12 @@ func ConversationOfRows(rows *sql.Rows, default_format string, duration time.Dur return "", err } return result, nil + case "JSONEachRow", "NDJSON": + result, err := rowsToNDJSON(rows, duration) + if err != nil { + return "", err + } + return result, nil case "CSVWithNames": result, err := rowsToCSV(rows, true) if err != nil { From 69817ba2b0739120594bb8e7fc79d3ba8a1a1a1c Mon Sep 17 00:00:00 2001 From: Lorenzo Mangani Date: Fri, 18 Oct 2024 19:52:45 +0200 Subject: [PATCH 3/6] Remove stats from NDJSON --- utils/helper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/helper.go b/utils/helper.go index 044de81..6dfe4cc 100644 --- a/utils/helper.go +++ b/utils/helper.go @@ -31,7 +31,7 @@ func ConversationOfRows(rows *sql.Rows, default_format string, duration time.Dur } return result, nil case "JSONEachRow", "NDJSON": - result, err := rowsToNDJSON(rows, duration) + result, err := rowsToNDJSON(rows) if err != nil { return "", err } From 71847c323e24a1e2f9a6680aa579daeae54cc77a Mon Sep 17 00:00:00 2001 From: Lorenzo Mangani Date: Fri, 18 Oct 2024 20:17:56 +0200 Subject: [PATCH 4/6] update ndjson --- utils/helper.go | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/utils/helper.go b/utils/helper.go index 6dfe4cc..a3fb202 100644 --- a/utils/helper.go +++ b/utils/helper.go @@ -136,43 +136,45 @@ func rowsToNDJSON(rows *sql.Rows) (string, error) { if err != nil { return "", err } - var ndjsonBuffer strings.Builder - for rows.Next() { - values := make([]interface{}, len(columns)) - for i := range columns { - values[i] = new(interface{}) - } - err := rows.Scan(values...) + var buffer bytes.Buffer + values := make([]interface{}, len(columns)) + scanArgs := make([]interface{}, len(columns)) + for i := range values { + scanArgs[i] = &values[i] + } + + for rows.Next() { + err := rows.Scan(scanArgs...) if err != nil { return "", err } - rowData := make(map[string]interface{}) - for i, column := range columns { - // Convert the value to the appropriate Go type - switch v := (*(values[i].(*interface{}))).(type) { - case []byte: - rowData[column] = string(v) - default: - rowData[column] = v + rowMap := make(map[string]interface{}) + for i, col := range columns { + val := values[i] + b, ok := val.([]byte) + if ok { + rowMap[col] = string(b) + } else { + rowMap[col] = val } } - rowJSON, err := json.Marshal(rowData) + jsonData, err := json.Marshal(rowMap) if err != nil { return "", err } - ndjsonBuffer.WriteString(string(rowJSON) + "\n") + buffer.Write(jsonData) + buffer.WriteByte('\n') } if err = rows.Err(); err != nil { return "", err } - return ndjsonBuffer.String(), nil -} + return buffer.String(), nil // rowsToTSV converts the rows to TSV string func rowsToTSV(rows *sql.Rows, cols bool) (string, error) { From b7e611be63668df086028947cfda459eac5b076d Mon Sep 17 00:00:00 2001 From: Lorenzo Mangani Date: Fri, 18 Oct 2024 20:18:09 +0200 Subject: [PATCH 5/6] Update helper.go --- utils/helper.go | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/helper.go b/utils/helper.go index a3fb202..9fcc691 100644 --- a/utils/helper.go +++ b/utils/helper.go @@ -4,6 +4,7 @@ import ( "database/sql" "encoding/json" "fmt" + "bytes" "quackpipe/model" "regexp" "strings" From e6ff18fe90ad1c0e88bd45093235abb397d66109 Mon Sep 17 00:00:00 2001 From: Lorenzo Mangani Date: Fri, 18 Oct 2024 20:21:01 +0200 Subject: [PATCH 6/6] Manual pain, no gain --- utils/helper.go | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/helper.go b/utils/helper.go index 9fcc691..35ee240 100644 --- a/utils/helper.go +++ b/utils/helper.go @@ -176,6 +176,7 @@ func rowsToNDJSON(rows *sql.Rows) (string, error) { } return buffer.String(), nil +} // rowsToTSV converts the rows to TSV string func rowsToTSV(rows *sql.Rows, cols bool) (string, error) {