Skip to content

Commit

Permalink
fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
Amogh-Bharadwaj committed Nov 18, 2024
1 parent 21958a9 commit 1bbe5b3
Show file tree
Hide file tree
Showing 3 changed files with 12 additions and 4 deletions.
4 changes: 2 additions & 2 deletions flow/connectors/postgres/qvalue_convert.go
Original file line number Diff line number Diff line change
Expand Up @@ -297,12 +297,12 @@ func parseFieldFromQValueKind(qvalueKind qvalue.QValueKind, value interface{}) (
upperBoundType := tstzrangeObject.UpperType
lowerTime, err := convertTimeRangeBound(tstzrangeObject.Lower)
if err != nil {
return nil, fmt.Errorf("[tstzrange]error for lower time bound: %v", err)
return nil, fmt.Errorf("[tstzrange]error for lower time bound: %w", err)
}

upperTime, err := convertTimeRangeBound(tstzrangeObject.Upper)
if err != nil {
return nil, fmt.Errorf("[tstzrange]error for upper time bound: %v", err)
return nil, fmt.Errorf("[tstzrange]error for upper time bound: %w", err)
}

lowerBracket := "["
Expand Down
6 changes: 5 additions & 1 deletion flow/model/qvalue/avro_converter.go
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,8 @@ func GetAvroSchemaFromQValueKind(kind QValueKind, targetDWH protos.DBType, preci
}, nil
}
return "string", nil
case QValueKindTSTZRange:
return "string", nil
case QValueKindHStore, QValueKindJSON, QValueKindJSONB, QValueKindStruct:
return "string", nil
case QValueKindArrayFloat32:
Expand Down Expand Up @@ -193,6 +195,8 @@ func GetAvroSchemaFromQValueKind(kind QValueKind, targetDWH protos.DBType, preci
Type: "array",
Items: "string",
}, nil
case QValueKindArrayJSON, QValueKindArrayJSONB:
return "string", nil
case QValueKindArrayString:
return AvroSchemaArray{
Type: "array",
Expand Down Expand Up @@ -315,7 +319,7 @@ func QValueToAvro(value QValue, field *QField, targetDWH protos.DBType, logger l
return t, nil
case QValueQChar:
return c.processNullableUnion("string", string(v.Val))
case QValueString, QValueCIDR, QValueINET, QValueMacaddr, QValueInterval:
case QValueString, QValueCIDR, QValueINET, QValueMacaddr, QValueInterval, QValueTSTZRange:
if c.TargetDWH == protos.DBType_SNOWFLAKE && v.Value() != nil &&
(len(v.Value().(string)) > 15*1024*1024) {
slog.Warn("Clearing TEXT value > 15MB for Snowflake!")
Expand Down
6 changes: 5 additions & 1 deletion flow/model/qvalue/kind.go
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,8 @@ var QValueKindToSnowflakeTypeMap = map[QValueKind]string{
QValueKindArrayTimestamp: "VARIANT",
QValueKindArrayTimestampTZ: "VARIANT",
QValueKindArrayBoolean: "VARIANT",
QValueKindArrayJSON: "VARIANT",
QValueKindArrayJSONB: "VARIANT",
}

var QValueKindToClickHouseTypeMap = map[QValueKind]string{
Expand All @@ -114,6 +116,7 @@ var QValueKindToClickHouseTypeMap = map[QValueKind]string{
QValueKindJSON: "String",
QValueKindTimestamp: "DateTime64(6)",
QValueKindTimestampTZ: "DateTime64(6)",
QValueKindTSTZRange: "String",
QValueKindTime: "DateTime64(6)",
QValueKindTimeTZ: "DateTime64(6)",
QValueKindDate: "Date32",
Expand All @@ -123,7 +126,6 @@ var QValueKindToClickHouseTypeMap = map[QValueKind]string{
QValueKindInvalid: "String",
QValueKindHStore: "String",

// array types will be mapped to VARIANT
QValueKindArrayFloat32: "Array(Float32)",
QValueKindArrayFloat64: "Array(Float64)",
QValueKindArrayInt32: "Array(Int32)",
Expand All @@ -134,6 +136,8 @@ var QValueKindToClickHouseTypeMap = map[QValueKind]string{
QValueKindArrayDate: "Array(Date)",
QValueKindArrayTimestamp: "Array(DateTime64(6))",
QValueKindArrayTimestampTZ: "Array(DateTime64(6))",
QValueKindArrayJSON: "String",
QValueKindArrayJSONB: "String",
}

func (kind QValueKind) ToDWHColumnType(dwhType protos.DBType) (string, error) {
Expand Down

0 comments on commit 1bbe5b3

Please sign in to comment.