Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

opt: encoder support Uint64ToString and Int64ToString #723

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions api.go
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,10 @@ type Config struct {

// Encode Infinity or Nan float into `null`, instead of returning an error.
EncodeNullForInfOrNan bool

// Uint64 or Int64 into strings on Marshal
Uint64ToString bool
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

为啥不需要Int64呢?我的意思是Int64和Uint64都用一个选项控制就行了

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

😓😓,我们是PHP(c++扩展)迁移go的,老逻辑只处理了uint64,int64没处理。如果同一个选项控制,会有大量diff😅
所以我给分开了,10多年的老代码了,diff太多迁移成本剧增🤡

Copy link
Author

@period331 period331 Dec 19, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@AsterDY 你看可以不?要么两个选项,要么只有一个uint64的选项,如果合并到一起,我们这边就用不了了😂😂😂

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

那就分开吧

Int64ToString bool
}

var (
Expand Down
135 changes: 135 additions & 0 deletions encode_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1224,4 +1224,139 @@ func TestMarshalInfOrNan(t *testing.T) {
assert.NotNil(t, err)
assert.True(t, strings.Contains(err.Error(), "json: unsupported value: NaN or ±Infinite"))
}
}

func TestUint64OrInt64ToString(t *testing.T) {
int64ptr := int64(432556670863027541)
uint64ptr := uint64(12372850276778298372)
cases := []struct {
name string
val interface{}
exceptUint64ToStr string
exceptInt64ToStr string
exceptFalse string
}{
{
name: "normal_map",
val: map[string]interface{}{
"int": int(12),
"uint": uint(99),
"int64": int64(34),
"uint64": uint64(56),
},
exceptUint64ToStr: `{"int":12,"uint":99,"int64":34,"uint64":"56"}`,
exceptInt64ToStr: `{"int":12,"uint":99,"int64":"34","uint64":56}`,
exceptFalse: `{"int":12,"uint":99,"int64":34,"uint64":56}`,
},
{
name: "int_key_map",
val: map[int64]interface{}{
int64(12): int(12),
int64(99): uint(99),
int64(34): int64(34),
int64(56): uint64(56),
},
exceptUint64ToStr: `{"12":12,"99":99,"34":34,"56":"56"}`,
exceptInt64ToStr: `{"12":12,"99":99,"34":"34","56":56}`,
exceptFalse: `{"12":12,"99":99,"34":34,"56":56}`,
},
{
name: "uint_key_map",
val: map[uint64]interface{}{
uint64(12): int(12),
uint64(99): uint(99),
uint64(34): int64(34),
uint64(56): uint64(56),
},
exceptUint64ToStr: `{"12":12,"99":99,"34":34,"56":"56"}`,
exceptInt64ToStr: `{"12":12,"99":99,"34":"34","56":56}`,
exceptFalse: `{"12":12,"99":99,"34":34,"56":56}`,
},
{
name: "normal_struct",
val: struct {
Int int `json:"int"`
Uint uint `json:"uint"`
Int64 int64 `json:"int64"`
Uint64 uint64 `json:"uint64"`
}{
Int: int(12),
Uint: uint(99),
Int64: int64(34),
Uint64: uint64(56),
},
exceptUint64ToStr: `{"int":12,"uint":99,"int64":34,"uint64":"56"}`,
exceptInt64ToStr: `{"int":12,"uint":99,"int64":"34","uint64":56}`,
exceptFalse: `{"int":12,"uint":99,"int64":34,"uint64":56}`,
},
{
name: "normal_slice",
val: []interface{}{
int(12), uint(99), int64(34), uint64(56),
},
exceptUint64ToStr: `[12,99,34,"56"]`,
exceptInt64ToStr: `[12,99,"34",56]`,
exceptFalse: `[12,99,34,56]`,
},
{
name: "single_int64",
val: int64(34),
exceptUint64ToStr: `34`,
exceptInt64ToStr: `"34"`,
exceptFalse: `34`,
},
{
name: "single_uint64",
val: uint64(56),
exceptUint64ToStr: `"56"`,
exceptInt64ToStr: `56`,
exceptFalse: `56`,
},
{
name: "int64ptr",
val: struct {
Map map[string]interface{}
}{map[string]interface{}{"val": struct {
Int64Ptr interface{}
Uint64Ptr interface{}
Int64 interface{}
Uint64 interface{}
}{
Int64Ptr: &int64ptr,
Uint64Ptr: &uint64ptr,
Int64: int64(123),
Uint64: uint64(456),
}}},
exceptUint64ToStr: `{"Map":{"val":{"Int64Ptr":432556670863027541,` +
`"Uint64Ptr":"12372850276778298372","Int64":123,"Uint64":"456"}}}`,
exceptInt64ToStr: `{"Map":{"val":{"Int64Ptr":"432556670863027541",` +
`"Uint64Ptr":12372850276778298372,"Int64":"123","Uint64":456}}}`,
exceptFalse: `{"Map":{"val":{"Int64Ptr":432556670863027541,` +
`"Uint64Ptr":12372850276778298372,"Int64":123,"Uint64":456}}}`,
},
}

check := func(t *testing.T, except string, testRes []byte) {
var tmp1 interface{}
assert.Nil(t, Unmarshal([]byte(testRes), &tmp1))
var tmp2 interface{}
assert.Nil(t, Unmarshal([]byte(except), &tmp2))
assert.Equal(t, tmp2, tmp1)
}

for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
b, e := Config{Uint64ToString: true}.Froze().Marshal(c.val)
assert.Nil(t, e)
check(t, c.exceptUint64ToStr, b)

b, e = Config{Int64ToString: true}.Froze().Marshal(c.val)
assert.Nil(t, e)
check(t, c.exceptInt64ToStr, b)

b, e = Config{}.Froze().Marshal(c.val)
assert.Nil(t, e)
check(t, c.exceptFalse, b)
})
}
}
4 changes: 4 additions & 0 deletions encoder/encoder_native.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,10 @@ const (

// Encode Infinity or Nan float into `null`, instead of returning an error.
EncodeNullForInfOrNan Options = encoder.EncodeNullForInfOrNan

// Uint64 or Int64 into strings on Marshal
Uint64ToString Options = encoder.Uint64ToString
Int64ToString Options = encoder.Int64ToString
)


Expand Down
4 changes: 3 additions & 1 deletion internal/encoder/alg/opts.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,9 @@ const (
BitValidateString
BitNoValidateJSONMarshaler
BitNoEncoderNewline
BitEncodeNullForInfOrNan
BitEncodeNullForInfOrNan
BitUint64ToString
BitInt64ToString

BitPointerValue = 63
)
8 changes: 4 additions & 4 deletions internal/encoder/compiler.go
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@ func (self *Compiler) compileOps(p *ir.Program, sp int, vt reflect.Type) {
case reflect.Bool:
p.Add(ir.OP_bool)
case reflect.Int:
p.Add(ir.OP_int())
p.Add(ir.OP_int(), ir.OP_i)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

这里没看懂,应该没必要加这个CompactOp。在具体的 OP_i64读取flag bit或IsMapKey()进行处理就行了

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

因为vm里是用uint64的逻辑处理int的,如果没有compatOp,则会把int也处理

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

底层的OP int和uint都是分开的,这里肯定是不需要的。“vm里是用uint64的逻辑处理int的”
vm代码里面明明是分开的,不知道你指的是啥

Copy link
Author

@period331 period331 Dec 25, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@AsterDY

底层的OP int和uint都是分开的

vm里确实是分开的,但是vm里Add时,调用了这样一个函数:

func OP_int() Op {
	switch _INT_SIZE {
	case 32:
		return OP_i32
	case 64:
		return OP_i64
	default:
		panic("unsupported int size")
	}
}

uint也有类似处理,这相当于int用int64、uint用uint64的相关函数来处理了
如果没有CompatOp的话,在int64/uint64的相关处理函数内,就不知道处理的是int/uint了

还有别的地方做差异化处理了吗?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

uint也可能会导致溢出,为啥不支持这个选项?我觉得这个选项只考虑uint64不合理

case reflect.Int8:
p.Add(ir.OP_i8)
case reflect.Int16:
Expand All @@ -189,7 +189,7 @@ func (self *Compiler) compileOps(p *ir.Program, sp int, vt reflect.Type) {
case reflect.Int64:
p.Add(ir.OP_i64)
case reflect.Uint:
p.Add(ir.OP_uint())
p.Add(ir.OP_uint(), ir.OP_u)
case reflect.Uint8:
p.Add(ir.OP_u8)
case reflect.Uint16:
Expand Down Expand Up @@ -301,7 +301,7 @@ func (self *Compiler) compileMapBodyTextKey(p *ir.Program, vk reflect.Type) {
case reflect.Bool:
p.Key(ir.OP_bool)
case reflect.Int:
p.Key(ir.OP_int())
p.Key(ir.OP_int(), ir.OP_i)
case reflect.Int8:
p.Key(ir.OP_i8)
case reflect.Int16:
Expand All @@ -311,7 +311,7 @@ func (self *Compiler) compileMapBodyTextKey(p *ir.Program, vk reflect.Type) {
case reflect.Int64:
p.Key(ir.OP_i64)
case reflect.Uint:
p.Key(ir.OP_uint())
p.Key(ir.OP_uint(), ir.OP_u)
case reflect.Uint8:
p.Key(ir.OP_u8)
case reflect.Uint16:
Expand Down
4 changes: 4 additions & 0 deletions internal/encoder/encoder.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,10 @@ const (

// Encode Infinity or Nan float into `null`, instead of returning an error.
EncodeNullForInfOrNan Options = 1 << alg.BitEncodeNullForInfOrNan

// Uint64 or Int64 into strings on Marshal
Uint64ToString Options = 1 << alg.BitUint64ToString
Int64ToString Options = 1 << alg.BitInt64ToString
)

// Encoder represents a specific set of encoder configurations.
Expand Down
38 changes: 32 additions & 6 deletions internal/encoder/ir/op.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,12 @@ const (
OP_i16
OP_i32
OP_i64
OP_i
OP_u8
OP_u16
OP_u32
OP_u64
OP_u
OP_f32
OP_f64
OP_str
Expand Down Expand Up @@ -99,10 +101,12 @@ var OpNames = [256]string{
OP_i16: "i16",
OP_i32: "i32",
OP_i64: "i64",
OP_i: "i",
OP_u8: "u8",
OP_u16: "u16",
OP_u32: "u32",
OP_u64: "u64",
OP_u: "u",
OP_f32: "f32",
OP_f64: "f64",
OP_str: "str",
Expand Down Expand Up @@ -197,12 +201,26 @@ func OP_is_zero_ints() Op {

type Instr struct {
o Op
co Op
mapKey bool
u int // union {op: 8, _: 8, vi: 48}, vi maybe int or len(str)
p unsafe.Pointer // maybe GoString.Ptr, or *GoType
}

func NewInsOp(op Op) Instr {
return Instr{o: op}
func NewInsOp(op Op, compatOp ...Op) Instr {
i := Instr{o: op, co: op}
if len(compatOp) == 1 {
i.co = compatOp[0]
}
return i
}

func NewInsKeyOp(op Op, compatOp ...Op) Instr {
i := Instr{o: op, co: op, mapKey: true}
if len(compatOp) == 1 {
i.co = compatOp[0]
}
return i
}

func NewInsVi(op Op, vi int) Instr {
Expand Down Expand Up @@ -255,6 +273,14 @@ func (self Instr) Op() Op {
return Op(self.o)
}

func (self Instr) CompatOp() Op {
return Op(self.co)
}

func (self Instr) IsMapKey() bool {
return self.mapKey
}

func (self Instr) Vi() int {
return self.u
}
Expand Down Expand Up @@ -410,14 +436,14 @@ func (self Program) Rel(v []int) {
}
}

func (self *Program) Add(op Op) {
*self = append(*self, NewInsOp(op))
func (self *Program) Add(op Op, co ...Op) {
*self = append(*self, NewInsOp(op, co...))
}

func (self *Program) Key(op Op) {
func (self *Program) Key(op Op, co ...Op) {
*self = append(*self,
NewInsVi(OP_byte, '"'),
NewInsOp(op),
NewInsKeyOp(op, co...),
NewInsVi(OP_byte, '"'),
)
}
Expand Down
18 changes: 18 additions & 0 deletions internal/encoder/vm/vm.go
Original file line number Diff line number Diff line change
Expand Up @@ -140,8 +140,17 @@ func Execute(b *[]byte, p unsafe.Pointer, s *vars.Stack, flags uint64, prog *ir.
v := *(*int32)(p)
buf = alg.I64toa(buf, int64(v))
case ir.OP_i64:
if ins.CompatOp() == ir.OP_i ||
ins.IsMapKey() ||
flags&(1<<alg.BitInt64ToString) == 0 {
v := *(*int64)(p)
buf = alg.I64toa(buf, int64(v))
continue
}
buf = append(buf, '"')
v := *(*int64)(p)
buf = alg.I64toa(buf, int64(v))
buf = append(buf, '"')
case ir.OP_u8:
v := *(*uint8)(p)
buf = alg.U64toa(buf, uint64(v))
Expand All @@ -152,8 +161,17 @@ func Execute(b *[]byte, p unsafe.Pointer, s *vars.Stack, flags uint64, prog *ir.
v := *(*uint32)(p)
buf = alg.U64toa(buf, uint64(v))
case ir.OP_u64:
if ins.CompatOp() == ir.OP_u ||
ins.IsMapKey() ||
flags&(1<<alg.BitUint64ToString) == 0 {
v := *(*uint64)(p)
buf = alg.U64toa(buf, uint64(v))
continue
}
buf = append(buf, '"')
v := *(*uint64)(p)
buf = alg.U64toa(buf, uint64(v))
buf = append(buf, '"')
case ir.OP_f32:
v := *(*float32)(p)
if math.IsNaN(float64(v)) || math.IsInf(float64(v), 0) {
Expand Down
31 changes: 29 additions & 2 deletions internal/encoder/x86/assembler_regabi_amd64.go
Original file line number Diff line number Diff line change
Expand Up @@ -844,8 +844,22 @@ func (self *Assembler) _asm_OP_i32(_ *ir.Instr) {
self.store_int(17, _F_i64toa, "MOVLQSX")
}

func (self *Assembler) _asm_OP_i64(_ *ir.Instr) {
func (self *Assembler) _asm_OP_i64(i *ir.Instr) {
if i.CompatOp() == ir.OP_i || i.IsMapKey() {
self.store_int(21, _F_i64toa, "MOVQ")
return
}
self.Emit("BTQ", jit.Imm(int64(alg.BitInt64ToString)), _ARG_fv)
self.Sjmp("JC", "_i64_to_string{n}")
self.store_int(21, _F_i64toa, "MOVQ")
self.Sjmp("JMP", "_i64_to_string_end{n}")
self.Link("_i64_to_string{n}")

self.add_char('"')
self.store_int(21, _F_i64toa, "MOVQ")
self.add_char('"')
self.Link("_i64_to_string_end{n}")

}

func (self *Assembler) _asm_OP_u8(_ *ir.Instr) {
Expand All @@ -860,8 +874,21 @@ func (self *Assembler) _asm_OP_u32(_ *ir.Instr) {
self.store_int(16, _F_u64toa, "MOVLQZX")
}

func (self *Assembler) _asm_OP_u64(_ *ir.Instr) {
func (self *Assembler) _asm_OP_u64(i *ir.Instr) {
if i.CompatOp() == ir.OP_u || i.IsMapKey() {
self.store_int(20, _F_u64toa, "MOVQ")
return
}
self.Emit("BTQ", jit.Imm(int64(alg.BitUint64ToString)), _ARG_fv)
self.Sjmp("JC", "_ui64_to_string{n}")
self.store_int(20, _F_u64toa, "MOVQ")
self.Sjmp("JMP", "_ui64_to_string_end{n}")
self.Link("_ui64_to_string{n}")

self.add_char('"')
self.store_int(20, _F_u64toa, "MOVQ")
self.add_char('"')
self.Link("_ui64_to_string_end{n}")
}

func (self *Assembler) _asm_OP_f32(_ *ir.Instr) {
Expand Down
Loading
Loading