From aa5abe655a053f6b9f6b9a0ea92510620be46b15 Mon Sep 17 00:00:00 2001 From: Kyle Xiao Date: Tue, 26 Nov 2024 16:58:25 +0800 Subject: [PATCH] feat(reflect): supports nocopy option --- internal/reflect/decoder.go | 56 ++++++++++++++++++++++++++++---- internal/reflect/decoder_test.go | 46 ++++++++++++++++++++++++++ internal/reflect/desc.go | 8 +++-- internal/reflect/hack.go | 14 ++++++++ 4 files changed, 115 insertions(+), 9 deletions(-) diff --git a/internal/reflect/decoder.go b/internal/reflect/decoder.go index 94f392f..6d822d4 100644 --- a/internal/reflect/decoder.go +++ b/internal/reflect/decoder.go @@ -125,7 +125,13 @@ func (d *tDecoder) Decode(b []byte, base unsafe.Pointer, sd *structDesc, maxdept if t.FixedSize > 0 { i += decodeFixedSizeTypes(t.T, b[i:], p) } else { - n, err := d.decodeType(t, b[i:], p, maxdepth-1) + var n int + var err error + if f.NoCopy { + n, err = decodeStringNoCopy(t, b[i:], p) + } else { + n, err = d.decodeType(t, b[i:], p, maxdepth-1) + } if err != nil { return i, fmt.Errorf("decode field %d of struct %s err: %w", fid, sd.rt.String(), err) } @@ -168,6 +174,39 @@ func decodeFixedSizeTypes(t ttype, b []byte, p unsafe.Pointer) int { } } +func decodeStringNoCopy(t *tType, b []byte, p unsafe.Pointer) (i int, err error) { + l := int(binary.BigEndian.Uint32(b)) + if l < 0 { + err = errNegativeSize + return + } + i += 4 + if l == 0 { + if t.Tag == defs.T_binary { + *(*[]byte)(p) = []byte{} + } else { + *(*string)(p) = "" + } + return + } + + // assert len, panic if []byte shorter than expected. + _ = b[i+l-1] + + if t.Tag == defs.T_binary { + h := (*sliceHeader)(p) + h.Data = uintptr(unsafe.Pointer(&b[i])) + h.Len = l + h.Cap = l + } else { // convert to str + h := (*stringHeader)(p) + h.Data = uintptr(unsafe.Pointer(&b[i])) + h.Len = l + } + i += l + return +} + func (d *tDecoder) decodeType(t *tType, b []byte, p unsafe.Pointer, maxdepth int) (int, error) { if maxdepth == 0 { return 0, errDepthLimitExceeded @@ -184,12 +223,16 @@ func (d *tDecoder) decodeType(t *tType, b []byte, p unsafe.Pointer, maxdepth int i := 4 if l == 0 { if t.Tag == defs.T_binary { - *(*sliceHeader)(p) = zeroSliceHeader + *(*[]byte)(p) = []byte{} } else { - *(*stringHeader)(p) = zeroStrHeader + *(*string)(p) = "" } return i, nil } + + // assert len, panic if []byte shorter than expected. + _ = b[i+l-1] + x := d.Malloc(l, 1, 0) if t.Tag == defs.T_binary { h := (*sliceHeader)(p) @@ -304,15 +347,14 @@ func (d *tDecoder) decodeType(t *tType, b []byte, p unsafe.Pointer, maxdepth int // decode list h := (*sliceHeader)(p) // update the slice field - h.Data = 0 - h.Len = l - h.Cap = l if l <= 0 { - *(*sliceHeader)(p) = zeroSliceHeader + h.Zero() return i, nil } x := d.Malloc(l*et.Size, et.Align, et.MallocAbiType) // malloc for slice. make([]Type, l, l) h.Data = uintptr(x) + h.Len = l + h.Cap = l // pre-allocate space for elements if they're pointers // like diff --git a/internal/reflect/decoder_test.go b/internal/reflect/decoder_test.go index f4c5311..9ef1e9a 100644 --- a/internal/reflect/decoder_test.go +++ b/internal/reflect/decoder_test.go @@ -17,6 +17,7 @@ package reflect import ( + "bytes" "math" "math/rand" "testing" @@ -358,3 +359,48 @@ func TestDecodeUnknownFields(t *testing.T) { assert.Equal(t, sz, len(testb)) assert.Equal(t, testb, p._unknownFields) } + +func TestDecodeNoCopy(t *testing.T) { + type Msg struct { + A string `frugal:"1,default,string,nocopy"` + B string `frugal:"2,default,string"` + C []byte `frugal:"3,default,binary,nocopy"` + } + + strA := "strA" + strB := "strB" + strC := "strC" + sz := 3*(fieldHeaderLen+strHeaderLen) + len(strA) + len(strB) + len(strC) + b := make([]byte, 0, sz) + b = appendStringField(b, 1, strA) + b = appendStringField(b, 2, strB) + b = appendStringField(b, 3, strC) + b = append(b, 0) // tSTOP + + p := &Msg{} + _, err := Decode(b, p) + require.NoError(t, err) + + assert.Equal(t, strA, p.A) + assert.Equal(t, strB, p.B) + assert.Equal(t, strC, string(p.C)) + + // update original buffer + // coz it's nocopy, the fields of p will be changed implicitly as well. + // update strA -> xtrA, strB -> xtrB, strC -> xtrC + for _, s := range []string{strA, strB, strC} { + i := bytes.Index(b, []byte(s)) + b[i] = 'x' + } + assert.Equal(t, "xtrA", p.A) + assert.Equal(t, "strB", p.B) // p.B has no `nocopy` option + assert.Equal(t, "xtrC", string(p.C)) + + type Msg2 struct { + A int32 `frugal:"4,default,i32,nocopy"` + } + p2 := &Msg2{} + _, err = Decode(b, p2) + require.Error(t, err) + _ = p2 +} diff --git a/internal/reflect/desc.go b/internal/reflect/desc.go index 8f99cda..c584fbd 100644 --- a/internal/reflect/desc.go +++ b/internal/reflect/desc.go @@ -221,10 +221,10 @@ type tField struct { Offset uintptr Type *tType - Opts defs.Options Spec defs.Requiredness Default unsafe.Pointer + NoCopy bool CanSkipEncodeIfNil bool CanSkipIfDefault bool } @@ -263,11 +263,15 @@ func (f *tField) fromDefsField(x defs.Field) { f.ID = x.ID f.Offset = uintptr(x.F) f.Type = newTType(x.Type) - f.Opts = x.Opts f.Spec = x.Spec t := f.Type + f.NoCopy = (x.Opts & defs.NoCopy) != 0 + if f.NoCopy && f.Type.WT != tSTRING { + // never goes here, defs will check the tag + panic("[BUG] nocopy on non-STRING type") + } // for map or slice, t.IsPointer() is false, // but we can consider the types as pointer as per lang spec // for defs.T_binary, actually it's []byte, like tLIST diff --git a/internal/reflect/hack.go b/internal/reflect/hack.go index 60d15f0..1d80366 100644 --- a/internal/reflect/hack.go +++ b/internal/reflect/hack.go @@ -239,5 +239,19 @@ func (h *sliceHeader) UnsafePointer() unsafe.Pointer { return *(*unsafe.Pointer)(unsafe.Pointer(h)) } +var ( + emptyslice = make([]byte, 0) + + // for slice, Data should points to zerobase var in `runtime` + // so that it can represent as []type{} instead of []type(nil) + zerobase = ((*sliceHeader)(unsafe.Pointer(&emptyslice))).Data +) + +func (h *sliceHeader) Zero() { + h.Len = 0 + h.Cap = 0 + h.Data = zerobase +} + //go:linkname mallocgc runtime.mallocgc func mallocgc(size uintptr, typ unsafe.Pointer, needzero bool) unsafe.Pointer