feat: new append method (#66)

frugal.go

@@ -17,6 +17,8 @@
 package frugal
 
 import (
+	"fmt"
+
 	"github.com/cloudwego/frugal/internal/opts"
 	"github.com/cloudwego/frugal/internal/reflect"
 	"github.com/cloudwego/gopkg/protocol/thrift"
@@ -34,7 +36,13 @@ func EncodedSize(val interface{}) int {
 // buf must be large enough to contain the entire serialization result.
 func EncodeObject(buf []byte, w thrift.NocopyWriter, val interface{}) (int, error) {
 	if !jit || opts.NoJIT {
-		return reflect.Encode(buf, val) // TODO: impl thrift.NocopyWriter
+		ret, err := reflect.Append(buf[:0], val)
+		if len(ret) > len(buf) {
+			return 0, fmt.Errorf("index out of range [%d] with length %d.\n"+
+				"Please make sure the input will not be changed after calling EncodedSize or during EncodeObject(concurrency issues).",
+				len(ret), len(buf))
+		}
+		return len(ret), err
 	}
 	return jitEncodeObject(buf, w, val)
 }

go.mod

@@ -1,6 +1,6 @@
 module github.com/cloudwego/frugal
 
-go 1.17
+go 1.18
 
 require (
 	github.com/cloudwego/gopkg v0.1.2

internal/reflect/append.go

@@ -0,0 +1,105 @@
+/*
+ * Copyright 2024 CloudWeGo Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package reflect
+
+import "unsafe"
+
+func appendStruct(t *tType, b []byte, base unsafe.Pointer) ([]byte, error) {
+	sd := t.Sd
+	if base == nil {
+		return append(b, byte(tSTOP)), nil
+	}
+	var err error
+	for _, f := range sd.fields {
+		t := f.Type
+		p := unsafe.Add(base, f.Offset)
+		if f.CanSkipEncodeIfNil && *(*unsafe.Pointer)(p) == nil {
+			continue
+		}
+		if f.CanSkipIfDefault && t.Equal(f.Default, p) {
+			continue
+		}
+
+		// field header
+		b = append(b, byte(t.WT), byte(f.ID>>8), byte(f.ID))
+
+		// field value
+		// the following code should be the same as func `appendAny`
+		// manually copy here for inlining:
+
+		if t.IsPointer {
+			p = *(*unsafe.Pointer)(p)
+		}
+		if t.SimpleType { // fast path
+			switch t.T {
+			case tBYTE, tBOOL:
+				b = append(b, *(*byte)(p)) // for tBOOL, true -> 1, false -> 0
+			case tI16:
+				b = appendUint16(b, *((*uint16)(p)))
+			case tI32:
+				b = appendUint32(b, *((*uint32)(p)))
+			case tENUM:
+				b = appendUint32(b, uint32(*((*int64)(p))))
+			case tI64, tDOUBLE:
+				b = appendUint64(b, *((*uint64)(p)))
+			case tSTRING:
+				s := *((*string)(p))
+				b = appendUint32(b, uint32(len(s)))
+				b = append(b, s...)
+			}
+		} else {
+			b, err = t.AppendFunc(t, b, p)
+			if err != nil {
+				return b, withFieldErr(err, sd, f)
+			}
+		}
+	}
+	if sd.hasUnknownFields {
+		xb := *(*[]byte)(unsafe.Add(base, sd.unknownFieldsOffset))
+		if len(xb) > 0 {
+			b = append(b, xb...)
+		}
+	}
+	return append(b, byte(tSTOP)), nil
+}
+
+func appendAny(t *tType, b []byte, p unsafe.Pointer) ([]byte, error) {
+	if t.IsPointer {
+		p = *(*unsafe.Pointer)(p)
+	}
+	if t.SimpleType {
+		switch t.T {
+		case tBYTE, tBOOL:
+			b = append(b, *(*byte)(p)) // for tBOOL, true -> 1, false -> 0
+		case tI16:
+			b = appendUint16(b, *((*uint16)(p)))
+		case tI32:
+			b = appendUint32(b, *((*uint32)(p)))
+		case tENUM:
+			b = appendUint32(b, uint32(*((*int64)(p))))
+		case tI64, tDOUBLE:
+			b = appendUint64(b, *((*uint64)(p)))
+		case tSTRING:
+			s := *((*string)(p))
+			b = appendUint32(b, uint32(len(s)))
+			b = append(b, s...)
+		}
+		return b, nil
+	} else {
+		return t.AppendFunc(t, b, p)
+	}
+}

internal/reflect/append_gen.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# Copyright 2024 CloudWeGo Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FRUGAL_GEN_APPEND_MAP_FILE="append_map_gen.go"
+FRUGAL_GEN_APPEND_LIST_FILE="append_list_gen.go"
+
+rm -f $FRUGAL_GEN_APPEND_MAP_FILE
+rm -f $FRUGAL_GEN_APPEND_LIST_FILE
+
+exec go test -v -run=TestGenAppend -gencode=true

internal/reflect/append_gen_test.go

@@ -0,0 +1,99 @@
+/*
+ * Copyright 2024 CloudWeGo Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package reflect
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"strings"
+)
+
+var (
+	gencode = flag.Bool("gencode", false, "generate list/map code for better performance")
+)
+
+var tOTHER = ttype(0xee) // must not in use, only for generating code
+
+func init() {
+	t2s[tOTHER] = "Other" // makes ttype2str work
+}
+
+func ttype2FuncType(t ttype) string {
+	switch t {
+	case tSTRUCT, tMAP, tSET, tLIST:
+		t = tOTHER
+	case tDOUBLE:
+		t = tI64
+	}
+	return ttype2str(t)
+}
+
+var (
+	defineErr = map[ttype]bool{tOTHER: true}
+	defineStr = map[ttype]bool{tSTRING: true}
+)
+
+func getAppendCode(typ ttype, t, p string) string {
+	t2c := map[ttype]string{
+		tBYTE:   "b = append(b, *((*byte)({p})))",
+		tI16:    "b = appendUint16(b, *((*uint16)({p})))",
+		tI32:    "b = appendUint32(b, *((*uint32)({p})))",
+		tI64:    "b = appendUint64(b, *((*uint64)({p})))",
+		tDOUBLE: "b = appendUint64(b, *((*uint64)({p})))",
+		tENUM:   "b = appendUint32(b, uint32(*((*int64)({p}))))",
+		tSTRING: "s = *((*string)({p})); b = appendUint32(b, uint32(len(s))); b = append(b, s...)",
+
+		// tSTRUCT, tMAP, tSET, tLIST -> tOTHER
+		tOTHER: `if {t}.IsPointer {
+		b, err = {t}.AppendFunc({t}, b, *(*unsafe.Pointer)({p}))
+	} else {
+		b, err = {t}.AppendFunc({t}, b, {p})
+	}
+	if err != nil {
+		return b, err
+}`,
+	}
+	s, ok := t2c[typ]
+	if !ok {
+		panic("type doesn't have code: " + ttype2str(typ))
+	}
+	s = strings.ReplaceAll(s, "{t}", t)
+	s = strings.ReplaceAll(s, "{p}", p)
+	return s
+}
+
+func codeWithLine(b []byte) string {
+	p := &strings.Builder{}
+	p.Grow(len(b) + 5*bytes.Count(b, []byte("\n")))
+
+	n := 1
+	i := 0
+	fmt.Fprintf(p, "%4d ", n)
+	for j := 0; j < len(b); j++ {
+		if b[j] == '\n' {
+			p.Write(b[i : j+1])
+			i = j + 1
+			n++
+			fmt.Fprintf(p, "%4d ", n)
+		}
+	}
+	if i < len(b) {
+		p.Write(b[i:])
+	}
+	return p.String()
+}

internal/reflect/append_list.go

@@ -0,0 +1,67 @@
+/*
+ * Copyright 2024 CloudWeGo Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package reflect
+
+import "unsafe"
+
+var listAppendFuncs = map[ttype]appendFuncType{}
+
+func updateListAppendFunc(t *tType) {
+	if t.T != tLIST && t.T != tSET {
+		panic("[bug] type mismatch, got: " + ttype2str(t.T))
+	}
+	f, ok := listAppendFuncs[t.V.T]
+	if ok {
+		t.AppendFunc = f
+		return
+	}
+	t.AppendFunc = appendListAny
+}
+
+func registerListAppendFunc(t ttype, f appendFuncType) {
+	listAppendFuncs[t] = f
+}
+
+func appendListHeader(t *tType, b []byte, p unsafe.Pointer) ([]byte, uint32, unsafe.Pointer) {
+	if *(*unsafe.Pointer)(p) == nil {
+		return append(b, byte(t.WT), 0, 0, 0, 0), 0, nil
+	}
+	h := (*sliceHeader)(p)
+	n := uint32(h.Len)
+	return append(b, byte(t.WT),
+			byte(n>>24), byte(n>>16), byte(n>>8), byte(n)),
+		n, h.UnsafePointer()
+}
+
+func appendListAny(t *tType, b []byte, p unsafe.Pointer) ([]byte, error) {
+	t = t.V
+	b, n, vp := appendListHeader(t, b, p)
+	if n == 0 {
+		return b, nil
+	}
+	var err error
+	for i := uint32(0); i < n; i++ {
+		if i != 0 {
+			vp = unsafe.Add(vp, t.Size) // move to next element
+		}
+		b, err = appendAny(t, b, vp)
+		if err != nil {
+			return b, err
+		}
+	}
+	return b, nil
+}