fix: add envs and reduce memory

bytedance · Jul 3, 2024 · 65f104b · 65f104b
1 parent 8bddb5a
commit 65f104b
Show file tree

Hide file tree

Showing 31 changed files with 34,114 additions and 34,207 deletions.
diff --git a/external_jsonlib_test/go.mod b/external_jsonlib_test/go.mod
@@ -13,6 +13,7 @@ require (
 )
 
 require (
+	github.com/bytedance/gopkg v0.0.0-20240531030433-5df24c0168e2 // indirect
 	github.com/bytedance/sonic/loader v0.1.1 // indirect
 	github.com/cloudwego/base64x v0.1.4 // indirect
 	github.com/cloudwego/iasm v0.2.0 // indirect

diff --git a/external_jsonlib_test/go.sum b/external_jsonlib_test/go.sum
@@ -1,5 +1,7 @@
 github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
 github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
+github.com/bytedance/gopkg v0.0.0-20240531030433-5df24c0168e2 h1:e+WTWDw35RetW+Zuhcy4YPFmNPRh7yPG7lAaWJeUQl0=
+github.com/bytedance/gopkg v0.0.0-20240531030433-5df24c0168e2/go.mod h1:FtQG3YbQG9L/91pbKSw787yBQPutC+457AvDW77fgUQ=
 github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM=
 github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
 github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y=
@@ -45,6 +47,13 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
 golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU=
 golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
+golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

diff --git a/fuzz/compat_amd64_test.go b/fuzz/compat_amd64_test.go
diff --git a/fuzz/compat_other.go b/fuzz/compat_other.go
diff --git a/fuzz/other_fuzz_test.go b/fuzz/other_fuzz_test.go
@@ -51,7 +51,7 @@ func fuzzHtmlEscape(t *testing.T, data []byte){
 func fuzzStream(t *testing.T, data []byte) {
 	r := bytes.NewBuffer(data)
 	dc := decoder.NewStreamDecoder(r)
-	decoderEnableValidateString(dc)
+	dc.ValidateString()
 	r1 := bytes.NewBuffer(data)
 	dc1 := decoder.NewStreamDecoder(r1)
 

diff --git a/internal/decoder/api/decoder.go b/internal/decoder/api/decoder.go
@@ -17,14 +17,16 @@
 package api
 
 import (
-    `reflect`
-
-    `github.com/bytedance/sonic/internal/native`
-    `github.com/bytedance/sonic/internal/native/types`
-	`github.com/bytedance/sonic/internal/decoder/consts`
-	`github.com/bytedance/sonic/internal/decoder/errors`
-    `github.com/bytedance/sonic/internal/rt`
-    `github.com/bytedance/sonic/option`
+	"reflect"
+
+	"github.com/bytedance/sonic/internal/decoder/consts"
+	"github.com/bytedance/sonic/internal/decoder/errors"
+	"github.com/bytedance/sonic/internal/decoder/optdec"
+	"github.com/bytedance/sonic/internal/envs"
+	"github.com/bytedance/sonic/internal/native"
+	"github.com/bytedance/sonic/internal/native/types"
+	"github.com/bytedance/sonic/internal/rt"
+	"github.com/bytedance/sonic/option"
 )
 
 const (
@@ -64,11 +66,21 @@ type Decoder struct {
     i int
     f uint64
     s string
+	owned bool
 }
 
+var padding string = "x\"x\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
+
 // NewDecoder creates a new decoder instance.
 func NewDecoder(s string) *Decoder {
-    return &Decoder{s: s}
+	var owned = false
+	/* clone into buffer and padding it before unmarshal */
+	if envs.UseOptDec {
+		s = optdec.PaddingJson(s)
+		owned = true
+	}
+
+    return &Decoder{s: s, owned: owned}
 }
 
 // Pos returns the current decoding position.

diff --git a/internal/decoder/api/stream.go b/internal/decoder/api/stream.go
@@ -61,25 +61,16 @@ func NewStreamDecoder(r io.Reader) *StreamDecoder {
 func (self *StreamDecoder) Decode(val interface{}) (err error) {
     // read more data into buf
     if self.More() {
-        // println(string(self.buf))
         var s = self.scanp
     try_skip:
         var e = len(self.buf)
-        // println("s:", s, "e:", e, "scanned:",self.scanned, "scanp:",self.scanp, self.buf)
         var src = rt.Mem2Str(self.buf[s:e])
-        // if len(src) > 5 {
-        //     println(src[:5], src[len(src)-5:])
-        // } else {
-        //     println(src)
-        // }
         // try skip
         var x = 0;
         if y := native.SkipOneFast(&src, &x); y < 0 {
             if self.readMore()  {
-                // println("more")
                 goto try_skip
             } else {
-                // println("no more")
                 err = SyntaxError{e, self.s, types.ParsingError(-s), ""}
                 self.setErr(err)
                 return
@@ -89,7 +80,6 @@ func (self *StreamDecoder) Decode(val interface{}) (err error) {
             e = x + s
         }
 
-        // println("decode: ", s, e)
         // must copy string here for safety
         self.Decoder.Reset(string(self.buf[s:e]))
         err = self.Decoder.Decode(val)
@@ -101,13 +91,11 @@ func (self *StreamDecoder) Decode(val interface{}) (err error) {
         self.scanp = e
         _, empty := self.scan()
         if empty {
-            // println("recycle")
             // no remain valid bytes, thus we just recycle buffer
             mem := self.buf
             self.buf = nil
             bufPool.Put(mem[:0])
         } else {
-            // println("keep")
             // remain undecoded bytes, move them onto head
             n := copy(self.buf, self.buf[self.scanp:])
             self.buf = self.buf[:n]
@@ -123,7 +111,6 @@ func (self *StreamDecoder) Decode(val interface{}) (err error) {
 // InputOffset returns the input stream byte offset of the current decoder position. 
 // The offset gives the location of the end of the most recently returned token and the beginning of the next token.
 func (self *StreamDecoder) InputOffset() int64 {
-    // println("input offset",self.scanned, self.scanp)
     return self.scanned + int64(self.scanp)
 }
 

diff --git a/internal/decoder/optdec/decoder.go b/internal/decoder/optdec/decoder.go
@@ -66,6 +66,7 @@ func Decode(s *string, i *int, f uint64, val interface{}) error {
 	if err != nil {
 		return err
 	}
+
 
 	/* parse into document */
 	ctx, err := NewContext(*s, *i, uint64(f), etp)