diff --git a/.codespellrc b/.codespellrc new file mode 100644 index 000000000..1ccef98d5 --- /dev/null +++ b/.codespellrc @@ -0,0 +1,5 @@ +[codespell] +# ignore test files, go project names, binary files via `skip` and special var/regex via `ignore-words` +skip = fuzz,*_test.tmpl,testdata,*_test.go,go.mod,go.sum,*.gz +ignore-words = .github/workflows/.ignore_words +check-filenames = true diff --git a/.github/workflows/.ignore_words b/.github/workflows/.ignore_words new file mode 100644 index 000000000..ee952acef --- /dev/null +++ b/.github/workflows/.ignore_words @@ -0,0 +1,6 @@ +socio-economic +nd +regArgs +oders +ure +alse diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 0f03348b3..28ec2cb5d 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -6,7 +6,7 @@ jobs: build: strategy: matrix: - os: [X64, arm] + os: [X64, macos-latest] runs-on: ${{ matrix.os }} steps: - name: Clear repository @@ -30,9 +30,9 @@ jobs: continue-on-error: true run: | export SONIC_NO_ASYNC_GC=1 - go test -run ^$ -count=100 -benchmem -bench 'BenchmarkDecoder_(Generic|Binding)_Sonic' ./decoder >> /var/tmp/sonic_bench_target_${{ github.run_id }}.out - go test -run ^$ -count=100 -benchmem -bench 'BenchmarkEncoder_(Generic|Binding)_Sonic' ./encoder >> /var/tmp/sonic_bench_target_${{ github.run_id }}.out - go test -run ^$ -count=100 -benchmem -bench 'Benchmark(Get|Set)One_Sonic|BenchmarkParseSeven_Sonic' ./ast >> /var/tmp/sonic_bench_target_${{ github.run_id }}.out + go test -run ^$ -count=20 -benchmem -bench 'BenchmarkDecoder_(Generic|Binding)_Sonic' ./decoder >> /var/tmp/sonic_bench_target_${{ github.run_id }}.out + go test -run ^$ -count=20 -benchmem -bench 'BenchmarkEncoder_(Generic|Binding)_Sonic' ./encoder >> /var/tmp/sonic_bench_target_${{ github.run_id }}.out + go test -run ^$ -count=20 -benchmem -bench 'Benchmark(Get|Set)One_Sonic|BenchmarkParseSeven_Sonic' ./ast >> /var/tmp/sonic_bench_target_${{ github.run_id }}.out - name: Clear repository run: sudo rm -fr $GITHUB_WORKSPACE && mkdir $GITHUB_WORKSPACE @@ -46,10 +46,9 @@ jobs: continue-on-error: true run: | export SONIC_NO_ASYNC_GC=1 - UNIQUE_ID=${{ github.run_id }} - go test -run ^$ -count=100 -benchmem -bench 'BenchmarkDecoder_(Generic|Binding)_Sonic' ./decoder >> /var/tmp/sonic_bench_main_${{ github.run_id }}.out - go test -run ^$ -count=100 -benchmem -bench 'BenchmarkEncoder_(Generic|Binding)_Sonic' ./encoder >> /var/tmp/sonic_bench_main_${{ github.run_id }}.out - go test -run ^$ -count=100 -benchmem -bench 'Benchmark(Get|Set)One_Sonic|BenchmarkParseSeven_Sonic' ./ast >> /var/tmp/sonic_bench_main_${{ github.run_id }}.out + go test -run ^$ -count=20 -benchmem -bench 'BenchmarkDecoder_(Generic|Binding)_Sonic' ./decoder >> /var/tmp/sonic_bench_main_${{ github.run_id }}.out + go test -run ^$ -count=20 -benchmem -bench 'BenchmarkEncoder_(Generic|Binding)_Sonic' ./encoder >> /var/tmp/sonic_bench_main_${{ github.run_id }}.out + go test -run ^$ -count=20 -benchmem -bench 'Benchmark(Get|Set)One_Sonic|BenchmarkParseSeven_Sonic' ./ast >> /var/tmp/sonic_bench_main_${{ github.run_id }}.out - name: Diff bench continue-on-error: true diff --git a/.github/workflows/compatibility_test.yml b/.github/workflows/compatibility_test.yml index 30f2f536c..1ed858dc6 100644 --- a/.github/workflows/compatibility_test.yml +++ b/.github/workflows/compatibility_test.yml @@ -7,7 +7,7 @@ jobs: strategy: matrix: go-version: [1.16.x, 1.17.x, 1.18.x, 1.19.x, 1.20.x, 1.21.x, 1.22.x, 1.23.x] - os: [arm, X64] + os: [X64, macos-latest] runs-on: ${{ matrix.os }} steps: - name: Clear repository @@ -38,6 +38,3 @@ jobs: - name: ast run: go test -race -v -gcflags="all=-l" github.com/bytedance/sonic/ast - - - name: qemu - run: sh scripts/qemu.sh \ No newline at end of file diff --git a/.github/workflows/fuzzing-linux-opt-X64.yml b/.github/workflows/fuzzing-linux-opt-X64.yml index a46d657f8..fc653c780 100644 --- a/.github/workflows/fuzzing-linux-opt-X64.yml +++ b/.github/workflows/fuzzing-linux-opt-X64.yml @@ -6,7 +6,7 @@ jobs: build: strategy: matrix: - os: [arm, X64] + os: [macos-latest, X64] runs-on: ${{ matrix.os }} steps: - name: Clear repository diff --git a/.github/workflows/fuzzing-linux-x64.yml b/.github/workflows/fuzzing-linux-x64.yml index e0c2a11b9..14d89ccbd 100644 --- a/.github/workflows/fuzzing-linux-x64.yml +++ b/.github/workflows/fuzzing-linux-x64.yml @@ -6,7 +6,7 @@ jobs: build: strategy: matrix: - os: [arm, X64] + os: [macos-latest, X64] runs-on: ${{ matrix.os }} steps: - name: Clear repository diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 000000000..e8ae2b90a --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,13 @@ +name: Lint + +on: pull_request + +jobs: + misc: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: spell check + uses: codespell-project/actions-codespell@v2 + with: + skip: loader/internal/iasm/obj/macho.go,loader/internal/iasm/obj/macho.go,loader/internal/iasm/obj/macho.go,loader/internal/iasm/obj/macho.go,loader/internal/iasm/x86_64/encodings.go:720,loader/internal/iasm/x86_64/program.go,loader/internal/iasm/x86_64/program.go,loader/internal/iasm/x86_64/program.go,loader/internal/iasm/expr/ast.go,loader/internal/iasm/expr/errors.go diff --git a/.github/workflows/unit_test-linux-arm.yml b/.github/workflows/unit_test-linux-arm.yml index cc932e129..c2a02a1b6 100644 --- a/.github/workflows/unit_test-linux-arm.yml +++ b/.github/workflows/unit_test-linux-arm.yml @@ -1,4 +1,4 @@ -name: Unit Test Linux arm +name: Unit Test Linux macos-latest on: push @@ -8,7 +8,7 @@ jobs: matrix: # TODO: 1.17.x, 1.18.x, 1.19.x not supported because golang asm bug go-version: [1.20.x, 1.21.x, 1.22.x, 1.23.x] - runs-on: [arm] + runs-on: [macos-latest] steps: - name: Clear repository run: sudo rm -fr $GITHUB_WORKSPACE && mkdir $GITHUB_WORKSPACE diff --git a/README.md b/README.md index 5f609b1c3..576c15bce 100644 --- a/README.md +++ b/README.md @@ -211,7 +211,7 @@ ret, err := Encode(v, EscapeHTML) // ret == `{"\u0026\u0026":{"X":"\u003c\u003e" ### Compact Format -Sonic encodes primitive objects (struct/map...) as compact-format JSON by default, except marshaling `json.RawMessage` or `json.Marshaler`: sonic ensures validating their output JSON but **DONOT** compacting them for performance concerns. We provide the option `encoder.CompactMarshaler` to add compacting process. +Sonic encodes primitive objects (struct/map...) as compact-format JSON by default, except marshaling `json.RawMessage` or `json.Marshaler`: sonic ensures validating their output JSON but **DO NOT** compacting them for performance concerns. We provide the option `encoder.CompactMarshaler` to add compacting process. ### Print Error @@ -480,9 +480,9 @@ But `ast.Visitor` is not a very handy API. You might need to write a lot of code ### Buffer Size -Sonic use memory pool in many places like `encoder.Encode`, `ast.Node.MarshalJSON` to improve performace, which may produce more memory usage (in-use) when server's load is high. See [issue 614](https://github.com/bytedance/sonic/issues/614). Therefore, we introduce some options to let user control the behavior of memory pool. See [option](https://pkg.go.dev/github.com/bytedance/sonic@v1.11.9/option#pkg-variables) package. +Sonic use memory pool in many places like `encoder.Encode`, `ast.Node.MarshalJSON` to improve performance, which may produce more memory usage (in-use) when server's load is high. See [issue 614](https://github.com/bytedance/sonic/issues/614). Therefore, we introduce some options to let user control the behavior of memory pool. See [option](https://pkg.go.dev/github.com/bytedance/sonic@v1.11.9/option#pkg-variables) package. -### Faster JSON skip +### Faster JSON Skip For security, sonic use [FSM](native/skip_one.c) algorithm to validate JSON when decoding raw JSON or encoding `json.Marshaler`, which is much slower (1~10x) than [SIMD-searching-pair](native/skip_one_fast.c) algorithm. If user has many redundant JSON value and DO NOT NEED to strictly validate JSON correctness, you can enable below options: @@ -490,6 +490,11 @@ For security, sonic use [FSM](native/skip_one.c) algorithm to validate JSON whe - `Config.NoValidateJSONMarshaler`: avoid validating JSON when encoding `json.Marshaler` - `SearchOption.ValidateJSON`: indicates if validate located JSON value when `Get` +## JSON-Path Support (GJSON) + +[tidwall/gjson](https://github.com/tidwall/gjson) has provided a comprehensive and popular JSON-Path API, and + a lot of older codes heavily relies on it. Therefore, we provides a wrapper library, which combines gjson's API with sonic's SIMD algorithm to boost up the performance. See [cloudwego/gjson](https://github.com/cloudwego/gjson). + ## Community Sonic is a subproject of [CloudWeGo](https://www.cloudwego.io/). We are committed to building a cloud native ecosystem. diff --git a/README_ZH_CN.md b/README_ZH_CN.md index 4f8980c53..cf6e80764 100644 --- a/README_ZH_CN.md +++ b/README_ZH_CN.md @@ -286,9 +286,9 @@ sub := root.Get("key3").Index(2).Int64() // == 3 `ast.Searcher`提供了一些选项，以满足用户的不同需求: -``` -opts:= ast.SearchOption{CopyReturn: true…} -Val, err:= sonic。gettwithoptions (JSON, opts， "key") +```go +opts := ast.SearchOption{CopyReturn: true…} +val, err := sonic.GetWithOptions(JSON, opts, "key") ``` - CopyReturn @@ -389,7 +389,7 @@ type Visitor interface { - `ConfigDefault`: sonic的默认配置 (`EscapeHTML=false`， `SortKeys=false`…) 保证性能同时兼顾安全性。 - `ConfigStd`: 与 `encoding/json` 保证完全兼容的配置 - `ConfigFastest`: 最快的配置(`NoQuoteTextMarshaler=true...`) 保证性能最优但是会缺少一些安全性检查（validate UTF8 等） -Sonic **不**确保支持所有环境，由于开发高性能代码的困难。在不支持声音的环境中，实现将回落到 `encoding/json`。因此上述配置将全部等于`ConfigStd`。 +Sonic **不**确保支持所有环境，由于开发高性能代码的困难。在不支持sonic的环境中，实现将回落到 `encoding/json`。因此上述配置将全部等于`ConfigStd`。 ## 注意事项 diff --git a/api.go b/api.go index af6be70a4..406715eca 100644 --- a/api.go +++ b/api.go @@ -77,7 +77,7 @@ type Config struct { // CopyString indicates decoder to decode string values by copying instead of referring. CopyString bool - // ValidateString indicates decoder and encoder to valid string values: decoder will return errors + // ValidateString indicates decoder and encoder to validate string values: decoder will return errors // when unescaped control chars(\u0000-\u001f) in the string value of JSON. ValidateString bool @@ -120,7 +120,7 @@ var ( // API is a binding of specific config. // This interface is inspired by github.com/json-iterator/go, -// and has same behaviors under equavilent config. +// and has same behaviors under equivalent config. type API interface { // MarshalToString returns the JSON encoding string of v MarshalToString(v interface{}) (string, error) diff --git a/ast/api_compat.go b/ast/api_compat.go index a349afc0b..6541e219d 100644 --- a/ast/api_compat.go +++ b/ast/api_compat.go @@ -34,7 +34,7 @@ func quote(buf *[]byte, val string) { quoteString(buf, val) } -// unquote unescapes a internal JSON string (it doesn't count quotas at the begining and end) +// unquote unescapes an internal JSON string (it doesn't count quotas at the beginning and end) func unquote(src string) (string, types.ParsingError) { sp := rt.IndexChar(src, -1) out, ok := unquoteBytes(rt.BytesFrom(sp, len(src)+2, len(src)+2)) diff --git a/ast/decode.go b/ast/decode.go index 6690d513e..135ee6eb8 100644 --- a/ast/decode.go +++ b/ast/decode.go @@ -17,16 +17,17 @@ package ast import ( - `encoding/base64` - `runtime` - `strconv` - `unsafe` - - `github.com/bytedance/sonic/internal/native/types` - `github.com/bytedance/sonic/internal/rt` + "encoding/base64" + "runtime" + "strconv" + "unsafe" + + "github.com/bytedance/sonic/internal/native/types" + "github.com/bytedance/sonic/internal/rt" + "github.com/bytedance/sonic/internal/utils" ) -// Hack: this is used for both checking space and cause firendly compile errors in 32-bit arch. +// Hack: this is used for both checking space and cause friendly compile errors in 32-bit arch. const _Sonic_Not_Support_32Bit_Arch__Checking_32Bit_Arch_Here = (1 << ' ') | (1 << '\t') | (1 << '\r') | (1 << '\n') var bytesNull = []byte("null") @@ -290,67 +291,7 @@ func decodeValue(src string, pos int, skipnum bool) (ret int, v types.JsonState) //go:nocheckptr func skipNumber(src string, pos int) (ret int) { - sp := uintptr(rt.IndexChar(src, pos)) - se := uintptr(rt.IndexChar(src, len(src))) - if uintptr(sp) >= se { - return -int(types.ERR_EOF) - } - - if c := *(*byte)(unsafe.Pointer(sp)); c == '-' { - sp += 1 - } - ss := sp - - var pointer bool - var exponent bool - var lastIsDigit bool - var nextNeedDigit = true - - for ; sp < se; sp += uintptr(1) { - c := *(*byte)(unsafe.Pointer(sp)) - if isDigit(c) { - lastIsDigit = true - nextNeedDigit = false - continue - } else if nextNeedDigit { - return -int(types.ERR_INVALID_CHAR) - } else if c == '.' { - if !lastIsDigit || pointer || exponent || sp == ss { - return -int(types.ERR_INVALID_CHAR) - } - pointer = true - lastIsDigit = false - nextNeedDigit = true - continue - } else if c == 'e' || c == 'E' { - if !lastIsDigit || exponent { - return -int(types.ERR_INVALID_CHAR) - } - if sp == se-1 { - return -int(types.ERR_EOF) - } - exponent = true - lastIsDigit = false - nextNeedDigit = false - continue - } else if c == '-' || c == '+' { - if prev := *(*byte)(unsafe.Pointer(sp - 1)); prev != 'e' && prev != 'E' { - return -int(types.ERR_INVALID_CHAR) - } - lastIsDigit = false - nextNeedDigit = true - continue - } else { - break - } - } - - if nextNeedDigit { - return -int(types.ERR_EOF) - } - - runtime.KeepAlive(src) - return int(uintptr(sp) - uintptr((*rt.GoString)(unsafe.Pointer(&src)).Ptr)) + return utils.SkipNumber(src, pos) } //go:nocheckptr diff --git a/ast/iterator.go b/ast/iterator.go index 076647154..1052dd0a0 100644 --- a/ast/iterator.go +++ b/ast/iterator.go @@ -173,7 +173,7 @@ type Scanner func(path Sequence, node *Node) bool // ForEach scans one V_OBJECT node's children from JSON head to tail, // and pass the Sequence and Node of corresponding JSON value. // -// Especailly, if the node is not V_ARRAY or V_OBJECT, +// Especially, if the node is not V_ARRAY or V_OBJECT, // the node itself will be returned and Sequence.Index == -1. // // NOTICE: A unsetted node WON'T trigger sc, but its index still counts into Path.Index diff --git a/ast/node.go b/ast/node.go index 0fbcf7835..2bb504850 100644 --- a/ast/node.go +++ b/ast/node.go @@ -140,7 +140,7 @@ func (self *Node) Check() error { // isRaw returns true if node's underlying value is raw json // -// Deprecated: not concurent safe +// Deprecated: not concurrent safe func (self Node) IsRaw() bool { return self.t & _V_RAW != 0 } @@ -440,7 +440,7 @@ func (self *Node) String() (string, error) { } } -// StrictString returns string value (unescaped), includeing V_STRING, V_ANY of string. +// StrictString returns string value (unescaped), including V_STRING, V_ANY of string. // In other cases, it will return empty string. func (self *Node) StrictString() (string, error) { if err := self.checkRaw(); err != nil { @@ -509,7 +509,7 @@ func (self *Node) Float64() (float64, error) { } } -// Float64 exports underlying float64 value, includeing V_NUMBER, V_ANY +// Float64 exports underlying float64 value, including V_NUMBER, V_ANY func (self *Node) StrictFloat64() (float64, error) { if err := self.checkRaw(); err != nil { return 0.0, err @@ -527,7 +527,7 @@ func (self *Node) StrictFloat64() (float64, error) { } } -/** Sequencial Value Methods **/ +/** Sequential Value Methods **/ // Len returns children count of a array|object|string node // WARN: For partially loaded node, it also works but only counts the parsed children @@ -611,7 +611,7 @@ func (self *Node) Unset(key string) (bool, error) { if err := self.should(types.V_OBJECT); err != nil { return false, err } - // NOTICE: must get acurate length before deduct + // NOTICE: must get accurate length before deduct if err := self.skipAllKey(); err != nil { return false, err } @@ -657,7 +657,7 @@ func (self *Node) SetAnyByIndex(index int, val interface{}) (bool, error) { return self.SetByIndex(index, NewAny(val)) } -// UnsetByIndex REOMVE (softly) the node of given index. +// UnsetByIndex REMOVE (softly) the node of given index. // // WARN: this will change address of elements, which is a dangerous action. // Use Unset() for object or Pop() for array instead. @@ -957,7 +957,7 @@ func (self *Node) MapUseNumber() (map[string]interface{}, error) { return self.toGenericObjectUseNumber() } -// MapUseNode scans both parsed and non-parsed chidren nodes, +// MapUseNode scans both parsed and non-parsed children nodes, // and map them by their keys func (self *Node) MapUseNode() (map[string]Node, error) { if self.isAny() { @@ -1102,7 +1102,7 @@ func (self *Node) ArrayUseNumber() ([]interface{}, error) { return self.toGenericArrayUseNumber() } -// ArrayUseNode copys both parsed and non-parsed chidren nodes, +// ArrayUseNode copies both parsed and non-parsed children nodes, // and indexes them by original order func (self *Node) ArrayUseNode() ([]Node, error) { if self.isAny() { @@ -1147,9 +1147,9 @@ func (self *Node) unsafeArray() (*linkedNodes, error) { return (*linkedNodes)(self.p), nil } -// Interface loads all children under all pathes from this node, +// Interface loads all children under all paths from this node, // and converts itself as generic type. -// WARN: all numberic nodes are casted to float64 +// WARN: all numeric nodes are casted to float64 func (self *Node) Interface() (interface{}, error) { if err := self.checkRaw(); err != nil { return nil, err @@ -1193,7 +1193,7 @@ func (self *Node) packAny() interface{} { } // InterfaceUseNumber works same with Interface() -// except numberic nodes are casted to json.Number +// except numeric nodes are casted to json.Number func (self *Node) InterfaceUseNumber() (interface{}, error) { if err := self.checkRaw(); err != nil { return nil, err diff --git a/decode_test.go b/decode_test.go index f3ed66ecc..53bcb3fd4 100644 --- a/decode_test.go +++ b/decode_test.go @@ -20,27 +20,28 @@ package sonic import ( - `bytes` - `encoding` - `encoding/json` - `errors` - `fmt` - `image` - `math` - `math/big` - `math/rand` - `net` - `reflect` - `strconv` - `strings` - `testing` - `time` - `unsafe` - - `github.com/bytedance/sonic/decoder` - `github.com/bytedance/sonic/internal/native/types` - `github.com/davecgh/go-spew/spew` - `github.com/stretchr/testify/assert` + "bytes" + "encoding" + "encoding/json" + "errors" + "fmt" + "image" + "math" + "math/big" + "math/rand" + "net" + "reflect" + "runtime" + "strconv" + "strings" + "testing" + "time" + "unsafe" + + "github.com/bytedance/sonic/decoder" + "github.com/bytedance/sonic/internal/native/types" + "github.com/davecgh/go-spew/spew" + "github.com/stretchr/testify/assert" ) type T struct { @@ -1008,6 +1009,8 @@ var unmarshalTests = []unmarshalTest{ ptr: new(map[string]json.Number), err: fmt.Errorf("json: invalid number literal, trying to unmarshal %q into Number", `"invalid"`), }, + + // UTF-8 and string validation tests {in: `\u`, ptr: new(interface{}), err: fmt.Errorf("json: invald char"), validateString: true}, {in: `\u`, ptr: new(string), err: fmt.Errorf("json: invald char"), validateString: true}, @@ -1019,6 +1022,17 @@ var unmarshalTests = []unmarshalTest{ {in: "\"\x00\"", ptr: new(string), out: "\x00", validateString: false}, {in: "\"\xff\"", ptr: new(interface{}), out: interface{}("\xff"), validateString: false}, {in: "\"\xff\"", ptr: new(string), out: "\xff", validateString: false}, + + // cases found by fuzz + { + in: `{"H":{"A": {}}}`, + ptr: new(struct { + F0 struct { + F1 json.Number "json:\"a,omitempty\"" + } "json:\"H,\"" + }), + err: fmt.Errorf("Mismatch type json.Number with value object.."), + }, } func trim(b []byte) []byte { @@ -2815,3 +2829,175 @@ func TestUseNumber(t *testing.T) { } } } + + +func BenchmarkDecoderRawMessage(b *testing.B) { + data := ` { + "coordinates": null, + "favorited": false, + "truncated": false, + "created_at": "Mon Sep 24 03:35:21 +0000 2012", + "id_str": "250075927172759552", + "entities": { + "urls": [ + + ], + "hashtags": [ + { + "text": "freebandnames", + "indices": [ + 20, + 34 + ] + } + ], + "user_mentions": [ + + ] + }, + "in_reply_to_user_id_str": null, + "contributors": null, + "text": "Aggressive Ponytail #freebandnames", + "metadata": { + "iso_language_code": "en", + "result_type": "recent" + }, + "retweet_count": 0, + "in_reply_to_status_id_str": null, + "id": 250075927172759552, + "geo": null, + "retweeted": false, + "in_reply_to_user_id": null, + "place": null, + "user": { + "profile_sidebar_fill_color": "DDEEF6", + "profile_sidebar_border_color": "C0DEED", + "profile_background_tile": false, + "name": "Sean Cummings", + "profile_image_url": "https://a0.twimg.com/profile_images/2359746665/1v6zfgqo8g0d3mk7ii5s_normal.jpeg", + "created_at": "Mon Apr 26 06:01:55 +0000 2010", + "location": "LA, CA", + "follow_request_sent": null, + "profile_link_color": "0084B4", + "is_translator": false, + "id_str": "137238150", + "entities": { + "url": { + "urls": [ + { + "expanded_url": null, + "url": "", + "indices": [ + 0, + 0 + ] + } + ] + }, + "description": { + "urls": [ + + ] + } + }, + "default_profile": true, + "contributors_enabled": false, + "favourites_count": 0, + "url": null, + "profile_image_url_https": "https://si0.twimg.com/profile_images/2359746665/1v6zfgqo8g0d3mk7ii5s_normal.jpeg", + "utc_offset": -28800, + "id": 137238150, + "profile_use_background_image": true, + "listed_count": 2, + "profile_text_color": "333333", + "lang": "en", + "followers_count": 70, + "protected": false, + "notifications": null, + "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", + "profile_background_color": "C0DEED", + "verified": false, + "geo_enabled": true, + "time_zone": "Pacific Time (US & Canada)", + "description": "Born 330 Live 310", + "default_profile_image": false, + "profile_background_image_url": "https://a0.twimg.com/images/themes/theme1/bg.png", + "statuses_count": 579, + "friends_count": 110, + "following": null, + "show_all_inline_media": false, + "screen_name": "sean_cummings" + }, + "in_reply_to_screen_name": null, + "source": "Twitter for Mac", + "in_reply_to_status_id": null + }` + + bench := func(b *testing.B, run func(b *testing.B) ) { + b.ResetTimer() + b.ReportAllocs() + b.SetBytes(int64(len(data))) + for i := 0; i < b.N; i++ { + run(b) + } + runtime.GC() + } + + b.Run("StdRawMessage", func(b *testing.B) { + bench(b, func(b *testing.B) { + var obj map[string]json.RawMessage + dc := decoder.NewDecoder(data) + dc.SetOptions(decoder.OptionUseNumber) + if err := dc.Decode(&obj); err!= nil { + b.Fatal(err.Error()) + } + _ = obj + }) + }) + + b.Run("NocopyRawMessage", func(b *testing.B) { + bench(b, func(b *testing.B) { + var obj map[string]NoCopyRawMessage + dc := decoder.NewDecoder(data) + dc.SetOptions(decoder.OptionUseNumber) + if err := dc.Decode(&obj); err!= nil { + b.Fatal(err.Error()) + } + _ = obj + }) + }) + + + b.Run("NocopyRawMessageWithoutValidation", func(b *testing.B) { + bench(b, func(b *testing.B) { + var obj map[string]NoCopyRawMessage + dc := decoder.NewDecoder(data) + dc.SetOptions(decoder.OptionNoValidateJSON | decoder.OptionUseNumber) + if err := dc.Decode(&obj); err!= nil { + b.Fatal(err.Error()) + } + _ = obj + }) + }) +} + + +func TestJsonNumber(t *testing.T) { + api := Config { + UseNumber: true, + }.Froze() + + + type Foo struct { + A json.Number `json:"a"` + B json.Number `json:"b"` + C json.Number `json:"c"` + } + + data := []byte(`{"a": 1 , "b": "123", "c": "0.4e+56"}`) + var foo1, foo2 Foo + serr := api.Unmarshal(data, &foo1) + jerr := json.Unmarshal(data, &foo2) + assert.Equal(t, jerr, serr) + assert.Equal(t, foo2, foo1) +} \ No newline at end of file diff --git a/decoder/decoder_compat.go b/decoder/decoder_compat.go index 81e1ae4e3..408d59394 100644 --- a/decoder/decoder_compat.go +++ b/decoder/decoder_compat.go @@ -1,3 +1,4 @@ +//go:build (!amd64 && !arm64) || go1.24 || !go1.17 || (arm64 && !go1.20) // +build !amd64,!arm64 go1.24 !go1.17 arm64,!go1.20 /* @@ -19,14 +20,15 @@ package decoder import ( - `bytes` - `encoding/json` - `io` - `reflect` - `unsafe` - - `github.com/bytedance/sonic/internal/native/types` - `github.com/bytedance/sonic/option` + "bytes" + "encoding/json" + "io" + "reflect" + "unsafe" + + "github.com/bytedance/sonic/internal/decoder/consts" + "github.com/bytedance/sonic/internal/native/types" + "github.com/bytedance/sonic/option" ) func init() { @@ -34,15 +36,16 @@ func init() { } const ( - _F_use_int64 = 0 - _F_disable_urc = 2 - _F_disable_unknown = 3 - _F_copy_string = 4 + _F_use_int64 = consts.F_use_int64 + _F_disable_urc = consts.F_disable_unknown + _F_disable_unknown = consts.F_disable_unknown + _F_copy_string = consts.F_copy_string - _F_use_number = types.B_USE_NUMBER - _F_validate_string = types.B_VALIDATE_STRING - _F_allow_control = types.B_ALLOW_CONTROL - _F_no_validate_json = types.B_NO_VALIDATE_JSON + _F_use_number = consts.F_use_number + _F_validate_string = consts.F_validate_string + _F_allow_control = consts.F_allow_control + _F_no_validate_json = consts.F_no_validate_json + _F_case_sensitive = consts.F_case_sensitive ) type Options uint64 @@ -55,6 +58,7 @@ const ( OptionCopyString Options = 1 << _F_copy_string OptionValidateString Options = 1 << _F_validate_string OptionNoValidateJSON Options = 1 << _F_no_validate_json + OptionCaseSensitive Options = 1 << _F_case_sensitive ) func (self *Decoder) SetOptions(opts Options) { @@ -192,5 +196,5 @@ func (s SyntaxError) Error() string { return (*json.SyntaxError)(unsafe.Pointer(&s)).Error() } -// MismatchTypeError represents dismatching between json and object -type MismatchTypeError json.UnmarshalTypeError \ No newline at end of file +// MismatchTypeError represents mismatching between json and object +type MismatchTypeError json.UnmarshalTypeError diff --git a/decoder/decoder_native.go b/decoder/decoder_native.go index 9317d57f6..bf71e1bd4 100644 --- a/decoder/decoder_native.go +++ b/decoder/decoder_native.go @@ -30,7 +30,7 @@ type Decoder = api.Decoder // SyntaxError represents json syntax error type SyntaxError = api.SyntaxError -// MismatchTypeError represents dismatching between json and object +// MismatchTypeError represents mismatching between json and object type MismatchTypeError = api.MismatchTypeError // Options for decode. @@ -44,6 +44,7 @@ const ( OptionCopyString Options = api.OptionCopyString OptionValidateString Options = api.OptionValidateString OptionNoValidateJSON Options = api.OptionNoValidateJSON + OptionCaseSensitive Options = api.OptionCaseSensitive ) // StreamDecoder is the decoder context object for streaming input. diff --git a/decoder/decoder_native_test.go b/decoder/decoder_native_test.go index 9b435ca92..62e85b52a 100644 --- a/decoder/decoder_native_test.go +++ b/decoder/decoder_native_test.go @@ -33,6 +33,67 @@ import ( "github.com/stretchr/testify/require" ) +func TestDecoder_OptionCaseSensitive(t *testing.T) { + var js = `{"a":1,"normallllll":1,"longllllllllllllllllllllllllllllllllll":1}` + type TS struct{ + A int + Normallllll int + Longllllllllllllllllllllllllllllllllll int + } + var obj = TS{} + d := NewDecoder(js) + d.SetOptions(OptionCaseSensitive) + err := d.Decode(&obj) + require.NoError(t, err) + require.Equal(t, TS{}, obj) +} + + +func TestDecoder_MapWithIndirectElement(t *testing.T) { + var v map[string]struct { A [129]byte } + _, err := decode(`{"":{"A":[1,2,3,4,5]}}`, &v, false) + require.NoError(t, err) + assert.Equal(t, [129]byte{1, 2, 3, 4, 5}, v[""].A) +} + +func TestDecoder_OptionCaseSensitiveForManyKeys(t *testing.T) { + var js = `{"a":1,"b":2,"C":3,"DD":4,"eE":5,"fF":6,"G":7,"H":8,"I":9,"J":10,"K":11,"L":12,"M":13}` + type TS struct{ + A int + B int + C int `json:"c"` + Dd int `json:"dd"` + Ee int `json:"ee"` + Ff int `json:"Ff"` + G int `json:"g"` + H int `json:"h"` + I int `json:"i"` + J int `json:"j"` + K int `json:"k"` + L int `json:"l"` + M int `json:"m"` + } + + { + var obj = TS{} + err := json.Unmarshal([]byte(js), &obj) + require.NoError(t, err) + + var obj2 = TS{} + d := NewDecoder(js) + err2 := d.Decode(&obj2) + require.NoError(t, err2) + require.Equal(t, obj, obj2) + } + + var obj = TS{} + d := NewDecoder(js) + d.SetOptions(OptionCaseSensitive) + err := d.Decode(&obj) + require.NoError(t, err) + require.Equal(t, TS{}, obj) +} + func BenchmarkSkipValidate(b *testing.B) { type skiptype struct { diff --git a/decoder/decoder_test.go b/decoder/decoder_test.go index c58ed97a5..6da755857 100644 --- a/decoder/decoder_test.go +++ b/decoder/decoder_test.go @@ -26,7 +26,6 @@ import ( "time" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func TestMain(m *testing.M) { @@ -287,14 +286,6 @@ func TestDecoder_Binding(t *testing.T) { assert.Equal(t, _BindingValue, v, 0) } - -func TestDecoder_MapWithIndirectElement(t *testing.T) { - var v map[string]struct { A [129]byte } - _, err := decode(`{"":{"A":[1,2,3,4,5]}}`, &v, false) - require.NoError(t, err) - assert.Equal(t, [129]byte{1, 2, 3, 4, 5}, v[""].A) -} - func BenchmarkDecoder_Generic_Sonic(b *testing.B) { var w interface{} _, _ = decode(TwitterJson, &w, true) diff --git a/docs/INTRODUCTION.md b/docs/INTRODUCTION.md index 2ec8e4880..4266c46e6 100644 --- a/docs/INTRODUCTION.md +++ b/docs/INTRODUCTION.md @@ -14,7 +14,7 @@ Therefore, we decided to **develop a brand-new JSON library with high performanc Before starting our design, we need to figure out some questions: ### Why is Json-iterator faster than Standard Library? -First of all, the **schema-based processing mechanism** used by the standard library is commendable, in which the parser can obtain meta information in advance when scanning, thereby shortening the time of branch selection. However, its original implementation did not make good use of this mechanism, instead, **it spent a lot of time reflecting to obtain meta info of schema**. Meanwhile, The approach of json-iterator is: Interprete structure as field-by-field encoding and decoding functions, and then assembled and cached them, minimizing the performance loss cost by reflection. But does it work once and for all? No. In practical tests, we found that **the deeper and larger the input JSON got, the smaller the gap between json-iterator and other libraries gradually became** - eventually event got surpassed: +First of all, the **schema-based processing mechanism** used by the standard library is commendable, in which the parser can obtain meta information in advance when scanning, thereby shortening the time of branch selection. However, its original implementation did not make good use of this mechanism, instead, **it spent a lot of time reflecting to obtain meta info of schema**. Meanwhile, The approach of json-iterator is: Interpret structure as field-by-field encoding and decoding functions, and then assembled and cached them, minimizing the performance loss cost by reflection. But does it work once and for all? No. In practical tests, we found that **the deeper and larger the input JSON got, the smaller the gap between json-iterator and other libraries gradually became** - eventually event got surpassed: ![Scalability](./imgs/introduction-1.png) The reason is that **this implementation transforms into a large number of interface encapsulations and function calls**, followed by function-call losses: diff --git a/external_jsonlib_test/go.mod b/external_jsonlib_test/go.mod index 38f3958c3..8d4b1a531 100644 --- a/external_jsonlib_test/go.mod +++ b/external_jsonlib_test/go.mod @@ -13,9 +13,8 @@ require ( ) require ( - github.com/bytedance/sonic/loader v0.2.0 // indirect + github.com/bytedance/sonic/loader v0.2.2 // indirect github.com/cloudwego/base64x v0.1.4 // indirect - github.com/cloudwego/iasm v0.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect diff --git a/external_jsonlib_test/go.sum b/external_jsonlib_test/go.sum index 5ded92db3..aa13da7ab 100644 --- a/external_jsonlib_test/go.sum +++ b/external_jsonlib_test/go.sum @@ -1,11 +1,10 @@ github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= -github.com/bytedance/sonic/loader v0.2.0 h1:zNprn+lsIP06C/IqCHs3gPQIvnvpKbbxyXQP1iU4kWM= -github.com/bytedance/sonic/loader v0.2.0/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/bytedance/sonic/loader v0.2.2 h1:jxAJuN9fOot/cyz5Q6dUuMJF5OqQ6+5GfA8FjjQ0R4o= +github.com/bytedance/sonic/loader v0.2.2/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= -github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= diff --git a/fuzz/go.mod b/fuzz/go.mod index 2d3dd82ef..30698a7c8 100644 --- a/fuzz/go.mod +++ b/fuzz/go.mod @@ -10,9 +10,8 @@ require ( ) require ( - github.com/bytedance/sonic/loader v0.2.0 // indirect + github.com/bytedance/sonic/loader v0.2.2 // indirect github.com/cloudwego/base64x v0.1.4 // indirect - github.com/cloudwego/iasm v0.2.0 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect diff --git a/fuzz/go.sum b/fuzz/go.sum index 85bc056df..8bcc2570f 100644 --- a/fuzz/go.sum +++ b/fuzz/go.sum @@ -1,11 +1,10 @@ github.com/bytedance/gopkg v0.0.0-20221122125632-68358b8ecec6 h1:FCLDGi1EmB7JzjVVYNZiqc/zAJj2BQ5M0lfkVOxbfs8= github.com/bytedance/gopkg v0.0.0-20221122125632-68358b8ecec6/go.mod h1:5FoAH5xUHHCMDvQPy1rnj8moqLkLHFaDVBjHhcFwEi0= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= -github.com/bytedance/sonic/loader v0.2.0 h1:zNprn+lsIP06C/IqCHs3gPQIvnvpKbbxyXQP1iU4kWM= -github.com/bytedance/sonic/loader v0.2.0/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/bytedance/sonic/loader v0.2.2 h1:jxAJuN9fOot/cyz5Q6dUuMJF5OqQ6+5GfA8FjjQ0R4o= +github.com/bytedance/sonic/loader v0.2.2/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= -github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= diff --git a/generic_test/go.mod b/generic_test/go.mod index f565dca8c..20693cc24 100644 --- a/generic_test/go.mod +++ b/generic_test/go.mod @@ -10,9 +10,8 @@ require ( ) require ( - github.com/bytedance/sonic/loader v0.2.0 // indirect + github.com/bytedance/sonic/loader v0.2.2 // indirect github.com/cloudwego/base64x v0.1.4 // indirect - github.com/cloudwego/iasm v0.2.0 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect github.com/modern-go/reflect2 v1.0.2 // indirect diff --git a/generic_test/go.sum b/generic_test/go.sum index 1eb65c6e2..eccc76d2e 100644 --- a/generic_test/go.sum +++ b/generic_test/go.sum @@ -1,9 +1,8 @@ github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= -github.com/bytedance/sonic/loader v0.2.0 h1:zNprn+lsIP06C/IqCHs3gPQIvnvpKbbxyXQP1iU4kWM= -github.com/bytedance/sonic/loader v0.2.0/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/bytedance/sonic/loader v0.2.2 h1:jxAJuN9fOot/cyz5Q6dUuMJF5OqQ6+5GfA8FjjQ0R4o= +github.com/bytedance/sonic/loader v0.2.2/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= -github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= diff --git a/go.mod b/go.mod index c323a7cf9..fa2229850 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/bytedance/sonic go 1.17 require ( - github.com/bytedance/sonic/loader v0.2.0 + github.com/bytedance/sonic/loader v0.2.2 github.com/cloudwego/base64x v0.1.4 github.com/davecgh/go-spew v1.1.1 github.com/klauspost/cpuid/v2 v2.0.9 @@ -13,7 +13,6 @@ require ( ) require ( - github.com/cloudwego/iasm v0.2.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index ee7f50c6a..66352e717 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,8 @@ github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= -github.com/bytedance/sonic/loader v0.2.0 h1:zNprn+lsIP06C/IqCHs3gPQIvnvpKbbxyXQP1iU4kWM= -github.com/bytedance/sonic/loader v0.2.0/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/bytedance/sonic/loader v0.2.2 h1:jxAJuN9fOot/cyz5Q6dUuMJF5OqQ6+5GfA8FjjQ0R4o= +github.com/bytedance/sonic/loader v0.2.2/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= -github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= diff --git a/go.work.sum b/go.work.sum index 8ee8d8cc5..a451e8f31 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1,2 +1,4 @@ -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= -golang.org/x/sys v0.0.0-20221010170243-090e33056c14 h1:k5II8e6QD8mITdi+okbbmR/cIyEbeXLBhy5Ha4nevyc= +github.com/bytedance/sonic/loader v0.2.0/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/bytedance/sonic/loader v0.2.2/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= +github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= + diff --git a/internal/decoder/api/decoder.go b/internal/decoder/api/decoder.go index 0dc01998f..bae0584c2 100644 --- a/internal/decoder/api/decoder.go +++ b/internal/decoder/api/decoder.go @@ -35,6 +35,7 @@ const ( _F_use_int64 = consts.F_use_int64 _F_use_number = consts.F_use_number _F_validate_string = consts.F_validate_string + _F_case_sensitive = consts.F_case_sensitive _MaxStack = consts.MaxStack @@ -45,6 +46,7 @@ const ( OptionCopyString = consts.OptionCopyString OptionValidateString = consts.OptionValidateString OptionNoValidateJSON = consts.OptionNoValidateJSON + OptionCaseSensitive = consts.OptionCaseSensitive ) type ( diff --git a/internal/decoder/api/decoder_arm64.go b/internal/decoder/api/decoder_arm64.go index 65a9478b4..16e55965a 100644 --- a/internal/decoder/api/decoder_arm64.go +++ b/internal/decoder/api/decoder_arm64.go @@ -30,7 +30,7 @@ var ( func init() { - // whe in aarch64. we enable all optimize + // when in aarch64, we enable all optimization envs.EnableOptDec() envs.EnableFastMap() } diff --git a/internal/decoder/consts/option.go b/internal/decoder/consts/option.go index 4195ebda7..f8c49d500 100644 --- a/internal/decoder/consts/option.go +++ b/internal/decoder/consts/option.go @@ -12,11 +12,11 @@ const ( F_disable_unknown = 3 F_copy_string = 4 - F_use_number = types.B_USE_NUMBER F_validate_string = types.B_VALIDATE_STRING F_allow_control = types.B_ALLOW_CONTROL F_no_validate_json = types.B_NO_VALIDATE_JSON + F_case_sensitive = 7 ) type Options uint64 @@ -29,6 +29,7 @@ const ( OptionCopyString Options = 1 << F_copy_string OptionValidateString Options = 1 << F_validate_string OptionNoValidateJSON Options = 1 << F_no_validate_json + OptionCaseSensitive Options = 1 << F_case_sensitive ) const ( diff --git a/internal/decoder/jitdec/assembler_regabi_amd64.go b/internal/decoder/jitdec/assembler_regabi_amd64.go index 4ff3b1962..04e854df7 100644 --- a/internal/decoder/jitdec/assembler_regabi_amd64.go +++ b/internal/decoder/jitdec/assembler_regabi_amd64.go @@ -67,7 +67,7 @@ import ( */ const ( - _FP_args = 72 // 72 bytes to pass and spill register arguements + _FP_args = 72 // 72 bytes to pass and spill register arguments _FP_fargs = 80 // 80 bytes for passing arguments to other Go functions _FP_saves = 48 // 48 bytes for saving the registers before CALL instructions _FP_locals = 144 // 144 bytes for local variables @@ -203,9 +203,9 @@ var ( var _VAR_fl = jit.Ptr(_SP, _FP_fargs + _FP_saves + 112) var ( - _VAR_et = jit.Ptr(_SP, _FP_fargs + _FP_saves + 120) // save dismatched type + _VAR_et = jit.Ptr(_SP, _FP_fargs + _FP_saves + 120) // save mismatched type _VAR_pc = jit.Ptr(_SP, _FP_fargs + _FP_saves + 128) // save skip return pc - _VAR_ic = jit.Ptr(_SP, _FP_fargs + _FP_saves + 136) // save dismatched position + _VAR_ic = jit.Ptr(_SP, _FP_fargs + _FP_saves + 136) // save mismatched position ) type _Assembler struct { @@ -483,6 +483,7 @@ var ( _V_stackOverflow = jit.Imm(int64(uintptr(unsafe.Pointer(&stackOverflow)))) _I_json_UnsupportedValueError = jit.Itab(_T_error, reflect.TypeOf(new(json.UnsupportedValueError))) _I_json_MismatchTypeError = jit.Itab(_T_error, reflect.TypeOf(new(MismatchTypeError))) + _I_json_MismatchQuotedError = jit.Itab(_T_error, reflect.TypeOf(new(MismatchQuotedError))) ) func (self *_Assembler) type_error() { @@ -1129,15 +1130,19 @@ func (self *_Assembler) unmarshal_func(t reflect.Type, fn obj.Addr, deref bool) self.Emit("MOVQ" , _ARG_sv_n, _DI) // MOVQ sv.n, DI self.call_go(fn) // CALL_GO ${fn} self.Emit("TESTQ", _ET, _ET) // TESTQ ET, ET - self.Sjmp("JZ" , "_unmarshal_func_end_{n}") // JNZ _error - self.Emit("MOVQ", _I_json_MismatchTypeError, _CX) // MOVQ ET, VAR.et - self.Emit("CMPQ", _ET, _CX) // check if MismatchedError - self.Sjmp("JNE" , _LB_error) - self.Emit("MOVQ", jit.Type(t), _CX) // store current type - self.Emit("MOVQ", _CX, _VAR_et) // store current type - self.Emit("MOVQ", _VAR_ic, _IC) // recover the pos - self.Emit("XORL", _ET, _ET) - self.Link("_unmarshal_func_end_{n}") + if fn == _F_decodeJsonUnmarshalerQuoted { + self.Sjmp("JZ" , "_unmarshal_func_end_{n}") // JZ _unmarshal_func_end_{n} + self.Emit("MOVQ", _I_json_MismatchQuotedError, _CX) // MOVQ _I_json_MismatchQuotedError, CX + self.Emit("CMPQ", _ET, _CX) // check if MismatchQuotedError + self.Sjmp("JNE" , _LB_error) // JNE _error + self.Emit("MOVQ", jit.Type(t), _CX) // store current type + self.Emit("MOVQ", _CX, _VAR_et) // store current type as mismatched type + self.Emit("MOVQ", _VAR_ic, _IC) // recover the pos at mismatched, continue to parse + self.Emit("XORL", _ET, _ET) // clear ET + self.Link("_unmarshal_func_end_{n}") + } else { + self.Sjmp("JNE" , _LB_error) // JNE _error + } } /** Dynamic Decoding Routine **/ @@ -1356,7 +1361,7 @@ func (self *_Assembler) _asm_OP_num(_ *_Instr) { self.Emit("MOVQ", _R9, _VAR_pc) self.Sjmp("JMP" , _LB_skip_one) - /* assgin string */ + /* assign string */ self.Link("_num_next_{n}") self.slice_from_r(_AX, 0) self.Emit("BTQ", jit.Imm(_F_copy_string), _ARG_fv) @@ -1789,6 +1794,8 @@ func (self *_Assembler) _asm_OP_struct_field(p *_Instr) { self.Emit("MOVQ" , _R8, _VAR_sr) // MOVQ R8, sr self.Sjmp("JMP" , "_end_{n}") // JMP _end_{n} self.Link("_try_lowercase_{n}") // _try_lowercase_{n}: + self.Emit("BTQ" , jit.Imm(_F_case_sensitive), _ARG_fv) // check if enable option CaseSensitive + self.Sjmp("JC" , "_unknown_{n}") self.Emit("MOVQ" , jit.Imm(referenceFields(p.vf())), _AX) // MOVQ ${p.vf()}, AX self.Emit("MOVQ", _ARG_sv_p, _BX) // MOVQ sv, BX self.Emit("MOVQ", _ARG_sv_n, _CX) // MOVQ sv, CX @@ -1796,6 +1803,10 @@ func (self *_Assembler) _asm_OP_struct_field(p *_Instr) { self.Emit("MOVQ" , _AX, _VAR_sr) // MOVQ AX, _VAR_sr self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX self.Sjmp("JNS" , "_end_{n}") // JNS _end_{n} + self.Link("_unknown_{n}") + // HACK: because `_VAR_sr` maybe used in `F_vstring`, so we should clear here again for `_OP_switch`. + self.Emit("MOVQ" , jit.Imm(-1), _AX) // MOVQ $-1, AX + self.Emit("MOVQ" , _AX, _VAR_sr) // MOVQ AX, sr self.Emit("BTQ" , jit.Imm(_F_disable_unknown), _ARG_fv) // BTQ ${_F_disable_unknown}, fv self.Sjmp("JC" , _LB_field_error) // JC _field_error self.Link("_end_{n}") // _end_{n}: diff --git a/internal/decoder/jitdec/compiler.go b/internal/decoder/jitdec/compiler.go index a097d171d..33191262a 100644 --- a/internal/decoder/jitdec/compiler.go +++ b/internal/decoder/jitdec/compiler.go @@ -736,7 +736,7 @@ func (self *_Compiler) compilePtr(p *_Program, sp int, et reflect.Type) { self.tab[et] = true /* not inline the pointer type - * recursing the defined pointer type's elem will casue issue379. + * recursing the defined pointer type's elem will cause issue379. */ self.compileOps(p, sp, et) } diff --git a/internal/decoder/jitdec/decoder.go b/internal/decoder/jitdec/decoder.go index bbb4b4b61..de6325289 100644 --- a/internal/decoder/jitdec/decoder.go +++ b/internal/decoder/jitdec/decoder.go @@ -27,6 +27,7 @@ const ( _F_use_number = consts.F_use_number _F_no_validate_json = consts.F_no_validate_json _F_validate_string = consts.F_validate_string + _F_case_sensitive = consts.F_case_sensitive ) var ( diff --git a/internal/decoder/jitdec/pools.go b/internal/decoder/jitdec/pools.go index 01868cb2f..200af6a60 100644 --- a/internal/decoder/jitdec/pools.go +++ b/internal/decoder/jitdec/pools.go @@ -63,8 +63,8 @@ type _Decoder func( vp unsafe.Pointer, sb *_Stack, fv uint64, - sv string, // DO NOT pass value to this arguement, since it is only used for local _VAR_sv - vk unsafe.Pointer, // DO NOT pass value to this arguement, since it is only used for local _VAR_vk + sv string, // DO NOT pass value to this argument, since it is only used for local _VAR_sv + vk unsafe.Pointer, // DO NOT pass value to this argument, since it is only used for local _VAR_vk ) (int, error) var _KeepAlive struct { diff --git a/internal/decoder/jitdec/primitives.go b/internal/decoder/jitdec/primitives.go index 5adfc038a..9de885007 100644 --- a/internal/decoder/jitdec/primitives.go +++ b/internal/decoder/jitdec/primitives.go @@ -39,9 +39,16 @@ func decodeJsonUnmarshaler(vv interface{}, s string) error { return vv.(json.Unmarshaler).UnmarshalJSON(rt.Str2Mem(s)) } +// used to distinguish between MismatchQuoted and other MismatchedTyped errors, see issue #670 and #716 +type MismatchQuotedError struct {} + +func (*MismatchQuotedError) Error() string { + return "mismatch quoted" +} + func decodeJsonUnmarshalerQuoted(vv interface{}, s string) error { if len(s) < 2 || s[0] != '"' || s[len(s)-1] != '"' { - return &MismatchTypeError{} + return &MismatchQuotedError{} } return vv.(json.Unmarshaler).UnmarshalJSON(rt.Str2Mem(s[1:len(s)-1])) } diff --git a/internal/decoder/optdec/compiler.go b/internal/decoder/optdec/compiler.go index fd164af93..7d9d60a01 100644 --- a/internal/decoder/optdec/compiler.go +++ b/internal/decoder/optdec/compiler.go @@ -177,7 +177,7 @@ func (c *compiler) compilePtr(vt reflect.Type) decFunc { c.enter(vt) defer c.exit(vt) - // specail logic for Named Ptr, issue 379 + // special logic for Named Ptr, issue 379 if reflect.PtrTo(vt.Elem()) != vt { c.namedPtr = true return &ptrDecoder{ @@ -432,7 +432,7 @@ func (c *compiler) tryCompilePtrUnmarshaler(vt reflect.Type, strOpt bool) decFun /* check for `encoding.TextMarshaler` with pointer receiver */ if pt.Implements(encodingTextUnmarshalerType) { - /* TextUnmarshal not support ,strig tag */ + /* TextUnmarshal not support, string tag */ if strOpt { panicForInvalidStrType(vt) } diff --git a/internal/decoder/optdec/helper.go b/internal/decoder/optdec/helper.go index 1d76f8051..61faa6c80 100644 --- a/internal/decoder/optdec/helper.go +++ b/internal/decoder/optdec/helper.go @@ -5,42 +5,51 @@ import ( "strconv" "github.com/bytedance/sonic/internal/native" + "github.com/bytedance/sonic/internal/utils" "github.com/bytedance/sonic/internal/native/types" ) -func SkipNumberFast(json string, start int) (int, error) { - // find the number ending, we pasred in sonic-cpp, it alway valid +func SkipNumberFast(json string, start int) (int, bool) { + // find the number ending, we parsed in native, it always valid pos := start for pos < len(json) && json[pos] != ']' && json[pos] != '}' && json[pos] != ',' { if json[pos] >= '0' && json[pos] <= '9' || json[pos] == '.' || json[pos] == '-' || json[pos] == '+' || json[pos] == 'e' || json[pos] == 'E' { pos += 1 } else { - return pos, error_syntax(pos, json, "invalid number") + break } } - return pos, nil + + // if not found number, return false + if pos == start { + return pos, false + } + return pos, true } -func ValidNumberFast(json string) error { - // find the number ending, we pasred in sonic-cpp, it alway valid - pos := 0 - for pos < len(json) && json[pos] != ']' && json[pos] != '}' && json[pos] != ',' { - if json[pos] >= '0' && json[pos] <= '9' || json[pos] == '.' || json[pos] == '-' || json[pos] == '+' || json[pos] == 'e' || json[pos] == 'E' { - pos += 1 - } else { - return error_syntax(pos, json, "invalid number") - } + +func isSpace(c byte) bool { + return c == ' ' || c == '\t' || c == '\n' || c == '\r' +} + +// pos is the start index of the raw +func ValidNumberFast(raw string) bool { + ret := utils.SkipNumber(raw, 0) + if ret < 0 { + return false } - if pos == 0 { - return error_syntax(pos, json, "invalid number") + // check trailing chars + for ret < len(raw) { + return false } - return nil + + return true } func SkipOneFast2(json string, pos *int) (int, error) { - // find the number ending, we pasred in sonic-cpp, it alway valid + // find the number ending, we parsed in sonic-cpp, it always valid start := native.SkipOneFast(&json, pos) if start < 0 { return -1, error_syntax(*pos, json, types.ParsingError(-start).Error()) @@ -49,7 +58,7 @@ func SkipOneFast2(json string, pos *int) (int, error) { } func SkipOneFast(json string, pos int) (string, error) { - // find the number ending, we pasred in sonic-cpp, it alway valid + // find the number ending, we parsed in sonic-cpp, it always valid start := native.SkipOneFast(&json, &pos) if start < 0 { // TODO: details error code diff --git a/internal/decoder/optdec/native.go b/internal/decoder/optdec/native.go index 29a0136ae..5dadec0b7 100644 --- a/internal/decoder/optdec/native.go +++ b/internal/decoder/optdec/native.go @@ -61,13 +61,13 @@ type node struct { val uint64 } -// should consitent with native/parser.c +// should consistent with native/parser.c type _nospaceBlock struct { _ [8]byte _ [8]byte } -// should consitent with native/parser.c +// should consistent with native/parser.c type nodeBuf struct { ncur uintptr parent int64 @@ -84,7 +84,7 @@ func (self *nodeBuf) init(nodes []node) { self.parent = -1 } -// should consitent with native/parser.c +// should consistent with native/parser.c type Parser struct { Json string padded []byte diff --git a/internal/decoder/optdec/node.go b/internal/decoder/optdec/node.go index 8b49ebb3a..b23901e38 100644 --- a/internal/decoder/optdec/node.go +++ b/internal/decoder/optdec/node.go @@ -372,7 +372,7 @@ func (val Node) ParseF64(ctx *Context) (float64, bool) { } func (val Node) ParseString(ctx *Context) (string, bool) { - // shoud not use AsStrRef + // should not use AsStrRef s, ok := val.AsStr(ctx) if !ok { return "", false @@ -391,7 +391,7 @@ func (val Node) ParseString(ctx *Context) (string, bool) { func (val Node) ParseNumber(ctx *Context) (json.Number, bool) { - // shoud not use AsStrRef + // should not use AsStrRef s, ok := val.AsStr(ctx) if !ok { return json.Number(""), false @@ -401,9 +401,9 @@ func (val Node) ParseNumber(ctx *Context) (json.Number, bool) { return json.Number(""), true } - end, err := SkipNumberFast(s, 0) + end, ok := SkipNumberFast(s, 0) // has error or trailing chars - if err != nil || end != len(s) { + if !ok || end != len(s) { return json.Number(""), false } return json.Number(s), true @@ -509,12 +509,11 @@ func (val Node) AsNumber(ctx *Context) (json.Number, bool) { // parse JSON string as number if val.IsStr() { s, _ := val.AsStr(ctx) - err := ValidNumberFast(s) - if err != nil { + if !ValidNumberFast(s) { return "", false + } else { + return json.Number(s), true } - - return json.Number(s), true } return val.NonstrAsNumber(ctx) @@ -532,8 +531,8 @@ func (val Node) NonstrAsNumber(ctx *Context) (json.Number, bool) { } start := val.Position() - end, err := SkipNumberFast(ctx.Parser.Json, start) - if err != nil { + end, ok := SkipNumberFast(ctx.Parser.Json, start) + if !ok { return "", false } return json.Number(ctx.Parser.Json[start:end]), true diff --git a/internal/decoder/optdec/stringopts.go b/internal/decoder/optdec/stringopts.go index 627b5ebea..5af8c97e2 100644 --- a/internal/decoder/optdec/stringopts.go +++ b/internal/decoder/optdec/stringopts.go @@ -349,9 +349,9 @@ func (d *numberStringDecoder) FromDom(vp unsafe.Pointer, node Node, ctx *context return error_mismatch(node, ctx, jsonNumberType) } - end, err := SkipNumberFast(s, 0) + end, ok := SkipNumberFast(s, 0) // has error or trailing chars - if err != nil || end != len(s) { + if !ok || end != len(s) { return error_mismatch(node, ctx, jsonNumberType) } diff --git a/internal/decoder/optdec/structs.go b/internal/decoder/optdec/structs.go index bce2758f1..8b148eadf 100644 --- a/internal/decoder/optdec/structs.go +++ b/internal/decoder/optdec/structs.go @@ -4,6 +4,7 @@ import ( "reflect" "unsafe" + "github.com/bytedance/sonic/internal/decoder/consts" caching "github.com/bytedance/sonic/internal/optcaching" "github.com/bytedance/sonic/internal/resolver" ) @@ -38,7 +39,7 @@ func (d *structDecoder) FromDom(vp unsafe.Pointer, node Node, ctx *context) erro next = val.Next() // find field idx - idx := d.fieldMap.Get(key) + idx := d.fieldMap.Get(key, ctx.Options()&uint64(consts.OptionCaseSensitive) != 0) if idx == -1 { if Options(ctx.Options())&OptionDisableUnknown != 0 { return error_field(key) diff --git a/internal/encoder/alg/sort.go b/internal/encoder/alg/sort.go index 9b69bce9a..5bb0f9011 100644 --- a/internal/encoder/alg/sort.go +++ b/internal/encoder/alg/sort.go @@ -46,7 +46,7 @@ func radixQsort(kvs []_MapPair, d, maxDepth int) { } // kvs[0:lt] < v = kvs[lt:gt] < kvs[gt:len(kvs)] - // Native implemention: + // Native implementation: // radixQsort(kvs[:lt], d, maxDepth) // if p > -1 { // radixQsort(kvs[lt:gt], d+1, maxDepth) diff --git a/internal/encoder/x86/assembler_regabi_amd64.go b/internal/encoder/x86/assembler_regabi_amd64.go index 75e3ecde6..c0912fb81 100644 --- a/internal/encoder/x86/assembler_regabi_amd64.go +++ b/internal/encoder/x86/assembler_regabi_amd64.go @@ -673,7 +673,7 @@ func (self *Assembler) encode_string(doubleQuote bool) { self.Sjmp("JMP", _LB_panic) self.Link("_str_next_{n}") - /* openning quote, check for double quote */ + /* opening quote, check for double quote */ if !doubleQuote { self.check_size_r(_AX, 2) // SIZE $2 self.add_char('"') // CHAR $'"' @@ -1050,8 +1050,9 @@ func (self *Assembler) _asm_OP_recurse(p *ir.Instr) { self.Emit("MOVQ", _ST, _DI) // MOVQ ST, DI self.Emit("MOVQ", _ARG_fv, _SI) // MOVQ $fv, SI if pv { - self.Emit("BTCQ", jit.Imm(alg.BitPointerValue), _SI) // BTCQ $1, SI + self.Emit("BTSQ", jit.Imm(alg.BitPointerValue), _SI) // BTSQ $1, SI } + self.call_encoder(_F_encodeTypedPointer) // CALL encodeTypedPointer self.Emit("TESTQ", _ET, _ET) // TESTQ ET, ET self.Sjmp("JNZ", _LB_error) // JNZ _error diff --git a/internal/native/neon/f32toa_arm64.s b/internal/native/neon/f32toa_arm64.s index a4f74da8a..64eed6d71 100644 --- a/internal/native/neon/f32toa_arm64.s +++ b/internal/native/neon/f32toa_arm64.s @@ -916,7 +916,7 @@ _Digits: WORD $0x37393639 // .ascii 4, '96979899' WORD $0x39393839 // .ascii 4, '9899' WORD $0x00000000 // .p2align 3, 0x00 -_LB_15828841: // _pow10_ceil_sig_f32.g +_LB_3c57fe76: // _pow10_ceil_sig_f32.g WORD $0x4b43fcf5; WORD $0x81ceb32c // .quad -9093133594791772939 WORD $0x5e14fc32; WORD $0xa2425ff7 // .quad -6754730975062328270 WORD $0x359a3b3f; WORD $0xcad2f7f5 // .quad -3831727700400522433 diff --git a/internal/native/neon/f64toa_arm64.s b/internal/native/neon/f64toa_arm64.s index d1e7f08b8..12bf45299 100644 --- a/internal/native/neon/f64toa_arm64.s +++ b/internal/native/neon/f64toa_arm64.s @@ -1232,7 +1232,7 @@ _Digits: WORD $0x37393639 // .ascii 4, '96979899' WORD $0x39393839 // .ascii 4, '9899' // .p2align 3, 0x00 -_LB_3b41de77: // _pow10_ceil_sig.g +_LB_f262bdcb: // _pow10_ceil_sig.g WORD $0xbebcdc4f; WORD $0xff77b1fc // .quad -38366372719436721 WORD $0x13bb0f7b; WORD $0x25e8e89c // .quad 2731688931043774331 WORD $0xf73609b1; WORD $0x9faacf3d // .quad -6941508010590729807 diff --git a/internal/native/neon/lookup_small_key_arm64.s b/internal/native/neon/lookup_small_key_arm64.s index fb856f144..004c09374 100644 --- a/internal/native/neon/lookup_small_key_arm64.s +++ b/internal/native/neon/lookup_small_key_arm64.s @@ -64,250 +64,232 @@ _lookup_small_key: WORD $0xf940002b // ldr x11, [x1] WORD $0x12001d49 // and w9, w10, #0xff WORD $0x8b294928 // add x8, x9, w9, uxtw #2 - WORD $0x8b08016c // add x12, x11, x8 - WORD $0x39400188 // ldrb w8, [x12] - WORD $0x34001be8 // cbz w8, LBB0_46 $892(%rip) - WORD $0xf940000f // ldr x15, [x0] - WORD $0xb840118c // ldur w12, [x12, #1] - WORD $0x1102958d // add w13, w12, #165 - WORD $0x8b0d016d // add x13, x11, x13 - WORD $0x92401d4e // and x14, x10, #0xff + WORD $0x8b08016d // add x13, x11, x8 + WORD $0x394001a8 // ldrb w8, [x13] + WORD $0x340019e8 // cbz w8, LBB0_46 $828(%rip) + WORD $0xf940000c // ldr x12, [x0] + WORD $0xb84011ad // ldur w13, [x13, #1] + WORD $0x110295ae // add w14, w13, #165 + WORD $0x8b0e016e // add x14, x11, x14 + WORD $0x92401d4f // and x15, x10, #0xff WORD $0x7100253f // cmp w9, #9 WORD $0x54000942 // b.hs LBB0_20 $296(%rip) WORD $0x11000530 // add w16, w9, #1 - WORD $0x394001f1 // ldrb w17, [x15] + WORD $0x39400191 // ldrb w17, [x12] WORD $0x528000e0 // mov w0, #7 WORD $0xaa0803e1 // mov x1, x8 WORD $0x14000007 // b LBB0_5 $28(%rip) LBB0_3: WORD $0x52800003 // mov w3, #0 - WORD $0x6b0e007f // cmp w3, w14 + WORD $0x6b0f007f // cmp w3, w15 WORD $0x540007a2 // b.hs LBB0_19 $244(%rip) LBB0_4: - WORD $0x8b1001ad // add x13, x13, x16 + WORD $0x8b1001ce // add x14, x14, x16 WORD $0x71000421 // subs w1, w1, #1 - WORD $0x54000b60 // b.eq LBB0_23 $364(%rip) + WORD $0x54000b40 // b.eq LBB0_23 $360(%rip) LBB0_5: - WORD $0x394001a3 // ldrb w3, [x13] + WORD $0x394001c3 // ldrb w3, [x14] WORD $0x6b11007f // cmp w3, w17 WORD $0x54ffff01 // b.ne LBB0_3 $-32(%rip) - WORD $0x394005a3 // ldrb w3, [x13, #1] - WORD $0x394005e4 // ldrb w4, [x15, #1] + WORD $0x394005c3 // ldrb w3, [x14, #1] + WORD $0x39400584 // ldrb w4, [x12, #1] WORD $0x6b04007f // cmp w3, w4 WORD $0x54000381 // b.ne LBB0_13 $112(%rip) - WORD $0x394009a3 // ldrb w3, [x13, #2] - WORD $0x394009e4 // ldrb w4, [x15, #2] + WORD $0x394009c3 // ldrb w3, [x14, #2] + WORD $0x39400984 // ldrb w4, [x12, #2] WORD $0x6b04007f // cmp w3, w4 WORD $0x54000381 // b.ne LBB0_14 $112(%rip) - WORD $0x39400da3 // ldrb w3, [x13, #3] - WORD $0x39400de4 // ldrb w4, [x15, #3] + WORD $0x39400dc3 // ldrb w3, [x14, #3] + WORD $0x39400d84 // ldrb w4, [x12, #3] WORD $0x6b04007f // cmp w3, w4 WORD $0x54000381 // b.ne LBB0_15 $112(%rip) - WORD $0x394011a3 // ldrb w3, [x13, #4] - WORD $0x394011e4 // ldrb w4, [x15, #4] + WORD $0x394011c3 // ldrb w3, [x14, #4] + WORD $0x39401184 // ldrb w4, [x12, #4] WORD $0x6b04007f // cmp w3, w4 WORD $0x54000381 // b.ne LBB0_16 $112(%rip) - WORD $0x394015a3 // ldrb w3, [x13, #5] - WORD $0x394015e4 // ldrb w4, [x15, #5] + WORD $0x394015c3 // ldrb w3, [x14, #5] + WORD $0x39401584 // ldrb w4, [x12, #5] WORD $0x6b04007f // cmp w3, w4 WORD $0x54000381 // b.ne LBB0_17 $112(%rip) - WORD $0x394019a3 // ldrb w3, [x13, #6] - WORD $0x394019e4 // ldrb w4, [x15, #6] + WORD $0x394019c3 // ldrb w3, [x14, #6] + WORD $0x39401984 // ldrb w4, [x12, #6] WORD $0x6b04007f // cmp w3, w4 WORD $0x54000381 // b.ne LBB0_18 $112(%rip) - WORD $0x39401da3 // ldrb w3, [x13, #7] - WORD $0x39401de4 // ldrb w4, [x15, #7] + WORD $0x39401dc3 // ldrb w3, [x14, #7] + WORD $0x39401d84 // ldrb w4, [x12, #7] WORD $0x6b04007f // cmp w3, w4 WORD $0x1a801403 // cinc w3, w0, eq - WORD $0x6b0e007f // cmp w3, w14 + WORD $0x6b0f007f // cmp w3, w15 WORD $0x54fffba3 // b.lo LBB0_4 $-140(%rip) WORD $0x14000018 // b LBB0_19 $96(%rip) LBB0_13: WORD $0x52800023 // mov w3, #1 - WORD $0x6b0e007f // cmp w3, w14 + WORD $0x6b0f007f // cmp w3, w15 WORD $0x54fffb23 // b.lo LBB0_4 $-156(%rip) WORD $0x14000014 // b LBB0_19 $80(%rip) LBB0_14: WORD $0x52800043 // mov w3, #2 - WORD $0x6b0e007f // cmp w3, w14 + WORD $0x6b0f007f // cmp w3, w15 WORD $0x54fffaa3 // b.lo LBB0_4 $-172(%rip) WORD $0x14000010 // b LBB0_19 $64(%rip) LBB0_15: WORD $0x52800063 // mov w3, #3 - WORD $0x6b0e007f // cmp w3, w14 + WORD $0x6b0f007f // cmp w3, w15 WORD $0x54fffa23 // b.lo LBB0_4 $-188(%rip) WORD $0x1400000c // b LBB0_19 $48(%rip) LBB0_16: WORD $0x52800083 // mov w3, #4 - WORD $0x6b0e007f // cmp w3, w14 + WORD $0x6b0f007f // cmp w3, w15 WORD $0x54fff9a3 // b.lo LBB0_4 $-204(%rip) WORD $0x14000008 // b LBB0_19 $32(%rip) LBB0_17: WORD $0x528000a3 // mov w3, #5 - WORD $0x6b0e007f // cmp w3, w14 + WORD $0x6b0f007f // cmp w3, w15 WORD $0x54fff923 // b.lo LBB0_4 $-220(%rip) WORD $0x14000004 // b LBB0_19 $16(%rip) LBB0_18: WORD $0x528000c3 // mov w3, #6 - WORD $0x6b0e007f // cmp w3, w14 + WORD $0x6b0f007f // cmp w3, w15 WORD $0x54fff8a3 // b.lo LBB0_4 $-236(%rip) LBB0_19: - WORD $0x8b0e01a8 // add x8, x13, x14 + WORD $0x8b0f01c8 // add x8, x14, x15 WORD $0x39400100 // ldrb w0, [x8] WORD $0xa940fbfd // ldp fp, lr, [sp, #8] WORD $0x910083ff // add sp, sp, #32 WORD $0xd65f03c0 // ret LBB0_20: - WORD $0xad4001e1 // ldp q1, q0, [x15] - WORD $0x9280000f // mov x15, #-1 - WORD $0x9ace21ef // lsl x15, x15, x14 - WORD $0x11000530 // add w16, w9, #1 + WORD $0xad400580 // ldp q0, q1, [x12] + WORD $0x92800010 // mov x16, #-1 + WORD $0x9acf2210 // lsl x16, x16, x15 + WORD $0x11000531 // add w17, w9, #1 Lloh0: - WORD $0x10fff331 // adr x17, lCPI0_0 $-412(%rip) + WORD $0x10fff320 // adr x0, lCPI0_0 $-412(%rip) Lloh1: - WORD $0x3dc00222 // ldr q2, [x17, lCPI0_0@PAGEOFF] $0(%rip) + WORD $0x3dc00002 // ldr q2, [x0, lCPI0_0@PAGEOFF] $0(%rip) Lloh2: - WORD $0x10fff371 // adr x17, lCPI0_1 $-404(%rip) + WORD $0x10fff360 // adr x0, lCPI0_1 $-404(%rip) Lloh3: - WORD $0x3dc00223 // ldr q3, [x17, lCPI0_1@PAGEOFF] $0(%rip) - WORD $0xaa0803f1 // mov x17, x8 + WORD $0x3dc00003 // ldr q3, [x0, lCPI0_1@PAGEOFF] $0(%rip) + WORD $0xaa0803e0 // mov x0, x8 LBB0_21: - WORD $0xad4015a4 // ldp q4, q5, [x13] - WORD $0x6e248c24 // cmeq.16b v4, v1, v4 - WORD $0x6e258c05 // cmeq.16b v5, v0, v5 + WORD $0xad4015c4 // ldp q4, q5, [x14] + WORD $0x6e248c04 // cmeq.16b v4, v0, v4 + WORD $0x6e258c25 // cmeq.16b v5, v1, v5 WORD $0x4e221c84 // and.16b v4, v4, v2 WORD $0x4e030084 // tbl.16b v4, { v4 }, v3 WORD $0x4e71b884 // addv.8h h4, v4 - WORD $0x1e260080 // fmov w0, s4 + WORD $0x1e260081 // fmov w1, s4 WORD $0x4e221ca4 // and.16b v4, v5, v2 WORD $0x4e030084 // tbl.16b v4, { v4 }, v3 WORD $0x4e71b884 // addv.8h h4, v4 - WORD $0x1e260081 // fmov w1, s4 - WORD $0x33103c20 // bfi w0, w1, #16, #16 - WORD $0x2a0f0000 // orr w0, w0, w15 - WORD $0x3100041f // cmn w0, #1 + WORD $0x1e260083 // fmov w3, s4 + WORD $0x33103c61 // bfi w1, w3, #16, #16 + WORD $0x2a100021 // orr w1, w1, w16 + WORD $0x3100043f // cmn w1, #1 WORD $0x54fffc80 // b.eq LBB0_19 $-112(%rip) - WORD $0x8b1001ad // add x13, x13, x16 - WORD $0x71000631 // subs w17, w17, #1 + WORD $0x8b1101ce // add x14, x14, x17 + WORD $0x71000400 // subs w0, w0, #1 WORD $0x54fffde1 // b.ne LBB0_21 $-68(%rip) - WORD $0x14000002 // b LBB0_24 $8(%rip) LBB0_23: - WORD $0xad4001e1 // ldp q1, q0, [x15] -LBB0_24: - WORD $0x4f05e7e2 // movi.16b v2, #191 - WORD $0x4e228424 // add.16b v4, v1, v2 - WORD $0x4f00e743 // movi.16b v3, #26 - WORD $0x6e243465 // cmhi.16b v5, v3, v4 - WORD $0x4f01e404 // movi.16b v4, #32 - WORD $0x4e241ca5 // and.16b v5, v5, v4 - WORD $0x4e2184a1 // add.16b v1, v5, v1 - WORD $0x8b0c016b // add x11, x11, x12 + WORD $0xb100045f // cmn x2, #1 + WORD $0x54000c40 // b.eq LBB0_46 $392(%rip) + WORD $0x3dc00180 // ldr q0, [x12] + WORD $0x4f05e7e1 // movi.16b v1, #191 + WORD $0x4e218403 // add.16b v3, v0, v1 + WORD $0x4f00e742 // movi.16b v2, #26 + WORD $0x6e233444 // cmhi.16b v4, v2, v3 + WORD $0x4f01e403 // movi.16b v3, #32 + WORD $0x4e231c84 // and.16b v4, v4, v3 + WORD $0x4e208480 // add.16b v0, v4, v0 + WORD $0x8b0d016b // add x11, x11, x13 WORD $0x8b02016b // add x11, x11, x2 WORD $0x92401d4a // and x10, x10, #0xff WORD $0x7100253f // cmp w9, #9 - WORD $0x54000922 // b.hs LBB0_43 $292(%rip) - WORD $0x0e013c2c // umov.b w12, v1[0] - WORD $0x0e033c2d // umov.b w13, v1[1] - WORD $0x0e053c2e // umov.b w14, v1[2] - WORD $0x0e073c2f // umov.b w15, v1[3] - WORD $0x0e093c30 // umov.b w16, v1[4] - WORD $0x0e0b3c31 // umov.b w17, v1[5] + WORD $0x540006e2 // b.hs LBB0_43 $220(%rip) + WORD $0x0e013c0c // umov.b w12, v0[0] + WORD $0x0e033c0d // umov.b w13, v0[1] + WORD $0x0e053c0e // umov.b w14, v0[2] + WORD $0x0e073c0f // umov.b w15, v0[3] + WORD $0x0e093c10 // umov.b w16, v0[4] + WORD $0x0e0b3c11 // umov.b w17, v0[5] WORD $0x11000529 // add w9, w9, #1 - WORD $0x0e0d3c20 // umov.b w0, v1[6] + WORD $0x0e0d3c00 // umov.b w0, v0[6] WORD $0x528000e1 // mov w1, #7 - WORD $0x0e0f3c22 // umov.b w2, v1[7] - WORD $0x14000007 // b LBB0_28 $28(%rip) + WORD $0x0e0f3c02 // umov.b w2, v0[7] LBB0_26: - WORD $0x52800003 // mov w3, #0 - WORD $0x6b0a007f // cmp w3, w10 - WORD $0x540006c2 // b.hs LBB0_42 $216(%rip) -LBB0_27: - WORD $0x8b09016b // add x11, x11, x9 - WORD $0x71000508 // subs w8, w8, #1 - WORD $0x54000aa0 // b.eq LBB0_46 $340(%rip) -LBB0_28: WORD $0x39400163 // ldrb w3, [x11] WORD $0x6b2c007f // cmp w3, w12, uxtb - WORD $0x54ffff01 // b.ne LBB0_26 $-32(%rip) + WORD $0x540002e1 // b.ne LBB0_34 $92(%rip) WORD $0x39400563 // ldrb w3, [x11, #1] WORD $0x6b2d007f // cmp w3, w13, uxtb - WORD $0x540002c1 // b.ne LBB0_36 $88(%rip) + WORD $0x540002c1 // b.ne LBB0_35 $88(%rip) WORD $0x39400963 // ldrb w3, [x11, #2] WORD $0x6b2e007f // cmp w3, w14, uxtb - WORD $0x540002e1 // b.ne LBB0_37 $92(%rip) + WORD $0x540002a1 // b.ne LBB0_36 $84(%rip) WORD $0x39400d63 // ldrb w3, [x11, #3] WORD $0x6b2f007f // cmp w3, w15, uxtb - WORD $0x54000301 // b.ne LBB0_38 $96(%rip) + WORD $0x54000281 // b.ne LBB0_37 $80(%rip) WORD $0x39401163 // ldrb w3, [x11, #4] WORD $0x6b30007f // cmp w3, w16, uxtb - WORD $0x54000321 // b.ne LBB0_39 $100(%rip) + WORD $0x54000261 // b.ne LBB0_38 $76(%rip) WORD $0x39401563 // ldrb w3, [x11, #5] WORD $0x6b31007f // cmp w3, w17, uxtb - WORD $0x54000341 // b.ne LBB0_40 $104(%rip) + WORD $0x54000241 // b.ne LBB0_39 $72(%rip) WORD $0x39401963 // ldrb w3, [x11, #6] WORD $0x6b20007f // cmp w3, w0, uxtb - WORD $0x54000361 // b.ne LBB0_41 $108(%rip) + WORD $0x54000221 // b.ne LBB0_40 $68(%rip) WORD $0x39401d63 // ldrb w3, [x11, #7] WORD $0x6b22007f // cmp w3, w2, uxtb WORD $0x1a811423 // cinc w3, w1, eq - WORD $0x6b0a007f // cmp w3, w10 - WORD $0x54fffc83 // b.lo LBB0_27 $-112(%rip) - WORD $0x14000018 // b LBB0_42 $96(%rip) -LBB0_36: + WORD $0x1400000e // b LBB0_41 $56(%rip) +LBB0_34: + WORD $0x52800003 // mov w3, #0 + WORD $0x1400000c // b LBB0_41 $48(%rip) +LBB0_35: WORD $0x52800023 // mov w3, #1 - WORD $0x6b0a007f // cmp w3, w10 - WORD $0x54fffc03 // b.lo LBB0_27 $-128(%rip) - WORD $0x14000014 // b LBB0_42 $80(%rip) -LBB0_37: + WORD $0x1400000a // b LBB0_41 $40(%rip) +LBB0_36: WORD $0x52800043 // mov w3, #2 - WORD $0x6b0a007f // cmp w3, w10 - WORD $0x54fffb83 // b.lo LBB0_27 $-144(%rip) - WORD $0x14000010 // b LBB0_42 $64(%rip) -LBB0_38: + WORD $0x14000008 // b LBB0_41 $32(%rip) +LBB0_37: WORD $0x52800063 // mov w3, #3 - WORD $0x6b0a007f // cmp w3, w10 - WORD $0x54fffb03 // b.lo LBB0_27 $-160(%rip) - WORD $0x1400000c // b LBB0_42 $48(%rip) -LBB0_39: + WORD $0x14000006 // b LBB0_41 $24(%rip) +LBB0_38: WORD $0x52800083 // mov w3, #4 - WORD $0x6b0a007f // cmp w3, w10 - WORD $0x54fffa83 // b.lo LBB0_27 $-176(%rip) - WORD $0x14000008 // b LBB0_42 $32(%rip) -LBB0_40: + WORD $0x14000004 // b LBB0_41 $16(%rip) +LBB0_39: WORD $0x528000a3 // mov w3, #5 - WORD $0x6b0a007f // cmp w3, w10 - WORD $0x54fffa03 // b.lo LBB0_27 $-192(%rip) - WORD $0x14000004 // b LBB0_42 $16(%rip) -LBB0_41: + WORD $0x14000002 // b LBB0_41 $8(%rip) +LBB0_40: WORD $0x528000c3 // mov w3, #6 +LBB0_41: WORD $0x6b0a007f // cmp w3, w10 - WORD $0x54fff983 // b.lo LBB0_27 $-208(%rip) -LBB0_42: - WORD $0x8b0a0168 // add x8, x11, x10 - WORD $0x39400100 // ldrb w0, [x8] - WORD $0xa940fbfd // ldp fp, lr, [sp, #8] - WORD $0x910083ff // add sp, sp, #32 - WORD $0xd65f03c0 // ret + WORD $0x540004e2 // b.hs LBB0_47 $156(%rip) + WORD $0x8b09016b // add x11, x11, x9 + WORD $0x71000508 // subs w8, w8, #1 + WORD $0x54fffac1 // b.ne LBB0_26 $-168(%rip) + WORD $0x1400001f // b LBB0_46 $124(%rip) LBB0_43: - WORD $0x4e228402 // add.16b v2, v0, v2 - WORD $0x6e223462 // cmhi.16b v2, v3, v2 - WORD $0x4e241c42 // and.16b v2, v2, v4 - WORD $0x4e208440 // add.16b v0, v2, v0 + WORD $0x3dc00584 // ldr q4, [x12, #16] + WORD $0x4e218481 // add.16b v1, v4, v1 + WORD $0x6e213441 // cmhi.16b v1, v2, v1 + WORD $0x4e231c21 // and.16b v1, v1, v3 + WORD $0x4e248421 // add.16b v1, v1, v4 WORD $0x9280000c // mov x12, #-1 WORD $0x9aca218c // lsl x12, x12, x10 WORD $0x11000529 // add w9, w9, #1 Lloh4: - WORD $0x10ffe4ad // adr x13, lCPI0_0 $-876(%rip) + WORD $0x10ffe6ad // adr x13, lCPI0_0 $-812(%rip) Lloh5: WORD $0x3dc001a2 // ldr q2, [x13, lCPI0_0@PAGEOFF] $0(%rip) Lloh6: - WORD $0x10ffe4ed // adr x13, lCPI0_1 $-868(%rip) + WORD $0x10ffe6ed // adr x13, lCPI0_1 $-804(%rip) Lloh7: WORD $0x3dc001a3 // ldr q3, [x13, lCPI0_1@PAGEOFF] $0(%rip) LBB0_44: WORD $0xad401564 // ldp q4, q5, [x11] - WORD $0x6e248c24 // cmeq.16b v4, v1, v4 - WORD $0x6e258c05 // cmeq.16b v5, v0, v5 + WORD $0x6e248c04 // cmeq.16b v4, v0, v4 + WORD $0x6e258c25 // cmeq.16b v5, v1, v5 WORD $0x4e221c84 // and.16b v4, v4, v2 WORD $0x4e030084 // tbl.16b v4, { v4 }, v3 WORD $0x4e71b884 // addv.8h h4, v4 @@ -319,7 +301,7 @@ LBB0_44: WORD $0x33103dcd // bfi w13, w14, #16, #16 WORD $0x2a0c01ad // orr w13, w13, w12 WORD $0x310005bf // cmn w13, #1 - WORD $0x54fffc40 // b.eq LBB0_42 $-120(%rip) + WORD $0x54000100 // b.eq LBB0_47 $32(%rip) WORD $0x8b09016b // add x11, x11, x9 WORD $0x71000508 // subs w8, w8, #1 WORD $0x54fffde1 // b.ne LBB0_44 $-68(%rip) @@ -328,6 +310,12 @@ LBB0_46: WORD $0xa940fbfd // ldp fp, lr, [sp, #8] WORD $0x910083ff // add sp, sp, #32 WORD $0xd65f03c0 // ret +LBB0_47: + WORD $0x8b0a0168 // add x8, x11, x10 + WORD $0x39400100 // ldrb w0, [x8] + WORD $0xa940fbfd // ldp fp, lr, [sp, #8] + WORD $0x910083ff // add sp, sp, #32 + WORD $0xd65f03c0 // ret TEXT ·__lookup_small_key(SB), NOSPLIT, $0-32 NO_LOCAL_POINTERS diff --git a/internal/optcaching/fcache.go b/internal/optcaching/fcache.go index afa7e7105..010028203 100644 --- a/internal/optcaching/fcache.go +++ b/internal/optcaching/fcache.go @@ -32,7 +32,7 @@ const _PaddingSize = 32 type FieldLookup interface { Set(fields []resolver.FieldMeta) - Get(name string) int + Get(name string, caseSensitive bool) int } func isAscii(s string) bool { @@ -84,7 +84,7 @@ func NewSmallFieldMap (hint int) *SmallFieldMap { func (self *SmallFieldMap) Set(fields []resolver.FieldMeta) { if len(fields) > 8 { - panic("small field map shoud use in small struct") + panic("small field map should use in small struct") } for i, f := range fields { @@ -93,13 +93,15 @@ func (self *SmallFieldMap) Set(fields []resolver.FieldMeta) { } } -func (self *SmallFieldMap) Get(name string) int { +func (self *SmallFieldMap) Get(name string, caseSensitive bool) int { for i, k := range self.keys { if len(k) == len(name) && k == name { return i } } - + if caseSensitive { + return -1 + } name = strings.ToLower(name) for i, k := range self.lowerKeys { if len(k) == len(name) && k == name { @@ -153,16 +155,20 @@ const _HdrSlot = 33 const _HdrSize = _HdrSlot * 5 // use native SIMD to accelerate it -func (self *NormalFieldMap) Get(name string) int { +func (self *NormalFieldMap) Get(name string, caseSensitive bool) int { // small keys use native C if len(name) <= 32 { - _ = native.LookupSmallKey - return native.LookupSmallKey(&name, &self.keys, self.lowOffset); + _ = native.LookupSmallKey + lowOffset := self.lowOffset + if caseSensitive { + lowOffset = -1 + } + return native.LookupSmallKey(&name, &self.keys, lowOffset); } - return self.getLongKey(name) + return self.getLongKey(name, caseSensitive) } -func (self *NormalFieldMap) getLongKey(name string) int { +func (self *NormalFieldMap) getLongKey(name string, caseSensitive bool) int { for _, k := range self.longKeys { if len(k.key) != len(name) { continue; @@ -172,6 +178,10 @@ func (self *NormalFieldMap) getLongKey(name string) int { } } + if caseSensitive { + return -1 + } + lower := strings.ToLower(name) for _, k := range self.longKeys { if len(k.key) != len(name) { @@ -254,7 +264,7 @@ type keysInfo struct { func (self *NormalFieldMap) Set(fields []resolver.FieldMeta) { if len(fields) <=8 || len(fields) > 128 { - panic("normal field map shoud use in small struct") + panic("normal field map should use in small struct") } // allocate the flat map in []byte @@ -278,7 +288,7 @@ func (self *NormalFieldMap) Set(fields []resolver.FieldMeta) { } - // add a padding size at last to make it firendly for SIMD. + // add a padding size at last to make it friendly for SIMD. self.keys = make([]byte, _HdrSize + 2 * kvLen, _HdrSize + 2 * kvLen + _PaddingSize) self.lowOffset = _HdrSize + kvLen @@ -329,11 +339,13 @@ type FallbackFieldMap struct { } } - func (self *FallbackFieldMap) Get(name string) int { + func (self *FallbackFieldMap) Get(name string, caseSensitive bool) int { if i, ok := self.inner[name]; ok { return i - } else { + } else if !caseSensitive { return self.getCaseInsensitive(name) + } else { + return -1 } } diff --git a/internal/rt/stubs.go b/internal/rt/stubs.go index 5104a0793..1074f491b 100644 --- a/internal/rt/stubs.go +++ b/internal/rt/stubs.go @@ -153,7 +153,7 @@ func MakeSlice(oldPtr unsafe.Pointer, et *GoType, newLen int) *GoSlice { new := GrowSlice(et, *old, newLen) - // we sould clear the memory from [oldLen:newLen] + // we should clear the memory from [oldLen:newLen] if et.PtrData == 0 { oldlenmem := uintptr(old.Len) * et.Size newlenmem := uintptr(newLen) * et.Size diff --git a/internal/utils/skip.go b/internal/utils/skip.go new file mode 100644 index 000000000..e42bfe759 --- /dev/null +++ b/internal/utils/skip.go @@ -0,0 +1,79 @@ + +package utils + +import ( + `runtime` + `unsafe` + + `github.com/bytedance/sonic/internal/native/types` + `github.com/bytedance/sonic/internal/rt` +) + +func isDigit(c byte) bool { + return c >= '0' && c <= '9' +} + +//go:nocheckptr +func SkipNumber(src string, pos int) (ret int) { + sp := uintptr(rt.IndexChar(src, pos)) + se := uintptr(rt.IndexChar(src, len(src))) + if uintptr(sp) >= se { + return -int(types.ERR_EOF) + } + + if c := *(*byte)(unsafe.Pointer(sp)); c == '-' { + sp += 1 + } + ss := sp + + var pointer bool + var exponent bool + var lastIsDigit bool + var nextNeedDigit = true + + for ; sp < se; sp += uintptr(1) { + c := *(*byte)(unsafe.Pointer(sp)) + if isDigit(c) { + lastIsDigit = true + nextNeedDigit = false + continue + } else if nextNeedDigit { + return -int(types.ERR_INVALID_CHAR) + } else if c == '.' { + if !lastIsDigit || pointer || exponent || sp == ss { + return -int(types.ERR_INVALID_CHAR) + } + pointer = true + lastIsDigit = false + nextNeedDigit = true + continue + } else if c == 'e' || c == 'E' { + if !lastIsDigit || exponent { + return -int(types.ERR_INVALID_CHAR) + } + if sp == se-1 { + return -int(types.ERR_EOF) + } + exponent = true + lastIsDigit = false + nextNeedDigit = false + continue + } else if c == '-' || c == '+' { + if prev := *(*byte)(unsafe.Pointer(sp - 1)); prev != 'e' && prev != 'E' { + return -int(types.ERR_INVALID_CHAR) + } + lastIsDigit = false + nextNeedDigit = true + continue + } else { + break + } + } + + if nextNeedDigit { + return -int(types.ERR_EOF) + } + + runtime.KeepAlive(src) + return int(uintptr(sp) - uintptr((*rt.GoString)(unsafe.Pointer(&src)).Ptr)) +} \ No newline at end of file diff --git a/issue_test/issue670_test.go b/issue_test/issue670_test.go index f4605ab8a..15f554994 100644 --- a/issue_test/issue670_test.go +++ b/issue_test/issue670_test.go @@ -30,7 +30,6 @@ func TestIssue670_JSONMarshaler(t *testing.T) { so, _ := sonic.MarshalString(obj) eo, _ := json.Marshal(obj) assert.Equal(t, string(eo), so) - println(string(eo)) } func TestIssue670_JSONUnmarshaler(t *testing.T) { @@ -50,11 +49,8 @@ func TestIssue670_JSONUnmarshaler(t *testing.T) { func testUnmarshal(t *testing.T, eo []byte, rt reflect.Type, checkobj bool) { obj := reflect.New(rt).Interface() - println(string(eo)) - println("sonic") es := sonic.Unmarshal(eo, obj) obj2 := reflect.New(rt).Interface() - println("std") ee := json.Unmarshal(eo, obj2) assert.Equal(t, ee ==nil, es == nil, es) if checkobj { @@ -107,7 +103,6 @@ func (d *Date) UnmarshalJSON(in []byte) error { return nil } - println("hook ", string(in)) t, err := time.Parse("2006-01-02", string(in)) if err != nil { return err @@ -125,7 +120,6 @@ type Issue670TextMarshaler struct { type Date2 int64 func (d Date2) MarshalText() ([]byte, error) { - println("hook 1") if d == 0 { return []byte("null"), nil } @@ -133,7 +127,6 @@ func (d Date2) MarshalText() ([]byte, error) { } func (d *Date2) UnmarshalText(in []byte) error { - println("hook 2", string(in)) if string(in) == "null" { *d = 0 return nil diff --git a/issue_test/issue716_test.go b/issue_test/issue716_test.go new file mode 100644 index 000000000..b874cb11a --- /dev/null +++ b/issue_test/issue716_test.go @@ -0,0 +1,54 @@ +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +package issue_test + +import ( + "fmt" + "testing" + + "github.com/bytedance/sonic" + "github.com/stretchr/testify/assert" +) + +type UnmFoo struct { + Name string + Age int +} + +func (p *UnmFoo) UnmarshalJSON(data []byte) error { + var aux struct { + Name string `json:"name"` + Age int `json:"age"` + } + + if err := sonic.Unmarshal(data, &aux); err != nil { + return err + } + + p.Name = aux.Name + p.Age = aux.Age + return nil +} + +func TestIssue716(t *testing.T) { + jsonData := `{"name": "Alice", "age": "30"}` + var obj UnmFoo + err := sonic.Unmarshal([]byte(jsonData), &obj) + assert.Error(t, err) + if err != nil { + fmt.Println("Error:", err) + } +} diff --git a/loader/funcdata_compat.go b/loader/funcdata_compat.go index 17c840613..b4a24bcd6 100644 --- a/loader/funcdata_compat.go +++ b/loader/funcdata_compat.go @@ -329,7 +329,7 @@ func makeModuledata(name string, filenames []string, funcsp *[]Func, text []byte funcnametab, nameOffs := makeFuncnameTab(funcs) mod.funcnametab = funcnametab - // mmap() text and funcdata segements + // mmap() text and funcdata segments p := os.Getpagesize() size := int(rnd(int64(len(text)), int64(p))) addr := mmap(size) @@ -389,7 +389,7 @@ func makeModuledata(name string, filenames []string, funcsp *[]Func, text []byte pclnOffset: getOffsetOf(moduledata{}, "pclntable"), } - // sepecial case: gcdata and gcbss must by non-empty + // special case: gcdata and gcbss must by non-empty mod.gcdata = uintptr(unsafe.Pointer(&emptyByte)) mod.gcbss = uintptr(unsafe.Pointer(&emptyByte)) diff --git a/loader/funcdata_go117.go b/loader/funcdata_go117.go index 623283d09..1cae26898 100644 --- a/loader/funcdata_go117.go +++ b/loader/funcdata_go117.go @@ -330,7 +330,7 @@ func makeModuledata(name string, filenames []string, funcsp *[]Func, text []byte funcnametab, nameOffs := makeFuncnameTab(funcs) mod.funcnametab = funcnametab - // mmap() text and funcdata segements + // mmap() text and funcdata segments p := os.Getpagesize() size := int(rnd(int64(len(text)), int64(p))) addr := mmap(size) @@ -390,7 +390,7 @@ func makeModuledata(name string, filenames []string, funcsp *[]Func, text []byte pclnOffset: getOffsetOf(moduledata{}, "pclntable"), } - // sepecial case: gcdata and gcbss must by non-empty + // special case: gcdata and gcbss must by non-empty mod.gcdata = uintptr(unsafe.Pointer(&emptyByte)) mod.gcbss = uintptr(unsafe.Pointer(&emptyByte)) diff --git a/loader/funcdata_latest.go b/loader/funcdata_latest.go index b19fa6b7d..8b6018bdc 100644 --- a/loader/funcdata_latest.go +++ b/loader/funcdata_latest.go @@ -222,7 +222,7 @@ func makeModuledata(name string, filenames []string, funcsp *[]Func, text []byte funcnametab, nameOffs := makeFuncnameTab(funcs) mod.funcnametab = funcnametab - // mmap() text and funcdata segements + // mmap() text and funcdata segments p := os.Getpagesize() size := int(rnd(int64(len(text)), int64(p))) addr := mmap(size) @@ -283,7 +283,7 @@ func makeModuledata(name string, filenames []string, funcsp *[]Func, text []byte pclnOffset: getOffsetOf(moduledata{}, "pclntable"), } - // sepecial case: gcdata and gcbss must by non-empty + // special case: gcdata and gcbss must by non-empty mod.gcdata = uintptr(unsafe.Pointer(&emptyByte)) mod.gcbss = uintptr(unsafe.Pointer(&emptyByte)) diff --git a/loader/go.mod b/loader/go.mod index 8f27f97f1..1d334f54a 100644 --- a/loader/go.mod +++ b/loader/go.mod @@ -3,6 +3,6 @@ module github.com/bytedance/sonic/loader go 1.16 require ( - github.com/cloudwego/iasm v0.2.0 + github.com/davecgh/go-spew v1.1.1 github.com/stretchr/testify v1.8.1 ) diff --git a/loader/go.sum b/loader/go.sum index a4a6fd4ef..2ec90f70f 100644 --- a/loader/go.sum +++ b/loader/go.sum @@ -1,16 +1,11 @@ -github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= -github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= @@ -20,4 +15,3 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= diff --git a/loader/internal/abi/abi_amd64.go b/loader/internal/abi/abi_amd64.go index c2b45a8e1..2969c3bba 100644 --- a/loader/internal/abi/abi_amd64.go +++ b/loader/internal/abi/abi_amd64.go @@ -17,266 +17,285 @@ package abi import ( - `fmt` - `reflect` - `unsafe` + "fmt" + "reflect" + "unsafe" - . `github.com/cloudwego/iasm/x86_64` + x64 "github.com/bytedance/sonic/loader/internal/iasm/x86_64" +) + +type ( + Register = x64.Register + Register64 = x64.Register64 + XMMRegister = x64.XMMRegister + Program = x64.Program + MemoryOperand = x64.MemoryOperand + Label = x64.Label +) + +var ( + Ptr = x64.Ptr + DefaultArch = x64.DefaultArch + CreateLabel = x64.CreateLabel ) const ( - PtrSize = 8 // pointer size - PtrAlign = 8 // pointer alignment + RAX = x64.RAX + RSP = x64.RSP + RBP = x64.RBP + R12 = x64.R12 + R14 = x64.R14 + R15 = x64.R15 +) + +const ( + PtrSize = 8 // pointer size + PtrAlign = 8 // pointer alignment ) var iregOrderC = []Register{ - RDI, - RSI, - RDX, - RCX, - R8, - R9, + x64.RDI, + x64.RSI, + x64.RDX, + x64.RCX, + x64.R8, + x64.R9, } var xregOrderC = []Register{ - XMM0, - XMM1, - XMM2, - XMM3, - XMM4, - XMM5, - XMM6, - XMM7, + x64.XMM0, + x64.XMM1, + x64.XMM2, + x64.XMM3, + x64.XMM4, + x64.XMM5, + x64.XMM6, + x64.XMM7, } var ( - intType = reflect.TypeOf(0) - ptrType = reflect.TypeOf(unsafe.Pointer(nil)) + intType = reflect.TypeOf(0) + ptrType = reflect.TypeOf(unsafe.Pointer(nil)) ) func (self *Frame) argv(i int) *MemoryOperand { - return Ptr(RSP, int32(self.Prev() + self.desc.Args[i].Mem)) + return Ptr(RSP, int32(self.Prev()+self.desc.Args[i].Mem)) } // spillv is used for growstack spill registers func (self *Frame) spillv(i int) *MemoryOperand { - // remain one slot for caller return pc - return Ptr(RSP, PtrSize + int32(self.desc.Args[i].Mem)) + // remain one slot for caller return pc + return Ptr(RSP, PtrSize+int32(self.desc.Args[i].Mem)) } func (self *Frame) retv(i int) *MemoryOperand { - return Ptr(RSP, int32(self.Prev() + self.desc.Rets[i].Mem)) + return Ptr(RSP, int32(self.Prev()+self.desc.Rets[i].Mem)) } func (self *Frame) resv(i int) *MemoryOperand { - return Ptr(RSP, int32(self.Offs() - uint32((i+1) * PtrSize))) + return Ptr(RSP, int32(self.Offs()-uint32((i+1)*PtrSize))) } func (self *Frame) emitGrowStack(p *Program, entry *Label) { - // spill all register arguments - for i, v := range self.desc.Args { - if v.InRegister { - if v.IsFloat == floatKind64 { - p.MOVSD(v.Reg, self.spillv(i)) - } else if v.IsFloat == floatKind32 { - p.MOVSS(v.Reg, self.spillv(i)) - }else { - p.MOVQ(v.Reg, self.spillv(i)) - } - } - } - - // call runtime.morestack_noctxt - p.MOVQ(F_morestack_noctxt, R12) - p.CALLQ(R12) - // load all register arguments - for i, v := range self.desc.Args { - if v.InRegister { - if v.IsFloat == floatKind64 { - p.MOVSD(self.spillv(i), v.Reg) - } else if v.IsFloat == floatKind32 { - p.MOVSS(self.spillv(i), v.Reg) - }else { - p.MOVQ(self.spillv(i), v.Reg) - } - } - } - - // jump back to the function entry - p.JMP(entry) + // spill all register arguments + for i, v := range self.desc.Args { + if v.InRegister { + if v.IsFloat == floatKind64 { + p.MOVSD(v.Reg, self.spillv(i)) + } else if v.IsFloat == floatKind32 { + p.MOVSS(v.Reg, self.spillv(i)) + } else { + p.MOVQ(v.Reg, self.spillv(i)) + } + } + } + + // call runtime.morestack_noctxt + p.MOVQ(F_morestack_noctxt, R12) + p.CALLQ(R12) + // load all register arguments + for i, v := range self.desc.Args { + if v.InRegister { + if v.IsFloat == floatKind64 { + p.MOVSD(self.spillv(i), v.Reg) + } else if v.IsFloat == floatKind32 { + p.MOVSS(self.spillv(i), v.Reg) + } else { + p.MOVQ(self.spillv(i), v.Reg) + } + } + } + + // jump back to the function entry + p.JMP(entry) } func (self *Frame) GrowStackTextSize() uint32 { - p := DefaultArch.CreateProgram() - // spill all register arguments - for i, v := range self.desc.Args { - if v.InRegister { - if v.IsFloat == floatKind64 { - p.MOVSD(v.Reg, self.spillv(i)) - } else if v.IsFloat == floatKind32 { - p.MOVSS(v.Reg, self.spillv(i)) - }else { - p.MOVQ(v.Reg, self.spillv(i)) - } - } - } - - // call runtime.morestack_noctxt - p.MOVQ(F_morestack_noctxt, R12) - p.CALLQ(R12) - // load all register arguments - for i, v := range self.desc.Args { - if v.InRegister { - if v.IsFloat == floatKind64 { - p.MOVSD(self.spillv(i), v.Reg) - } else if v.IsFloat == floatKind32 { - p.MOVSS(self.spillv(i), v.Reg) - } else { - p.MOVQ(self.spillv(i), v.Reg) - } - } - } - - // jump back to the function entry - l := CreateLabel("") - p.Link(l) - p.JMP(l) - - return uint32(len(p.Assemble(0))) + p := DefaultArch.CreateProgram() + // spill all register arguments + for i, v := range self.desc.Args { + if v.InRegister { + if v.IsFloat == floatKind64 { + p.MOVSD(v.Reg, self.spillv(i)) + } else if v.IsFloat == floatKind32 { + p.MOVSS(v.Reg, self.spillv(i)) + } else { + p.MOVQ(v.Reg, self.spillv(i)) + } + } + } + + // call runtime.morestack_noctxt + p.MOVQ(F_morestack_noctxt, R12) + p.CALLQ(R12) + // load all register arguments + for i, v := range self.desc.Args { + if v.InRegister { + if v.IsFloat == floatKind64 { + p.MOVSD(self.spillv(i), v.Reg) + } else if v.IsFloat == floatKind32 { + p.MOVSS(self.spillv(i), v.Reg) + } else { + p.MOVQ(self.spillv(i), v.Reg) + } + } + } + + // jump back to the function entry + l := CreateLabel("") + p.Link(l) + p.JMP(l) + + return uint32(len(p.Assemble(0))) } func (self *Frame) emitPrologue(p *Program) { - p.SUBQ(self.Size(), RSP) - p.MOVQ(RBP, Ptr(RSP, int32(self.Offs()))) - p.LEAQ(Ptr(RSP, int32(self.Offs())), RBP) + p.SUBQ(self.Size(), RSP) + p.MOVQ(RBP, Ptr(RSP, int32(self.Offs()))) + p.LEAQ(Ptr(RSP, int32(self.Offs())), RBP) } func (self *Frame) emitEpilogue(p *Program) { - p.MOVQ(Ptr(RSP, int32(self.Offs())), RBP) - p.ADDQ(self.Size(), RSP) - p.RET() + p.MOVQ(Ptr(RSP, int32(self.Offs())), RBP) + p.ADDQ(self.Size(), RSP) + p.RET() } func (self *Frame) emitReserveRegs(p *Program) { - // spill reserved registers - for i, r := range ReservedRegs(self.ccall) { - switch r.(type) { - case Register64: - p.MOVQ(r, self.resv(i)) - case XMMRegister: - p.MOVSD(r, self.resv(i)) - default: - panic(fmt.Sprintf("unsupported register type %t to reserve", r)) - } - } + // spill reserved registers + for i, r := range ReservedRegs(self.ccall) { + switch r.(type) { + case Register64: + p.MOVQ(r, self.resv(i)) + case XMMRegister: + p.MOVSD(r, self.resv(i)) + default: + panic(fmt.Sprintf("unsupported register type %t to reserve", r)) + } + } } func (self *Frame) emitSpillPtrs(p *Program) { - // spill pointer argument registers - for i, r := range self.desc.Args { - if r.InRegister && r.IsPointer { - p.MOVQ(r.Reg, self.argv(i)) - } - } + // spill pointer argument registers + for i, r := range self.desc.Args { + if r.InRegister && r.IsPointer { + p.MOVQ(r.Reg, self.argv(i)) + } + } } func (self *Frame) emitClearPtrs(p *Program) { - // spill pointer argument registers - for i, r := range self.desc.Args { - if r.InRegister && r.IsPointer { - p.MOVQ(int64(0), self.argv(i)) - } - } + // spill pointer argument registers + for i, r := range self.desc.Args { + if r.InRegister && r.IsPointer { + p.MOVQ(int64(0), self.argv(i)) + } + } } func (self *Frame) emitCallC(p *Program, addr uintptr) { - p.MOVQ(addr, RAX) - p.CALLQ(RAX) + p.MOVQ(addr, RAX) + p.CALLQ(RAX) } type floatKind uint8 const ( - notFloatKind floatKind = iota - floatKind32 - floatKind64 + notFloatKind floatKind = iota + floatKind32 + floatKind64 ) type Parameter struct { - InRegister bool - IsPointer bool - IsFloat floatKind - Reg Register - Mem uint32 - Type reflect.Type + InRegister bool + IsPointer bool + IsFloat floatKind + Reg Register + Mem uint32 + Type reflect.Type } func mkIReg(vt reflect.Type, reg Register64) (p Parameter) { - p.Reg = reg - p.Type = vt - p.InRegister = true - p.IsPointer = isPointer(vt) - return + p.Reg = reg + p.Type = vt + p.InRegister = true + p.IsPointer = isPointer(vt) + return } func isFloat(vt reflect.Type) floatKind { - switch vt.Kind() { - case reflect.Float32: - return floatKind32 - case reflect.Float64: - return floatKind64 - default: - return notFloatKind - } + switch vt.Kind() { + case reflect.Float32: + return floatKind32 + case reflect.Float64: + return floatKind64 + default: + return notFloatKind + } } func mkXReg(vt reflect.Type, reg XMMRegister) (p Parameter) { - p.Reg = reg - p.Type = vt - p.InRegister = true - p.IsFloat = isFloat(vt) - return + p.Reg = reg + p.Type = vt + p.InRegister = true + p.IsFloat = isFloat(vt) + return } func mkStack(vt reflect.Type, mem uint32) (p Parameter) { - p.Mem = mem - p.Type = vt - p.InRegister = false - p.IsPointer = isPointer(vt) - p.IsFloat = isFloat(vt) - return + p.Mem = mem + p.Type = vt + p.InRegister = false + p.IsPointer = isPointer(vt) + p.IsFloat = isFloat(vt) + return } func (self Parameter) String() string { - if self.InRegister { - return fmt.Sprintf("[%%%s, Pointer(%v), Float(%v)]", self.Reg, self.IsPointer, self.IsFloat) - } else { - return fmt.Sprintf("[%d(FP), Pointer(%v), Float(%v)]", self.Mem, self.IsPointer, self.IsFloat) - } + if self.InRegister { + return fmt.Sprintf("[%%%s, Pointer(%v), Float(%v)]", self.Reg, self.IsPointer, self.IsFloat) + } else { + return fmt.Sprintf("[%d(FP), Pointer(%v), Float(%v)]", self.Mem, self.IsPointer, self.IsFloat) + } } func CallC(addr uintptr, fr Frame, maxStack uintptr) []byte { - p := DefaultArch.CreateProgram() - - stack := CreateLabel("_stack_grow") - entry := CreateLabel("_entry") - p.Link(entry) - fr.emitStackCheck(p, stack, maxStack) - fr.emitPrologue(p) - fr.emitReserveRegs(p) - fr.emitSpillPtrs(p) - fr.emitExchangeArgs(p) - fr.emitCallC(p, addr) - fr.emitExchangeRets(p) - fr.emitRestoreRegs(p) - fr.emitEpilogue(p) - p.Link(stack) - fr.emitGrowStack(p, entry) - - return p.Assemble(0) + p := DefaultArch.CreateProgram() + + stack := CreateLabel("_stack_grow") + entry := CreateLabel("_entry") + p.Link(entry) + fr.emitStackCheck(p, stack, maxStack) + fr.emitPrologue(p) + fr.emitReserveRegs(p) + fr.emitSpillPtrs(p) + fr.emitExchangeArgs(p) + fr.emitCallC(p, addr) + fr.emitExchangeRets(p) + fr.emitRestoreRegs(p) + fr.emitEpilogue(p) + p.Link(stack) + fr.emitGrowStack(p, entry) + + return p.Assemble(0) } - - -func (self *Frame) emitDebug(p *Program) { - p.INT(3) -} \ No newline at end of file diff --git a/loader/internal/abi/abi_legacy_amd64.go b/loader/internal/abi/abi_legacy_amd64.go index 298c48178..722c0696f 100644 --- a/loader/internal/abi/abi_legacy_amd64.go +++ b/loader/internal/abi/abi_legacy_amd64.go @@ -20,163 +20,196 @@ package abi import ( - `fmt` - `reflect` - `runtime` - - . `github.com/cloudwego/iasm/x86_64` + "fmt" + "reflect" + "runtime" ) func ReservedRegs(callc bool) []Register { - return nil + return nil } func salloc(p []Parameter, sp uint32, vt reflect.Type) (uint32, []Parameter) { - switch vt.Kind() { - case reflect.Bool : return sp + 8, append(p, mkStack(reflect.TypeOf(false), sp)) - case reflect.Int : return sp + 8, append(p, mkStack(intType, sp)) - case reflect.Int8 : return sp + 8, append(p, mkStack(reflect.TypeOf(int8(0)), sp)) - case reflect.Int16 : return sp + 8, append(p, mkStack(reflect.TypeOf(int16(0)), sp)) - case reflect.Int32 : return sp + 8, append(p, mkStack(reflect.TypeOf(int32(0)), sp)) - case reflect.Int64 : return sp + 8, append(p, mkStack(reflect.TypeOf(int64(0)), sp)) - case reflect.Uint : return sp + 8, append(p, mkStack(reflect.TypeOf(uint(0)), sp)) - case reflect.Uint8 : return sp + 8, append(p, mkStack(reflect.TypeOf(uint8(0)), sp)) - case reflect.Uint16 : return sp + 8, append(p, mkStack(reflect.TypeOf(uint16(0)), sp)) - case reflect.Uint32 : return sp + 8, append(p, mkStack(reflect.TypeOf(uint32(0)), sp)) - case reflect.Uint64 : return sp + 8, append(p, mkStack(reflect.TypeOf(uint64(0)), sp)) - case reflect.Uintptr : return sp + 8, append(p, mkStack(reflect.TypeOf(uintptr(0)), sp)) - case reflect.Float32 : return sp + 8, append(p, mkStack(reflect.TypeOf(float32(0)), sp)) - case reflect.Float64 : return sp + 8, append(p, mkStack(reflect.TypeOf(float64(0)), sp)) - case reflect.Complex64 : panic("abi: go116: not implemented: complex64") - case reflect.Complex128 : panic("abi: go116: not implemented: complex128") - case reflect.Array : panic("abi: go116: not implemented: arrays") - case reflect.Chan : return sp + 8, append(p, mkStack(reflect.TypeOf((chan int)(nil)), sp)) - case reflect.Func : return sp + 8, append(p, mkStack(reflect.TypeOf((func())(nil)), sp)) - case reflect.Map : return sp + 8, append(p, mkStack(reflect.TypeOf((map[int]int)(nil)), sp)) - case reflect.Ptr : return sp + 8, append(p, mkStack(reflect.TypeOf((*int)(nil)), sp)) - case reflect.UnsafePointer : return sp + 8, append(p, mkStack(ptrType, sp)) - case reflect.Interface : return sp + 16, append(p, mkStack(ptrType, sp), mkStack(ptrType, sp + 8)) - case reflect.Slice : return sp + 24, append(p, mkStack(ptrType, sp), mkStack(intType, sp + 8), mkStack(intType, sp + 16)) - case reflect.String : return sp + 16, append(p, mkStack(ptrType, sp), mkStack(intType, sp + 8)) - case reflect.Struct : panic("abi: go116: not implemented: structs") - default : panic("abi: invalid value type") - } + switch vt.Kind() { + case reflect.Bool: + return sp + 8, append(p, mkStack(reflect.TypeOf(false), sp)) + case reflect.Int: + return sp + 8, append(p, mkStack(intType, sp)) + case reflect.Int8: + return sp + 8, append(p, mkStack(reflect.TypeOf(int8(0)), sp)) + case reflect.Int16: + return sp + 8, append(p, mkStack(reflect.TypeOf(int16(0)), sp)) + case reflect.Int32: + return sp + 8, append(p, mkStack(reflect.TypeOf(int32(0)), sp)) + case reflect.Int64: + return sp + 8, append(p, mkStack(reflect.TypeOf(int64(0)), sp)) + case reflect.Uint: + return sp + 8, append(p, mkStack(reflect.TypeOf(uint(0)), sp)) + case reflect.Uint8: + return sp + 8, append(p, mkStack(reflect.TypeOf(uint8(0)), sp)) + case reflect.Uint16: + return sp + 8, append(p, mkStack(reflect.TypeOf(uint16(0)), sp)) + case reflect.Uint32: + return sp + 8, append(p, mkStack(reflect.TypeOf(uint32(0)), sp)) + case reflect.Uint64: + return sp + 8, append(p, mkStack(reflect.TypeOf(uint64(0)), sp)) + case reflect.Uintptr: + return sp + 8, append(p, mkStack(reflect.TypeOf(uintptr(0)), sp)) + case reflect.Float32: + return sp + 8, append(p, mkStack(reflect.TypeOf(float32(0)), sp)) + case reflect.Float64: + return sp + 8, append(p, mkStack(reflect.TypeOf(float64(0)), sp)) + case reflect.Complex64: + panic("abi: go116: not implemented: complex64") + case reflect.Complex128: + panic("abi: go116: not implemented: complex128") + case reflect.Array: + panic("abi: go116: not implemented: arrays") + case reflect.Chan: + return sp + 8, append(p, mkStack(reflect.TypeOf((chan int)(nil)), sp)) + case reflect.Func: + return sp + 8, append(p, mkStack(reflect.TypeOf((func())(nil)), sp)) + case reflect.Map: + return sp + 8, append(p, mkStack(reflect.TypeOf((map[int]int)(nil)), sp)) + case reflect.Ptr: + return sp + 8, append(p, mkStack(reflect.TypeOf((*int)(nil)), sp)) + case reflect.UnsafePointer: + return sp + 8, append(p, mkStack(ptrType, sp)) + case reflect.Interface: + return sp + 16, append(p, mkStack(ptrType, sp), mkStack(ptrType, sp+8)) + case reflect.Slice: + return sp + 24, append(p, mkStack(ptrType, sp), mkStack(intType, sp+8), mkStack(intType, sp+16)) + case reflect.String: + return sp + 16, append(p, mkStack(ptrType, sp), mkStack(intType, sp+8)) + case reflect.Struct: + panic("abi: go116: not implemented: structs") + default: + panic("abi: invalid value type") + } } func NewFunctionLayout(ft reflect.Type) FunctionLayout { - var sp uint32 - var fn FunctionLayout - - /* assign every arguments */ - for i := 0; i < ft.NumIn(); i++ { - sp, fn.Args = salloc(fn.Args, sp, ft.In(i)) - } - - /* assign every return value */ - for i := 0; i < ft.NumOut(); i++ { - sp, fn.Rets = salloc(fn.Rets, sp, ft.Out(i)) - } - - /* update function ID and stack pointer */ - fn.FP = sp - return fn + var sp uint32 + var fn FunctionLayout + + /* assign every arguments */ + for i := 0; i < ft.NumIn(); i++ { + sp, fn.Args = salloc(fn.Args, sp, ft.In(i)) + } + + /* assign every return value */ + for i := 0; i < ft.NumOut(); i++ { + sp, fn.Rets = salloc(fn.Rets, sp, ft.Out(i)) + } + + /* update function ID and stack pointer */ + fn.FP = sp + return fn } func (self *Frame) emitExchangeArgs(p *Program) { - iregArgs, xregArgs := 0, 0 - for _, v := range self.desc.Args { - if v.IsFloat != notFloatKind { - xregArgs += 1 - } else { - iregArgs += 1 - } - } - - if iregArgs > len(iregOrderC) { - panic("too many arguments, only support at most 6 integer arguments now") - } - if xregArgs > len(xregOrderC) { - panic("too many arguments, only support at most 8 float arguments now") - } - - ic, xc := iregArgs, xregArgs - for i := 0; i < len(self.desc.Args); i++ { - arg := self.desc.Args[i] - if arg.IsFloat == floatKind64 { - p.MOVSD(self.argv(i), xregOrderC[xregArgs - xc]) - xc -= 1 - } else if arg.IsFloat == floatKind32 { - p.MOVSS(self.argv(i), xregOrderC[xregArgs - xc]) - xc -= 1 - } else { - p.MOVQ(self.argv(i), iregOrderC[iregArgs - ic]) - ic -= 1 - } - } + iregArgs, xregArgs := 0, 0 + for _, v := range self.desc.Args { + if v.IsFloat != notFloatKind { + xregArgs += 1 + } else { + iregArgs += 1 + } + } + + if iregArgs > len(iregOrderC) { + panic("too many arguments, only support at most 6 integer arguments now") + } + if xregArgs > len(xregOrderC) { + panic("too many arguments, only support at most 8 float arguments now") + } + + ic, xc := iregArgs, xregArgs + for i := 0; i < len(self.desc.Args); i++ { + arg := self.desc.Args[i] + if arg.IsFloat == floatKind64 { + p.MOVSD(self.argv(i), xregOrderC[xregArgs-xc]) + xc -= 1 + } else if arg.IsFloat == floatKind32 { + p.MOVSS(self.argv(i), xregOrderC[xregArgs-xc]) + xc -= 1 + } else { + p.MOVQ(self.argv(i), iregOrderC[iregArgs-ic]) + ic -= 1 + } + } } func (self *Frame) emitStackCheck(p *Program, to *Label, maxStack uintptr) { - // get the current goroutine - switch runtime.GOOS { - case "linux" : p.MOVQ(Abs(-8), R14).FS() - case "darwin" : p.MOVQ(Abs(0x30), R14).GS() - case "windows": break // windows always stores G pointer at R14 - default : panic("unsupported operating system") - } - - // check the stack guard - p.LEAQ(Ptr(RSP, -int32(self.Size() + uint32(maxStack))), RAX) - p.CMPQ(Ptr(R14, _G_stackguard0), RAX) - p.JBE(to) + // get the current goroutine + switch runtime.GOOS { + case "linux": + p.MOVQ(Abs(-8), R14).FS() + case "darwin": + p.MOVQ(Abs(0x30), R14).GS() + case "windows": + break // windows always stores G pointer at R14 + default: + panic("unsupported operating system") + } + + // check the stack guard + p.LEAQ(Ptr(RSP, -int32(self.Size()+uint32(maxStack))), RAX) + p.CMPQ(Ptr(R14, _G_stackguard0), RAX) + p.JBE(to) } func (self *Frame) StackCheckTextSize() uint32 { - p := DefaultArch.CreateProgram() - - // get the current goroutine - switch runtime.GOOS { - case "linux" : p.MOVQ(Abs(-8), R14).FS() - case "darwin" : p.MOVQ(Abs(0x30), R14).GS() - case "windows": break // windows always stores G pointer at R14 - default : panic("unsupported operating system") - } - - // check the stack guard - p.LEAQ(Ptr(RSP, -int32(self.Size())), RAX) - p.CMPQ(Ptr(R14, _G_stackguard0), RAX) - l := CreateLabel("") - p.Link(l) - p.JBE(l) - - return uint32(len(p.Assemble(0))) + p := DefaultArch.CreateProgram() + + // get the current goroutine + switch runtime.GOOS { + case "linux": + p.MOVQ(Abs(-8), R14).FS() + case "darwin": + p.MOVQ(Abs(0x30), R14).GS() + case "windows": + break // windows always stores G pointer at R14 + default: + panic("unsupported operating system") + } + + // check the stack guard + p.LEAQ(Ptr(RSP, -int32(self.Size())), RAX) + p.CMPQ(Ptr(R14, _G_stackguard0), RAX) + l := CreateLabel("") + p.Link(l) + p.JBE(l) + + return uint32(len(p.Assemble(0))) } func (self *Frame) emitExchangeRets(p *Program) { - if len(self.desc.Rets) > 1 { - panic("too many results, only support one result now") - } - // store result - if len(self.desc.Rets) ==1 { - if self.desc.Rets[0].IsFloat == floatKind64 { - p.MOVSD(xregOrderC[0], self.retv(0)) - } else if self.desc.Rets[0].IsFloat == floatKind32 { - p.MOVSS(xregOrderC[0], self.retv(0)) - } else { - p.MOVQ(RAX, self.retv(0)) - } - } + if len(self.desc.Rets) > 1 { + panic("too many results, only support one result now") + } + // store result + if len(self.desc.Rets) == 1 { + if self.desc.Rets[0].IsFloat == floatKind64 { + p.MOVSD(xregOrderC[0], self.retv(0)) + } else if self.desc.Rets[0].IsFloat == floatKind32 { + p.MOVSS(xregOrderC[0], self.retv(0)) + } else { + p.MOVQ(RAX, self.retv(0)) + } + } } func (self *Frame) emitRestoreRegs(p *Program) { - // load reserved registers - for i, r := range ReservedRegs(self.ccall) { - switch r.(type) { - case Register64: - p.MOVQ(self.resv(i), r) - case XMMRegister: - p.MOVSD(self.resv(i), r) - default: - panic(fmt.Sprintf("unsupported register type %t to reserve", r)) - } - } -} \ No newline at end of file + // load reserved registers + for i, r := range ReservedRegs(self.ccall) { + switch r.(type) { + case Register64: + p.MOVQ(self.resv(i), r) + case XMMRegister: + p.MOVSD(self.resv(i), r) + default: + panic(fmt.Sprintf("unsupported register type %t to reserve", r)) + } + } +} diff --git a/loader/internal/abi/abi_regabi_amd64.go b/loader/internal/abi/abi_regabi_amd64.go index 5a31dea89..d4c940de3 100644 --- a/loader/internal/abi/abi_regabi_amd64.go +++ b/loader/internal/abi/abi_regabi_amd64.go @@ -26,10 +26,10 @@ package abi import ( - `fmt` - `reflect` + "fmt" + "reflect" - . `github.com/cloudwego/iasm/x86_64` + x64 "github.com/bytedance/sonic/loader/internal/iasm/x86_64" ) /** Frame Structure of the Generated Function @@ -59,258 +59,287 @@ offs() -------------------------------| RSP -------------------------------|↓ lower addresses */ -const zeroRegGo = XMM15 - -var iregOrderGo = [...]Register64 { - RAX,// RDI - RBX,// RSI - RCX,// RDX - RDI,// RCX - RSI,// R8 - R8, // R9 - R9, - R10, - R11, +const zeroRegGo = x64.XMM15 + +var iregOrderGo = [...]Register64{ + x64.RAX, // RDI + x64.RBX, // RSI + x64.RCX, // RDX + x64.RDI, // RCX + x64.RSI, // R8 + x64.R8, // R9 + x64.R9, + x64.R10, + x64.R11, } -var xregOrderGo = [...]XMMRegister { - XMM0, - XMM1, - XMM2, - XMM3, - XMM4, - XMM5, - XMM6, - XMM7, - XMM8, - XMM9, - XMM10, - XMM11, - XMM12, - XMM13, - XMM14, +var xregOrderGo = [...]XMMRegister{ + x64.XMM0, + x64.XMM1, + x64.XMM2, + x64.XMM3, + x64.XMM4, + x64.XMM5, + x64.XMM6, + x64.XMM7, + x64.XMM8, + x64.XMM9, + x64.XMM10, + x64.XMM11, + x64.XMM12, + x64.XMM13, + x64.XMM14, } func ReservedRegs(callc bool) []Register { - if callc { - return nil - } - return []Register { - R14, // current goroutine - R15, // GOT reference - } + if callc { + return nil + } + return []Register{ + R14, // current goroutine + R15, // GOT reference + } } type stackAlloc struct { - s uint32 - i int - x int + s uint32 + i int + x int } func (self *stackAlloc) reset() { - self.i, self.x = 0, 0 + self.i, self.x = 0, 0 } func (self *stackAlloc) ireg(vt reflect.Type) (p Parameter) { - p = mkIReg(vt, iregOrderGo[self.i]) - self.i++ - return + p = mkIReg(vt, iregOrderGo[self.i]) + self.i++ + return } func (self *stackAlloc) xreg(vt reflect.Type) (p Parameter) { - p = mkXReg(vt, xregOrderGo[self.x]) - self.x++ - return + p = mkXReg(vt, xregOrderGo[self.x]) + self.x++ + return } func (self *stackAlloc) stack(vt reflect.Type) (p Parameter) { - p = mkStack(vt, self.s) - self.s += uint32(vt.Size()) - return + p = mkStack(vt, self.s) + self.s += uint32(vt.Size()) + return } func (self *stackAlloc) spill(n uint32, a int) uint32 { - self.s = alignUp(self.s, a) + n - return self.s + self.s = alignUp(self.s, a) + n + return self.s } func (self *stackAlloc) alloc(p []Parameter, vt reflect.Type) []Parameter { - nb := vt.Size() - vk := vt.Kind() - - /* zero-sized objects are allocated on stack */ - if nb == 0 { - return append(p, mkStack(intType, self.s)) - } - - /* check for value type */ - switch vk { - case reflect.Bool : return self.valloc(p, reflect.TypeOf(false)) - case reflect.Int : return self.valloc(p, intType) - case reflect.Int8 : return self.valloc(p, reflect.TypeOf(int8(0))) - case reflect.Int16 : return self.valloc(p, reflect.TypeOf(int16(0))) - case reflect.Int32 : return self.valloc(p, reflect.TypeOf(uint32(0))) - case reflect.Int64 : return self.valloc(p, reflect.TypeOf(int64(0))) - case reflect.Uint : return self.valloc(p, reflect.TypeOf(uint(0))) - case reflect.Uint8 : return self.valloc(p, reflect.TypeOf(uint8(0))) - case reflect.Uint16 : return self.valloc(p, reflect.TypeOf(uint16(0))) - case reflect.Uint32 : return self.valloc(p, reflect.TypeOf(uint32(0))) - case reflect.Uint64 : return self.valloc(p, reflect.TypeOf(uint64(0))) - case reflect.Uintptr : return self.valloc(p, reflect.TypeOf(uintptr(0))) - case reflect.Float32 : return self.valloc(p, reflect.TypeOf(float32(0))) - case reflect.Float64 : return self.valloc(p, reflect.TypeOf(float64(0))) - case reflect.Complex64 : panic("abi: go117: not implemented: complex64") - case reflect.Complex128 : panic("abi: go117: not implemented: complex128") - case reflect.Array : panic("abi: go117: not implemented: arrays") - case reflect.Chan : return self.valloc(p, reflect.TypeOf((chan int)(nil))) - case reflect.Func : return self.valloc(p, reflect.TypeOf((func())(nil))) - case reflect.Map : return self.valloc(p, reflect.TypeOf((map[int]int)(nil))) - case reflect.Ptr : return self.valloc(p, reflect.TypeOf((*int)(nil))) - case reflect.UnsafePointer : return self.valloc(p, ptrType) - case reflect.Interface : return self.valloc(p, ptrType, ptrType) - case reflect.Slice : return self.valloc(p, ptrType, intType, intType) - case reflect.String : return self.valloc(p, ptrType, intType) - case reflect.Struct : panic("abi: go117: not implemented: structs") - default : panic("abi: invalid value type") - } + nb := vt.Size() + vk := vt.Kind() + + /* zero-sized objects are allocated on stack */ + if nb == 0 { + return append(p, mkStack(intType, self.s)) + } + + /* check for value type */ + switch vk { + case reflect.Bool: + return self.valloc(p, reflect.TypeOf(false)) + case reflect.Int: + return self.valloc(p, intType) + case reflect.Int8: + return self.valloc(p, reflect.TypeOf(int8(0))) + case reflect.Int16: + return self.valloc(p, reflect.TypeOf(int16(0))) + case reflect.Int32: + return self.valloc(p, reflect.TypeOf(uint32(0))) + case reflect.Int64: + return self.valloc(p, reflect.TypeOf(int64(0))) + case reflect.Uint: + return self.valloc(p, reflect.TypeOf(uint(0))) + case reflect.Uint8: + return self.valloc(p, reflect.TypeOf(uint8(0))) + case reflect.Uint16: + return self.valloc(p, reflect.TypeOf(uint16(0))) + case reflect.Uint32: + return self.valloc(p, reflect.TypeOf(uint32(0))) + case reflect.Uint64: + return self.valloc(p, reflect.TypeOf(uint64(0))) + case reflect.Uintptr: + return self.valloc(p, reflect.TypeOf(uintptr(0))) + case reflect.Float32: + return self.valloc(p, reflect.TypeOf(float32(0))) + case reflect.Float64: + return self.valloc(p, reflect.TypeOf(float64(0))) + case reflect.Complex64: + panic("abi: go117: not implemented: complex64") + case reflect.Complex128: + panic("abi: go117: not implemented: complex128") + case reflect.Array: + panic("abi: go117: not implemented: arrays") + case reflect.Chan: + return self.valloc(p, reflect.TypeOf((chan int)(nil))) + case reflect.Func: + return self.valloc(p, reflect.TypeOf((func())(nil))) + case reflect.Map: + return self.valloc(p, reflect.TypeOf((map[int]int)(nil))) + case reflect.Ptr: + return self.valloc(p, reflect.TypeOf((*int)(nil))) + case reflect.UnsafePointer: + return self.valloc(p, ptrType) + case reflect.Interface: + return self.valloc(p, ptrType, ptrType) + case reflect.Slice: + return self.valloc(p, ptrType, intType, intType) + case reflect.String: + return self.valloc(p, ptrType, intType) + case reflect.Struct: + panic("abi: go117: not implemented: structs") + default: + panic("abi: invalid value type") + } } func (self *stackAlloc) valloc(p []Parameter, vts ...reflect.Type) []Parameter { - for _, vt := range vts { - enum := isFloat(vt) - if enum != notFloatKind && self.x < len(xregOrderGo) { - p = append(p, self.xreg(vt)) - } else if enum == notFloatKind && self.i < len(iregOrderGo) { - p = append(p, self.ireg(vt)) - } else { - p = append(p, self.stack(vt)) - } - } - return p + for _, vt := range vts { + enum := isFloat(vt) + if enum != notFloatKind && self.x < len(xregOrderGo) { + p = append(p, self.xreg(vt)) + } else if enum == notFloatKind && self.i < len(iregOrderGo) { + p = append(p, self.ireg(vt)) + } else { + p = append(p, self.stack(vt)) + } + } + return p } func NewFunctionLayout(ft reflect.Type) FunctionLayout { - var sa stackAlloc - var fn FunctionLayout - - /* assign every arguments */ - for i := 0; i < ft.NumIn(); i++ { - fn.Args = sa.alloc(fn.Args, ft.In(i)) - } - - /* reset the register counter, and add a pointer alignment field */ - sa.reset() - - /* assign every return value */ - for i := 0; i < ft.NumOut(); i++ { - fn.Rets = sa.alloc(fn.Rets, ft.Out(i)) - } - - sa.spill(0, PtrAlign) - - /* assign spill slots */ - for i := 0; i < len(fn.Args); i++ { - if fn.Args[i].InRegister { - fn.Args[i].Mem = sa.spill(PtrSize, PtrAlign) - PtrSize - } - } - - /* add the final pointer alignment field */ - fn.FP = sa.spill(0, PtrAlign) - return fn + var sa stackAlloc + var fn FunctionLayout + + /* assign every arguments */ + for i := 0; i < ft.NumIn(); i++ { + fn.Args = sa.alloc(fn.Args, ft.In(i)) + } + + /* reset the register counter, and add a pointer alignment field */ + sa.reset() + + /* assign every return value */ + for i := 0; i < ft.NumOut(); i++ { + fn.Rets = sa.alloc(fn.Rets, ft.Out(i)) + } + + sa.spill(0, PtrAlign) + + /* assign spill slots */ + for i := 0; i < len(fn.Args); i++ { + if fn.Args[i].InRegister { + fn.Args[i].Mem = sa.spill(PtrSize, PtrAlign) - PtrSize + } + } + + /* add the final pointer alignment field */ + fn.FP = sa.spill(0, PtrAlign) + return fn } func (self *Frame) emitExchangeArgs(p *Program) { - iregArgs := make([]Parameter, 0, len(self.desc.Args)) - xregArgs := 0 - for _, v := range self.desc.Args { - if v.InRegister { - if v.IsFloat != notFloatKind { - xregArgs += 1 - } else { - iregArgs = append(iregArgs, v) - } - } else { - panic("not support stack-assgined arguments now") - } - } - if xregArgs > len(xregOrderC) { - panic("too many arguments, only support at most 8 integer register arguments now") - } - - switch len(iregArgs) { - case 0, 1, 2, 3: { - //Fast-Path: when arguments count are less than four, just exchange the registers - for i := 0; i < len(iregArgs); i++ { - p.MOVQ(iregOrderGo[i], iregOrderC[i]) - } - } - case 4, 5, 6: { - // need to spill 3th ~ regArgs registers before exchange - for i := 3; i < len(iregArgs); i++ { - arg := iregArgs[i] - // pointer args have already been spilled - if !arg.IsPointer { - p.MOVQ(iregOrderGo[i], Ptr(RSP, int32(self.Prev() + arg.Mem))) - } - } - p.MOVQ(iregOrderGo[0], iregOrderC[0]) - p.MOVQ(iregOrderGo[1], iregOrderC[1]) - p.MOVQ(iregOrderGo[2], iregOrderC[2]) - for i := 3; i < len(iregArgs); i++ { - arg := iregArgs[i] - p.MOVQ(Ptr(RSP, int32(self.Prev() + arg.Mem)), iregOrderC[i]) - } - } - default: - panic("too many arguments, only support at most 6 integer register arguments now") - } + iregArgs := make([]Parameter, 0, len(self.desc.Args)) + xregArgs := 0 + for _, v := range self.desc.Args { + if v.InRegister { + if v.IsFloat != notFloatKind { + xregArgs += 1 + } else { + iregArgs = append(iregArgs, v) + } + } else { + panic("not support stack-assgined arguments now") + } + } + if xregArgs > len(xregOrderC) { + panic("too many arguments, only support at most 8 integer register arguments now") + } + + switch len(iregArgs) { + case 0, 1, 2, 3: + { + //Fast-Path: when arguments count are less than four, just exchange the registers + for i := 0; i < len(iregArgs); i++ { + p.MOVQ(iregOrderGo[i], iregOrderC[i]) + } + } + case 4, 5, 6: + { + // need to spill 3th ~ regArgs registers before exchange + for i := 3; i < len(iregArgs); i++ { + arg := iregArgs[i] + // pointer args have already been spilled + if !arg.IsPointer { + p.MOVQ(iregOrderGo[i], Ptr(RSP, int32(self.Prev()+arg.Mem))) + } + } + p.MOVQ(iregOrderGo[0], iregOrderC[0]) + p.MOVQ(iregOrderGo[1], iregOrderC[1]) + p.MOVQ(iregOrderGo[2], iregOrderC[2]) + for i := 3; i < len(iregArgs); i++ { + arg := iregArgs[i] + p.MOVQ(Ptr(RSP, int32(self.Prev()+arg.Mem)), iregOrderC[i]) + } + } + default: + panic("too many arguments, only support at most 6 integer register arguments now") + } } func (self *Frame) emitStackCheck(p *Program, to *Label, maxStack uintptr) { - p.LEAQ(Ptr(RSP, int32(-(self.Size() + uint32(maxStack)))), R12) - p.CMPQ(Ptr(R14, _G_stackguard0), R12) - p.JBE(to) + p.LEAQ(Ptr(RSP, int32(-(self.Size()+uint32(maxStack)))), R12) + p.CMPQ(Ptr(R14, _G_stackguard0), R12) + p.JBE(to) } func (self *Frame) StackCheckTextSize() uint32 { - p := DefaultArch.CreateProgram() - p.LEAQ(Ptr(RSP, int32(-(self.Size()))), R12) - p.CMPQ(Ptr(R14, _G_stackguard0), R12) - to := CreateLabel("") - p.Link(to) - p.JBE(to) - return uint32(len(p.Assemble(0))) + p := DefaultArch.CreateProgram() + p.LEAQ(Ptr(RSP, int32(-(self.Size()))), R12) + p.CMPQ(Ptr(R14, _G_stackguard0), R12) + to := CreateLabel("") + p.Link(to) + p.JBE(to) + return uint32(len(p.Assemble(0))) } func (self *Frame) emitExchangeRets(p *Program) { - if len(self.desc.Rets) > 1 { - panic("too many results, only support one result now") - } - // store result - if len(self.desc.Rets) == 1 && !self.desc.Rets[0].InRegister { - if self.desc.Rets[0].IsFloat == floatKind64 { - p.MOVSD(xregOrderC[0], self.retv(0)) - } else if self.desc.Rets[0].IsFloat == floatKind32 { - p.MOVSS(xregOrderC[0], self.retv(0)) - } else { - p.MOVQ(RAX, self.retv(0)) - } - } + if len(self.desc.Rets) > 1 { + panic("too many results, only support one result now") + } + // store result + if len(self.desc.Rets) == 1 && !self.desc.Rets[0].InRegister { + if self.desc.Rets[0].IsFloat == floatKind64 { + p.MOVSD(xregOrderC[0], self.retv(0)) + } else if self.desc.Rets[0].IsFloat == floatKind32 { + p.MOVSS(xregOrderC[0], self.retv(0)) + } else { + p.MOVQ(RAX, self.retv(0)) + } + } } func (self *Frame) emitRestoreRegs(p *Program) { - // load reserved registers - for i, r := range ReservedRegs(self.ccall) { - switch r.(type) { - case Register64: - p.MOVQ(self.resv(i), r) - case XMMRegister: - p.MOVSD(self.resv(i), r) - default: - panic(fmt.Sprintf("unsupported register type %t to reserve", r)) - } - } - // zero xmm15 for go abi - p.XORPS(zeroRegGo, zeroRegGo) -} \ No newline at end of file + // load reserved registers + for i, r := range ReservedRegs(self.ccall) { + switch r.(type) { + case Register64: + p.MOVQ(self.resv(i), r) + case XMMRegister: + p.MOVSD(self.resv(i), r) + default: + panic(fmt.Sprintf("unsupported register type %t to reserve", r)) + } + } + // zero xmm15 for go abi + p.XORPS(zeroRegGo, zeroRegGo) +} diff --git a/loader/internal/iasm/expr/ast.go b/loader/internal/iasm/expr/ast.go new file mode 100644 index 000000000..d340c5c55 --- /dev/null +++ b/loader/internal/iasm/expr/ast.go @@ -0,0 +1,273 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package expr + +import ( + "fmt" +) + +// Type is tyep expression type. +type Type int + +const ( + // CONST indicates that the expression is a constant. + CONST Type = iota + + // TERM indicates that the expression is a Term reference. + TERM + + // EXPR indicates that the expression is a unary or binary expression. + EXPR +) + +var typeNames = map[Type]string{ + EXPR: "Expr", + TERM: "Term", + CONST: "Const", +} + +// String returns the string representation of a Type. +func (self Type) String() string { + if v, ok := typeNames[self]; ok { + return v + } else { + return fmt.Sprintf("expr.Type(%d)", self) + } +} + +// Operator represents an operation to perform when Type is EXPR. +type Operator uint8 + +const ( + // ADD performs "Add Expr.Left and Expr.Right". + ADD Operator = iota + + // SUB performs "Subtract Expr.Left by Expr.Right". + SUB + + // MUL performs "Multiply Expr.Left by Expr.Right". + MUL + + // DIV performs "Divide Expr.Left by Expr.Right". + DIV + + // MOD performs "Modulo Expr.Left by Expr.Right". + MOD + + // AND performs "Bitwise AND Expr.Left and Expr.Right". + AND + + // OR performs "Bitwise OR Expr.Left and Expr.Right". + OR + + // XOR performs "Bitwise XOR Expr.Left and Expr.Right". + XOR + + // SHL performs "Bitwise Shift Expr.Left to the Left by Expr.Right Bits". + SHL + + // SHR performs "Bitwise Shift Expr.Left to the Right by Expr.Right Bits". + SHR + + // POW performs "Raise Expr.Left to the power of Expr.Right" + POW + + // NOT performs "Bitwise Invert Expr.Left". + NOT + + // NEG performs "Negate Expr.Left". + NEG +) + +var operatorNames = map[Operator]string{ + ADD: "Add", + SUB: "Subtract", + MUL: "Multiply", + DIV: "Divide", + MOD: "Modulo", + AND: "And", + OR: "Or", + XOR: "ExclusiveOr", + SHL: "ShiftLeft", + SHR: "ShiftRight", + POW: "Power", + NOT: "Invert", + NEG: "Negate", +} + +// String returns the string representation of a Type. +func (self Operator) String() string { + if v, ok := operatorNames[self]; ok { + return v + } else { + return fmt.Sprintf("expr.Operator(%d)", self) + } +} + +// Expr represents an expression node. +type Expr struct { + Type Type + Term Term + Op Operator + Left *Expr + Right *Expr + Const int64 +} + +// Ref creates an expression from a Term. +func Ref(t Term) (p *Expr) { + p = newExpression() + p.Term = t + p.Type = TERM + return +} + +// Int creates an expression from an integer. +func Int(v int64) (p *Expr) { + p = newExpression() + p.Type = CONST + p.Const = v + return +} + +func (self *Expr) clear() { + if self.Term != nil { + self.Term.Free() + } + if self.Left != nil { + self.Left.Free() + } + if self.Right != nil { + self.Right.Free() + } +} + +// Free returns the Expr into pool. +// Any operation performed after Free is undefined behavior. +func (self *Expr) Free() { + self.clear() + freeExpression(self) +} + +// Evaluate evaluates the expression into an integer. +// It also implements the Term interface. +func (self *Expr) Evaluate() (int64, error) { + switch self.Type { + case EXPR: + return self.eval() + case TERM: + return self.Term.Evaluate() + case CONST: + return self.Const, nil + default: + panic("invalid expression type: " + self.Type.String()) + } +} + +/** Expression Combinator **/ + +func combine(a *Expr, op Operator, b *Expr) (r *Expr) { + r = newExpression() + r.Op = op + r.Type = EXPR + r.Left = a + r.Right = b + return +} + +func (self *Expr) Add(v *Expr) *Expr { return combine(self, ADD, v) } +func (self *Expr) Sub(v *Expr) *Expr { return combine(self, SUB, v) } +func (self *Expr) Mul(v *Expr) *Expr { return combine(self, MUL, v) } +func (self *Expr) Div(v *Expr) *Expr { return combine(self, DIV, v) } +func (self *Expr) Mod(v *Expr) *Expr { return combine(self, MOD, v) } +func (self *Expr) And(v *Expr) *Expr { return combine(self, AND, v) } +func (self *Expr) Or(v *Expr) *Expr { return combine(self, OR, v) } +func (self *Expr) Xor(v *Expr) *Expr { return combine(self, XOR, v) } +func (self *Expr) Shl(v *Expr) *Expr { return combine(self, SHL, v) } +func (self *Expr) Shr(v *Expr) *Expr { return combine(self, SHR, v) } +func (self *Expr) Pow(v *Expr) *Expr { return combine(self, POW, v) } +func (self *Expr) Not() *Expr { return combine(self, NOT, nil) } +func (self *Expr) Neg() *Expr { return combine(self, NEG, nil) } + +/** Expression Evaluator **/ + +var binaryEvaluators = [256]func(int64, int64) (int64, error){ + ADD: func(a, b int64) (int64, error) { return a + b, nil }, + SUB: func(a, b int64) (int64, error) { return a - b, nil }, + MUL: func(a, b int64) (int64, error) { return a * b, nil }, + DIV: idiv, + MOD: imod, + AND: func(a, b int64) (int64, error) { return a & b, nil }, + OR: func(a, b int64) (int64, error) { return a | b, nil }, + XOR: func(a, b int64) (int64, error) { return a ^ b, nil }, + SHL: func(a, b int64) (int64, error) { return a << b, nil }, + SHR: func(a, b int64) (int64, error) { return a >> b, nil }, + POW: ipow, +} + +func (self *Expr) eval() (int64, error) { + var lhs int64 + var rhs int64 + var err error + var vfn func(int64, int64) (int64, error) + + /* evaluate LHS */ + if lhs, err = self.Left.Evaluate(); err != nil { + return 0, err + } + + /* check for unary operators */ + switch self.Op { + case NOT: + return self.unaryNot(lhs) + case NEG: + return self.unaryNeg(lhs) + } + + /* check for operators */ + if vfn = binaryEvaluators[self.Op]; vfn == nil { + panic("invalid operator: " + self.Op.String()) + } + + /* must be a binary expression */ + if self.Right == nil { + panic("operator " + self.Op.String() + " is a binary operator") + } + + /* evaluate RHS, and call the operator */ + if rhs, err = self.Right.Evaluate(); err != nil { + return 0, err + } else { + return vfn(lhs, rhs) + } +} + +func (self *Expr) unaryNot(v int64) (int64, error) { + if self.Right == nil { + return ^v, nil + } else { + panic("operator Invert is an unary operator") + } +} + +func (self *Expr) unaryNeg(v int64) (int64, error) { + if self.Right == nil { + return -v, nil + } else { + panic("operator Negate is an unary operator") + } +} diff --git a/loader/internal/iasm/expr/errors.go b/loader/internal/iasm/expr/errors.go new file mode 100644 index 000000000..791c18bd6 --- /dev/null +++ b/loader/internal/iasm/expr/errors.go @@ -0,0 +1,53 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package expr + +import ( + "fmt" +) + +// SyntaxError represents a syntax error in the expression. +type SyntaxError struct { + Pos int + Reason string +} + +func newSyntaxError(pos int, reason string) *SyntaxError { + return &SyntaxError{ + Pos: pos, + Reason: reason, + } +} + +func (self *SyntaxError) Error() string { + return fmt.Sprintf("Syntax error at position %d: %s", self.Pos, self.Reason) +} + +// RuntimeError is an error which would occure at run time. +type RuntimeError struct { + Reason string +} + +func newRuntimeError(reason string) *RuntimeError { + return &RuntimeError{ + Reason: reason, + } +} + +func (self *RuntimeError) Error() string { + return "Runtime error: " + self.Reason +} diff --git a/loader/internal/iasm/expr/ops.go b/loader/internal/iasm/expr/ops.go new file mode 100644 index 000000000..8502dbc9d --- /dev/null +++ b/loader/internal/iasm/expr/ops.go @@ -0,0 +1,67 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package expr + +import ( + "fmt" +) + +func idiv(v int64, d int64) (int64, error) { + if d != 0 { + return v / d, nil + } else { + return 0, newRuntimeError("division by zero") + } +} + +func imod(v int64, d int64) (int64, error) { + if d != 0 { + return v % d, nil + } else { + return 0, newRuntimeError("division by zero") + } +} + +func ipow(v int64, e int64) (int64, error) { + mul := v + ret := int64(1) + + /* value must be 0 or positive */ + if v < 0 { + return 0, newRuntimeError(fmt.Sprintf("negative base value: %d", v)) + } + + /* exponent must be non-negative */ + if e < 0 { + return 0, newRuntimeError(fmt.Sprintf("negative exponent: %d", e)) + } + + /* fast power first round */ + if (e & 1) != 0 { + ret *= mul + } + + /* fast power remaining rounds */ + for e >>= 1; e != 0; e >>= 1 { + if mul *= mul; (e & 1) != 0 { + ret *= mul + } + } + + /* all done */ + return ret, nil +} diff --git a/loader/internal/iasm/expr/parser.go b/loader/internal/iasm/expr/parser.go new file mode 100644 index 000000000..98d5db68c --- /dev/null +++ b/loader/internal/iasm/expr/parser.go @@ -0,0 +1,331 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package expr + +import ( + "strconv" + "unicode" + "unsafe" +) + +type _TokenKind uint8 + +const ( + _T_end _TokenKind = iota + 1 + _T_int + _T_punc + _T_name +) + +const ( + _OP2 = 0x80 + _POW = _OP2 | '*' + _SHL = _OP2 | '<' + _SHR = _OP2 | '>' +) + +type _Slice struct { + p unsafe.Pointer + n int + c int +} + +type _Token struct { + pos int + ptr *rune + u64 uint64 + tag _TokenKind +} + +func (self _Token) str() (v string) { + return string(self.rbuf()) +} + +func (self _Token) rbuf() (v []rune) { + (*_Slice)(unsafe.Pointer(&v)).c = int(self.u64) + (*_Slice)(unsafe.Pointer(&v)).n = int(self.u64) + (*_Slice)(unsafe.Pointer(&v)).p = unsafe.Pointer(self.ptr) + return +} + +func tokenEnd(p int) _Token { + return _Token{ + pos: p, + tag: _T_end, + } +} + +func tokenInt(p int, v uint64) _Token { + return _Token{ + pos: p, + u64: v, + tag: _T_int, + } +} + +func tokenPunc(p int, v rune) _Token { + return _Token{ + pos: p, + tag: _T_punc, + u64: uint64(v), + } +} + +func tokenName(p int, v []rune) _Token { + return _Token{ + pos: p, + ptr: &v[0], + tag: _T_name, + u64: uint64(len(v)), + } +} + +// Repository represents a repository of Term's. +type Repository interface { + Get(name string) (Term, error) +} + +// Parser parses an expression string to it's AST representation. +type Parser struct { + pos int + src []rune +} + +var binaryOps = [...]func(*Expr, *Expr) *Expr{ + '+': (*Expr).Add, + '-': (*Expr).Sub, + '*': (*Expr).Mul, + '/': (*Expr).Div, + '%': (*Expr).Mod, + '&': (*Expr).And, + '^': (*Expr).Xor, + '|': (*Expr).Or, + _SHL: (*Expr).Shl, + _SHR: (*Expr).Shr, + _POW: (*Expr).Pow, +} + +var precedence = [...]map[int]bool{ + {_SHL: true, _SHR: true}, + {'|': true}, + {'^': true}, + {'&': true}, + {'+': true, '-': true}, + {'*': true, '/': true, '%': true}, + {_POW: true}, +} + +func (self *Parser) ch() rune { + return self.src[self.pos] +} + +func (self *Parser) eof() bool { + return self.pos >= len(self.src) +} + +func (self *Parser) rch() (v rune) { + v, self.pos = self.src[self.pos], self.pos+1 + return +} + +func (self *Parser) hex(ss []rune) bool { + if len(ss) == 1 && ss[0] == '0' { + return unicode.ToLower(self.ch()) == 'x' + } else if len(ss) <= 1 || unicode.ToLower(ss[1]) != 'x' { + return unicode.IsDigit(self.ch()) + } else { + return ishexdigit(self.ch()) + } +} + +func (self *Parser) int(p int, ss []rune) (_Token, error) { + var err error + var val uint64 + + /* find all the digits */ + for !self.eof() && self.hex(ss) { + ss = append(ss, self.rch()) + } + + /* parse the value */ + if val, err = strconv.ParseUint(string(ss), 0, 64); err != nil { + return _Token{}, err + } else { + return tokenInt(p, val), nil + } +} + +func (self *Parser) name(p int, ss []rune) _Token { + for !self.eof() && isident(self.ch()) { + ss = append(ss, self.rch()) + } + return tokenName(p, ss) +} + +func (self *Parser) read(p int, ch rune) (_Token, error) { + if isdigit(ch) { + return self.int(p, []rune{ch}) + } else if isident0(ch) { + return self.name(p, []rune{ch}), nil + } else if isop2ch(ch) && !self.eof() && self.ch() == ch { + return tokenPunc(p, _OP2|self.rch()), nil + } else if isop1ch(ch) { + return tokenPunc(p, ch), nil + } else { + return _Token{}, newSyntaxError(self.pos, "invalid character "+strconv.QuoteRuneToASCII(ch)) + } +} + +func (self *Parser) next() (_Token, error) { + for { + var p int + var c rune + + /* check for EOF */ + if self.eof() { + return tokenEnd(self.pos), nil + } + + /* read the next char */ + p = self.pos + c = self.rch() + + /* parse the token if not a space */ + if !unicode.IsSpace(c) { + return self.read(p, c) + } + } +} + +func (self *Parser) grab(tk _Token, repo Repository) (*Expr, error) { + if repo == nil { + return nil, newSyntaxError(tk.pos, "unresolved symbol: "+tk.str()) + } else if term, err := repo.Get(tk.str()); err != nil { + return nil, err + } else { + return Ref(term), nil + } +} + +func (self *Parser) nest(nest int, repo Repository) (*Expr, error) { + var err error + var ret *Expr + var ntk _Token + + /* evaluate the nested expression */ + if ret, err = self.expr(0, nest+1, repo); err != nil { + return nil, err + } + + /* must follows with a ')' */ + if ntk, err = self.next(); err != nil { + return nil, err + } else if ntk.tag != _T_punc || ntk.u64 != ')' { + return nil, newSyntaxError(ntk.pos, "')' expected") + } else { + return ret, nil + } +} + +func (self *Parser) unit(nest int, repo Repository) (*Expr, error) { + if tk, err := self.next(); err != nil { + return nil, err + } else if tk.tag == _T_int { + return Int(int64(tk.u64)), nil + } else if tk.tag == _T_name { + return self.grab(tk, repo) + } else if tk.tag == _T_punc && tk.u64 == '(' { + return self.nest(nest, repo) + } else if tk.tag == _T_punc && tk.u64 == '+' { + return self.unit(nest, repo) + } else if tk.tag == _T_punc && tk.u64 == '-' { + return neg2(self.unit(nest, repo)) + } else if tk.tag == _T_punc && tk.u64 == '~' { + return not2(self.unit(nest, repo)) + } else { + return nil, newSyntaxError(tk.pos, "integer, unary operator or nested expression expected") + } +} + +func (self *Parser) term(prec int, nest int, repo Repository) (*Expr, error) { + var err error + var val *Expr + + /* parse the LHS operand */ + if val, err = self.expr(prec+1, nest, repo); err != nil { + return nil, err + } + + /* parse all the operators of the same precedence */ + for { + var op int + var rv *Expr + var tk _Token + + /* peek the next token */ + pp := self.pos + tk, err = self.next() + + /* check for errors */ + if err != nil { + return nil, err + } + + /* encountered EOF */ + if tk.tag == _T_end { + return val, nil + } + + /* must be an operator */ + if tk.tag != _T_punc { + return nil, newSyntaxError(tk.pos, "operators expected") + } + + /* check for the operator precedence */ + if op = int(tk.u64); !precedence[prec][op] { + self.pos = pp + return val, nil + } + + /* evaluate the RHS operand, and combine the value */ + if rv, err = self.expr(prec+1, nest, repo); err != nil { + return nil, err + } else { + val = binaryOps[op](val, rv) + } + } +} + +func (self *Parser) expr(prec int, nest int, repo Repository) (*Expr, error) { + if prec >= len(precedence) { + return self.unit(nest, repo) + } else { + return self.term(prec, nest, repo) + } +} + +// Parse parses the expression, and returns it's AST tree. +func (self *Parser) Parse(repo Repository) (*Expr, error) { + return self.expr(0, 0, repo) +} + +// SetSource resets the expression parser and sets the expression source. +func (self *Parser) SetSource(src string) *Parser { + self.pos = 0 + self.src = []rune(src) + return self +} diff --git a/loader/internal/iasm/expr/parser_test.go b/loader/internal/iasm/expr/parser_test.go new file mode 100644 index 000000000..ecb893904 --- /dev/null +++ b/loader/internal/iasm/expr/parser_test.go @@ -0,0 +1,33 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package expr + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParser_Eval(t *testing.T) { + p := new(Parser).SetSource(`3 - 2 * (5 + 6) ** 4 / 7 + (1 << (1234 % 23)) & 0x5436 ^ 0x5a5a - 2 | 1`) + v, err := p.Parse(nil) + require.NoError(t, err) + r, err := v.Evaluate() + require.NoError(t, err) + assert.Equal(t, int64(7805), r) +} diff --git a/loader/internal/iasm/expr/pools.go b/loader/internal/iasm/expr/pools.go new file mode 100644 index 000000000..77410b4be --- /dev/null +++ b/loader/internal/iasm/expr/pools.go @@ -0,0 +1,42 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package expr + +import ( + "sync" +) + +var ( + expressionPool sync.Pool +) + +func newExpression() *Expr { + if v := expressionPool.Get(); v == nil { + return new(Expr) + } else { + return resetExpression(v.(*Expr)) + } +} + +func freeExpression(p *Expr) { + expressionPool.Put(p) +} + +func resetExpression(p *Expr) *Expr { + *p = Expr{} + return p +} diff --git a/loader/internal/iasm/expr/term.go b/loader/internal/iasm/expr/term.go new file mode 100644 index 000000000..4fe723989 --- /dev/null +++ b/loader/internal/iasm/expr/term.go @@ -0,0 +1,23 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package expr + +// Term represents a value that can Evaluate() into an integer. +type Term interface { + Free() + Evaluate() (int64, error) +} diff --git a/loader/internal/iasm/expr/utils.go b/loader/internal/iasm/expr/utils.go new file mode 100644 index 000000000..57b4fd842 --- /dev/null +++ b/loader/internal/iasm/expr/utils.go @@ -0,0 +1,77 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package expr + +var op1ch = [...]bool{ + '+': true, + '-': true, + '*': true, + '/': true, + '%': true, + '&': true, + '|': true, + '^': true, + '~': true, + '(': true, + ')': true, +} + +var op2ch = [...]bool{ + '*': true, + '<': true, + '>': true, +} + +func neg2(v *Expr, err error) (*Expr, error) { + if err != nil { + return nil, err + } else { + return v.Neg(), nil + } +} + +func not2(v *Expr, err error) (*Expr, error) { + if err != nil { + return nil, err + } else { + return v.Not(), nil + } +} + +func isop1ch(ch rune) bool { + return ch >= 0 && int(ch) < len(op1ch) && op1ch[ch] +} + +func isop2ch(ch rune) bool { + return ch >= 0 && int(ch) < len(op2ch) && op2ch[ch] +} + +func isdigit(ch rune) bool { + return ch >= '0' && ch <= '9' +} + +func isident(ch rune) bool { + return isdigit(ch) || isident0(ch) +} + +func isident0(ch rune) bool { + return (ch == '_') || (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') +} + +func ishexdigit(ch rune) bool { + return isdigit(ch) || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F') +} diff --git a/loader/internal/iasm/obj/macho.go b/loader/internal/iasm/obj/macho.go new file mode 100644 index 000000000..10cbaf9dd --- /dev/null +++ b/loader/internal/iasm/obj/macho.go @@ -0,0 +1,252 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package obj + +import ( + "encoding/binary" + "io" + "unsafe" +) + +type MachOHeader struct { + Magic uint32 + CPUType uint32 + CPUSubType uint32 + FileType uint32 + CmdCount uint32 + CmdSize uint32 + Flags uint32 + _ uint32 +} + +type SegmentCommand struct { + Cmd uint32 + Size uint32 + Name [16]byte + VMAddr uint64 + VMSize uint64 + FileOffset uint64 + FileSize uint64 + MaxProtect uint32 + InitProtect uint32 + SectionCount uint32 + Flags uint32 +} + +type SegmentSection struct { + Name [16]byte + SegName [16]byte + Addr uint64 + Size uint64 + Offset uint32 + Align uint32 + RelOffset uint32 + RelCount uint32 + Flags uint32 + _ [3]uint32 +} + +type Registers struct { + RAX uint64 + RBX uint64 + RCX uint64 + RDX uint64 + RDI uint64 + RSI uint64 + RBP uint64 + RSP uint64 + R8 uint64 + R9 uint64 + R10 uint64 + R11 uint64 + R12 uint64 + R13 uint64 + R14 uint64 + R15 uint64 + RIP uint64 + RFLAGS uint64 + CS uint64 + FS uint64 + GS uint64 +} + +type UnixThreadCommand struct { + Cmd uint32 + Size uint32 + Flavor uint32 + Count uint32 + Regs Registers +} + +const ( + _MH_MAGIC_64 = 0xfeedfacf + _MH_EXECUTE = 0x02 + _MH_NOUNDEFS = 0x01 +) + +const ( + _CPU_TYPE_I386 = 0x00000007 + _CPU_ARCH_ABI64 = 0x01000000 +) + +const ( + _CPU_SUBTYPE_LIB64 = 0x80000000 + _CPU_SUBTYPE_I386_ALL = 0x00000003 +) + +const ( + _LC_SEGMENT_64 = 0x19 + _LC_UNIXTHREAD = 0x05 +) + +const ( + _VM_PROT_READ = 0x01 + _VM_PROT_WRITE = 0x02 + _VM_PROT_EXECUTE = 0x04 +) + +const ( + _S_ATTR_SOME_INSTRUCTIONS = 0x00000400 + _S_ATTR_PURE_INSTRUCTIONS = 0x80000000 +) + +const ( + _x86_THREAD_STATE64 = 0x04 + _x86_EXCEPTION_STATE64_COUNT = 42 +) + +const ( + _MACHO_SIZE = uint32(unsafe.Sizeof(MachOHeader{})) + _SEGMENT_SIZE = uint32(unsafe.Sizeof(SegmentCommand{})) + _SECTION_SIZE = uint32(unsafe.Sizeof(SegmentSection{})) + _UNIXTHREAD_SIZE = uint32(unsafe.Sizeof(UnixThreadCommand{})) +) + +const ( + _IMAGE_SIZE = 4096 + _IMAGE_BASE = 0x04000000 +) + +const ( + _HDR_SIZE = _MACHO_SIZE + _SEGMENT_SIZE*2 + _SECTION_SIZE + _UNIXTHREAD_SIZE + _ZERO_SIZE = (_IMAGE_SIZE - _HDR_SIZE%_IMAGE_SIZE) % _IMAGE_SIZE +) + +var ( + zeroBytes = [_ZERO_SIZE]byte{} +) + +func assembleMachO(w io.Writer, code []byte, base uint64, entry uint64) error { + var p0name [16]byte + var txname [16]byte + var tsname [16]byte + + /* segment names */ + copy(tsname[:], "__text") + copy(txname[:], "__TEXT") + copy(p0name[:], "__PAGEZERO") + + /* calculate size of code */ + clen := uint64(len(code)) + hlen := uint64(_HDR_SIZE + _ZERO_SIZE) + + /* Mach-O does not allow image base at zero */ + if base == 0 { + base = _IMAGE_BASE + entry += _IMAGE_BASE + } + + /* Page-0 Segment */ + p0 := SegmentCommand{ + Cmd: _LC_SEGMENT_64, + Size: _SEGMENT_SIZE, + Name: p0name, + VMSize: base, + } + + /* TEXT Segment */ + text := SegmentCommand{ + Cmd: _LC_SEGMENT_64, + Size: _SEGMENT_SIZE + _SECTION_SIZE, + Name: txname, + VMAddr: base, + VMSize: hlen + clen, + FileSize: hlen + clen, + MaxProtect: _VM_PROT_READ | _VM_PROT_WRITE | _VM_PROT_EXECUTE, + InitProtect: _VM_PROT_READ | _VM_PROT_EXECUTE, + SectionCount: 1, + } + + /* __TEXT.__text section */ + tsec := SegmentSection{ + Name: tsname, + SegName: txname, + Addr: base + hlen, + Size: clen, + Offset: uint32(hlen), + Flags: _S_ATTR_SOME_INSTRUCTIONS | _S_ATTR_PURE_INSTRUCTIONS, + } + + /* UNIX Thread Metadata */ + unix := UnixThreadCommand{ + Cmd: _LC_UNIXTHREAD, + Size: _UNIXTHREAD_SIZE, + Flavor: _x86_THREAD_STATE64, + Count: _x86_EXCEPTION_STATE64_COUNT, + Regs: Registers{RIP: hlen + entry}, + } + + /* Mach-O Header */ + macho := MachOHeader{ + Magic: _MH_MAGIC_64, + CPUType: _CPU_ARCH_ABI64 | _CPU_TYPE_I386, + CPUSubType: _CPU_SUBTYPE_LIB64 | _CPU_SUBTYPE_I386_ALL, + FileType: _MH_EXECUTE, + CmdCount: 3, + CmdSize: _SEGMENT_SIZE*2 + _SECTION_SIZE + _UNIXTHREAD_SIZE, + Flags: _MH_NOUNDEFS, + } + + /* write the headers */ + if err := binary.Write(w, binary.LittleEndian, &macho); err != nil { + return err + } + if err := binary.Write(w, binary.LittleEndian, &p0); err != nil { + return err + } + if err := binary.Write(w, binary.LittleEndian, &text); err != nil { + return err + } + if err := binary.Write(w, binary.LittleEndian, &tsec); err != nil { + return err + } + if err := binary.Write(w, binary.LittleEndian, &unix); err != nil { + return err + } + if err := binary.Write(w, binary.LittleEndian, &zeroBytes); err != nil { + return err + } + + /* write the code */ + if n, err := w.Write(code); err != nil { + return err + } else if n != len(code) { + return io.ErrShortWrite + } else { + return nil + } +} diff --git a/loader/internal/iasm/obj/macho_test.go b/loader/internal/iasm/obj/macho_test.go new file mode 100644 index 000000000..83bfb4721 --- /dev/null +++ b/loader/internal/iasm/obj/macho_test.go @@ -0,0 +1,59 @@ +//go:build darwin +// +build darwin + +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package obj + +import ( + "os" + "os/exec" + "testing" + + "github.com/davecgh/go-spew/spew" + "github.com/stretchr/testify/require" +) + +func TestMachO_Create(t *testing.T) { + fp, err := os.CreateTemp("", "macho_out-") + require.NoError(t, err) + code := []byte{ + 0x48, 0xc7, 0xc7, 0x01, 0x00, 0x00, 0x00, // MOVQ $1, %rdi + 0x48, 0x8d, 0x35, 0x1b, 0x00, 0x00, 0x00, // LEAQ 0x1b(%rip), %rsi + 0x48, 0xc7, 0xc2, 0x0e, 0x00, 0x00, 0x00, // MOVQ $14, %rdx + 0x48, 0xc7, 0xc0, 0x04, 0x00, 0x00, 0x02, // MOVQ $0x02000004, %rax + 0x0f, 0x05, // SYSCALL + 0x31, 0xff, // XORL %edi, %edi + 0x48, 0xc7, 0xc0, 0x01, 0x00, 0x00, 0x02, // MOVQ $0x02000001, %rax + 0x0f, 0x05, // SYSCALL + 'h', 'e', 'l', 'l', 'o', ',', ' ', + 'w', 'o', 'r', 'l', 'd', '\r', '\n', + } + err = assembleMachO(fp, code, 0, 0) + require.NoError(t, err) + err = fp.Close() + require.NoError(t, err) + err = os.Chmod(fp.Name(), 0755) + require.NoError(t, err) + println("Saved to", fp.Name()) + out, err := exec.Command(fp.Name()).Output() + require.NoError(t, err) + spew.Dump(out) + require.Equal(t, []byte("hello, world\r\n"), out) + err = os.Remove(fp.Name()) + require.NoError(t, err) +} diff --git a/loader/internal/iasm/obj/obj.go b/loader/internal/iasm/obj/obj.go new file mode 100644 index 000000000..3b891925f --- /dev/null +++ b/loader/internal/iasm/obj/obj.go @@ -0,0 +1,85 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package obj + +import ( + "fmt" + "io" + "os" +) + +// Format represents the saved binary file format. +type Format int + +const ( + // ELF indicates it should be saved as an ELF binary. + ELF Format = iota + + // MachO indicates it should be saved as a Mach-O binary. + MachO +) + +var formatTab = [...]func(w io.Writer, code []byte, base uint64, entry uint64) error{ + ELF: nil, + MachO: assembleMachO, +} + +var formatNames = map[Format]string{ + ELF: "ELF", + MachO: "Mach-O", +} + +// String returns the name of a specified format. +func (self Format) String() string { + if v, ok := formatNames[self]; ok { + return v + } else { + return fmt.Sprintf("Format(%d)", self) + } +} + +// Write assembles a binary executable. +func (self Format) Write(w io.Writer, code []byte, base uint64, entry uint64) error { + if self >= 0 && int(self) < len(formatTab) && formatTab[self] != nil { + return formatTab[self](w, code, base, entry) + } else { + return fmt.Errorf("unsupported format: %s", self) + } +} + +// Generate generates a binary executable file from the specified code. +func (self Format) Generate(name string, code []byte, base uint64, entry uint64) error { + var fp *os.File + var err error + + /* create the output file */ + if fp, err = os.Create(name); err != nil { + return err + } + + /* generate the code */ + if err = self.Write(fp, code, base, entry); err != nil { + _ = fp.Close() + _ = os.Remove(name) + return err + } + + /* close the file and make it executable */ + _ = fp.Close() + _ = os.Chmod(name, 0755) + return nil +} diff --git a/loader/internal/iasm/sync.sh b/loader/internal/iasm/sync.sh new file mode 100755 index 000000000..9423a7585 --- /dev/null +++ b/loader/internal/iasm/sync.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# +# Copyright 2025 ByteDance Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# This script is used to synchronize the code of cloudwego/iasm, +# and trim it for minimizing the size of the compiled binary file. + +set -e + +echo "Cloning cloudwego/iasm into 'tmp'..." + +IASM_TAG="v0.2.0" +rm -rf tmp +if git clone --branch $IASM_TAG --depth 1 git@github.com:cloudwego/iasm.git tmp >/dev/null 2>&1; then + echo "done" +else + echo "git clone failed" + exit 1 +fi + +rm -rf expr +cp -r tmp/expr . + +rm -rf obj +cp -r tmp/obj . + +rm -rf x86_64 +cp -r tmp/x86_64 . + +rm -rf tmp + +# rm unused code +rm ./x86_64/assembler* + +# replace import +sed -i.bak 's:github.com/cloudwego:github.com/bytedance/sonic/loader/internal:g' ./x86_64/*.go && rm ./x86_64/*.bak + +# trim file methods +./trim.py + +gofmt -w */*.go + +echo "done" diff --git a/loader/internal/iasm/trim.py b/loader/internal/iasm/trim.py new file mode 100755 index 000000000..f3e01ed86 --- /dev/null +++ b/loader/internal/iasm/trim.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +# +# Copyright 2025 ByteDance Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# This file is used by sync.sh for trimming x86_64/instructions.go and x86_64/instructions_table.go + +filename = "x86_64/instructions.go" +print("trimming", filename) + +keep_methods = { + "ADDQ", + "CALLQ", + "CMPQ", + "JBE", + "JMP", + "LEAQ", + "MOVQ", + "MOVSD", + "MOVSS", + "RET", + "SUBQ", + "XORPS", + + # for _test.go + "VPERMIL2PD", + "MOVSLQ", + "JMPQ", +} +lines = [] +keep = True + +for line in open(filename): + if line.startswith("//") and "performs" in line: + method = line.split(" ")[1] + if method in keep_methods: + keep = True + keep = method in keep_methods + if keep: + lines.append(line.rstrip()) + +with open(filename, "w") as f: + f.write("\n".join(lines)) + + +filename = "x86_64/instructions_table.go" +print("trimming", filename) + +keep_before = "// Instructions" +lines = [] +keep = True + +for line in open(filename): + if keep and keep_before in line: + keep = False + if keep: + lines.append(line.rstrip()) + +with open(filename, "w") as f: + f.write("\n".join(lines)) diff --git a/loader/internal/iasm/x86_64/arch.go b/loader/internal/iasm/x86_64/arch.go new file mode 100644 index 000000000..f33cc9ca6 --- /dev/null +++ b/loader/internal/iasm/x86_64/arch.go @@ -0,0 +1,251 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package x86_64 + +import ( + "fmt" +) + +// ISA represents an extension to x86-64 instruction set. +type ISA uint64 + +const ( + ISA_CPUID ISA = 1 << iota + ISA_RDTSC + ISA_RDTSCP + ISA_CMOV + ISA_MOVBE + ISA_POPCNT + ISA_LZCNT + ISA_TBM + ISA_BMI + ISA_BMI2 + ISA_ADX + ISA_MMX + ISA_MMX_PLUS + ISA_FEMMS + ISA_3DNOW + ISA_3DNOW_PLUS + ISA_SSE + ISA_SSE2 + ISA_SSE3 + ISA_SSSE3 + ISA_SSE4A + ISA_SSE4_1 + ISA_SSE4_2 + ISA_FMA3 + ISA_FMA4 + ISA_XOP + ISA_F16C + ISA_AVX + ISA_AVX2 + ISA_AVX512F + ISA_AVX512BW + ISA_AVX512DQ + ISA_AVX512VL + ISA_AVX512PF + ISA_AVX512ER + ISA_AVX512CD + ISA_AVX512VBMI + ISA_AVX512IFMA + ISA_AVX512VPOPCNTDQ + ISA_AVX512_4VNNIW + ISA_AVX512_4FMAPS + ISA_PREFETCH + ISA_PREFETCHW + ISA_PREFETCHWT1 + ISA_CLFLUSH + ISA_CLFLUSHOPT + ISA_CLWB + ISA_CLZERO + ISA_RDRAND + ISA_RDSEED + ISA_PCLMULQDQ + ISA_AES + ISA_SHA + ISA_MONITOR + ISA_MONITORX + ISA_ALL = ^ISA(0) +) + +var _ISA_NAMES = map[ISA]string{ + ISA_CPUID: "CPUID", + ISA_RDTSC: "RDTSC", + ISA_RDTSCP: "RDTSCP", + ISA_CMOV: "CMOV", + ISA_MOVBE: "MOVBE", + ISA_POPCNT: "POPCNT", + ISA_LZCNT: "LZCNT", + ISA_TBM: "TBM", + ISA_BMI: "BMI", + ISA_BMI2: "BMI2", + ISA_ADX: "ADX", + ISA_MMX: "MMX", + ISA_MMX_PLUS: "MMX+", + ISA_FEMMS: "FEMMS", + ISA_3DNOW: "3dnow!", + ISA_3DNOW_PLUS: "3dnow!+", + ISA_SSE: "SSE", + ISA_SSE2: "SSE2", + ISA_SSE3: "SSE3", + ISA_SSSE3: "SSSE3", + ISA_SSE4A: "SSE4A", + ISA_SSE4_1: "SSE4.1", + ISA_SSE4_2: "SSE4.2", + ISA_FMA3: "FMA3", + ISA_FMA4: "FMA4", + ISA_XOP: "XOP", + ISA_F16C: "F16C", + ISA_AVX: "AVX", + ISA_AVX2: "AVX2", + ISA_AVX512F: "AVX512F", + ISA_AVX512BW: "AVX512BW", + ISA_AVX512DQ: "AVX512DQ", + ISA_AVX512VL: "AVX512VL", + ISA_AVX512PF: "AVX512PF", + ISA_AVX512ER: "AVX512ER", + ISA_AVX512CD: "AVX512CD", + ISA_AVX512VBMI: "AVX512VBMI", + ISA_AVX512IFMA: "AVX512IFMA", + ISA_AVX512VPOPCNTDQ: "AVX512VPOPCNTDQ", + ISA_AVX512_4VNNIW: "AVX512_4VNNIW", + ISA_AVX512_4FMAPS: "AVX512_4FMAPS", + ISA_PREFETCH: "PREFETCH", + ISA_PREFETCHW: "PREFETCHW", + ISA_PREFETCHWT1: "PREFETCHWT1", + ISA_CLFLUSH: "CLFLUSH", + ISA_CLFLUSHOPT: "CLFLUSHOPT", + ISA_CLWB: "CLWB", + ISA_CLZERO: "CLZERO", + ISA_RDRAND: "RDRAND", + ISA_RDSEED: "RDSEED", + ISA_PCLMULQDQ: "PCLMULQDQ", + ISA_AES: "AES", + ISA_SHA: "SHA", + ISA_MONITOR: "MONITOR", + ISA_MONITORX: "MONITORX", +} + +var _ISA_MAPPING = map[string]ISA{ + "CPUID": ISA_CPUID, + "RDTSC": ISA_RDTSC, + "RDTSCP": ISA_RDTSCP, + "CMOV": ISA_CMOV, + "MOVBE": ISA_MOVBE, + "POPCNT": ISA_POPCNT, + "LZCNT": ISA_LZCNT, + "TBM": ISA_TBM, + "BMI": ISA_BMI, + "BMI2": ISA_BMI2, + "ADX": ISA_ADX, + "MMX": ISA_MMX, + "MMX+": ISA_MMX_PLUS, + "FEMMS": ISA_FEMMS, + "3dnow!": ISA_3DNOW, + "3dnow!+": ISA_3DNOW_PLUS, + "SSE": ISA_SSE, + "SSE2": ISA_SSE2, + "SSE3": ISA_SSE3, + "SSSE3": ISA_SSSE3, + "SSE4A": ISA_SSE4A, + "SSE4.1": ISA_SSE4_1, + "SSE4.2": ISA_SSE4_2, + "FMA3": ISA_FMA3, + "FMA4": ISA_FMA4, + "XOP": ISA_XOP, + "F16C": ISA_F16C, + "AVX": ISA_AVX, + "AVX2": ISA_AVX2, + "AVX512F": ISA_AVX512F, + "AVX512BW": ISA_AVX512BW, + "AVX512DQ": ISA_AVX512DQ, + "AVX512VL": ISA_AVX512VL, + "AVX512PF": ISA_AVX512PF, + "AVX512ER": ISA_AVX512ER, + "AVX512CD": ISA_AVX512CD, + "AVX512VBMI": ISA_AVX512VBMI, + "AVX512IFMA": ISA_AVX512IFMA, + "AVX512VPOPCNTDQ": ISA_AVX512VPOPCNTDQ, + "AVX512_4VNNIW": ISA_AVX512_4VNNIW, + "AVX512_4FMAPS": ISA_AVX512_4FMAPS, + "PREFETCH": ISA_PREFETCH, + "PREFETCHW": ISA_PREFETCHW, + "PREFETCHWT1": ISA_PREFETCHWT1, + "CLFLUSH": ISA_CLFLUSH, + "CLFLUSHOPT": ISA_CLFLUSHOPT, + "CLWB": ISA_CLWB, + "CLZERO": ISA_CLZERO, + "RDRAND": ISA_RDRAND, + "RDSEED": ISA_RDSEED, + "PCLMULQDQ": ISA_PCLMULQDQ, + "AES": ISA_AES, + "SHA": ISA_SHA, + "MONITOR": ISA_MONITOR, + "MONITORX": ISA_MONITORX, +} + +func (self ISA) String() string { + if v, ok := _ISA_NAMES[self]; ok { + return v + } else { + return fmt.Sprintf("(invalid: %#x)", uint64(self)) + } +} + +// ParseISA parses name into ISA, it will panic if the name is invalid. +func ParseISA(name string) ISA { + if v, ok := _ISA_MAPPING[name]; ok { + return v + } else { + panic("invalid ISA name: " + name) + } +} + +// Arch represents the x86_64 architecture. +type Arch struct { + isa ISA +} + +// DefaultArch is the default architecture with all ISA enabled. +var DefaultArch = CreateArch() + +// CreateArch creates a new Arch with all ISA enabled. +func CreateArch() *Arch { + return new(Arch).EnableISA(ISA_ALL) +} + +// HasISA checks if a particular ISA was enabled. +func (self *Arch) HasISA(isa ISA) bool { + return (self.isa & isa) != 0 +} + +// EnableISA enables a particular ISA. +func (self *Arch) EnableISA(isa ISA) *Arch { + self.isa |= isa + return self +} + +// DisableISA disables a particular ISA. +func (self *Arch) DisableISA(isa ISA) *Arch { + self.isa &^= isa + return self +} + +// CreateProgram creates a new empty program. +func (self *Arch) CreateProgram() *Program { + return newProgram(self) +} diff --git a/loader/internal/iasm/x86_64/asm.s b/loader/internal/iasm/x86_64/asm.s new file mode 100644 index 000000000..a00c41dfc --- /dev/null +++ b/loader/internal/iasm/x86_64/asm.s @@ -0,0 +1,16 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + diff --git a/loader/internal/iasm/x86_64/eface.go b/loader/internal/iasm/x86_64/eface.go new file mode 100644 index 000000000..eb7f3c406 --- /dev/null +++ b/loader/internal/iasm/x86_64/eface.go @@ -0,0 +1,79 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package x86_64 + +import ( + "reflect" + "unsafe" +) + +type _GoType struct { + size uintptr + pdata uintptr + hash uint32 + flags uint8 + align uint8 + falign uint8 + kflags uint8 + traits unsafe.Pointer + gcdata *byte + str int32 + ptrx int32 +} + +const ( + _KindMask = (1 << 5) - 1 +) + +func (self *_GoType) kind() reflect.Kind { + return reflect.Kind(self.kflags & _KindMask) +} + +type _GoSlice struct { + ptr unsafe.Pointer + len int + cap int +} + +type _GoEface struct { + vt *_GoType + ptr unsafe.Pointer +} + +func (self *_GoEface) kind() reflect.Kind { + if self.vt != nil { + return self.vt.kind() + } else { + return reflect.Invalid + } +} + +func (self *_GoEface) toInt64() int64 { + if self.vt.size == 8 { + return *(*int64)(self.ptr) + } else if self.vt.size == 4 { + return int64(*(*int32)(self.ptr)) + } else if self.vt.size == 2 { + return int64(*(*int16)(self.ptr)) + } else { + return int64(*(*int8)(self.ptr)) + } +} + +func efaceOf(v interface{}) _GoEface { + return *(*_GoEface)(unsafe.Pointer(&v)) +} diff --git a/loader/internal/iasm/x86_64/encodings.go b/loader/internal/iasm/x86_64/encodings.go new file mode 100644 index 000000000..f003be927 --- /dev/null +++ b/loader/internal/iasm/x86_64/encodings.go @@ -0,0 +1,836 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package x86_64 + +import ( + "encoding/binary" + "math" +) + +/** Operand Encoding Helpers **/ + +func imml(v interface{}) byte { + return byte(toImmAny(v) & 0x0f) +} + +func relv(v interface{}) int64 { + switch r := v.(type) { + case *Label: + return 0 + case RelativeOffset: + return int64(r) + default: + panic("invalid relative offset") + } +} + +func addr(v interface{}) interface{} { + switch a := v.(*MemoryOperand).Addr; a.Type { + case Memory: + return a.Memory + case Offset: + return a.Offset + case Reference: + return a.Reference + default: + panic("invalid memory operand type") + } +} + +func bcode(v interface{}) byte { + if m, ok := v.(*MemoryOperand); !ok { + panic("v is not a memory operand") + } else if m.Broadcast == 0 { + return 0 + } else { + return 1 + } +} + +func vcode(v interface{}) byte { + switch r := v.(type) { + case XMMRegister: + return byte(r) + case YMMRegister: + return byte(r) + case ZMMRegister: + return byte(r) + case MaskedRegister: + return vcode(r.Reg) + default: + panic("v is not a vector register") + } +} + +func kcode(v interface{}) byte { + switch r := v.(type) { + case KRegister: + return byte(r) + case XMMRegister: + return 0 + case YMMRegister: + return 0 + case ZMMRegister: + return 0 + case RegisterMask: + return byte(r.K) + case MaskedRegister: + return byte(r.Mask.K) + case *MemoryOperand: + return toKcodeMem(r) + default: + panic("v is not a maskable operand") + } +} + +func zcode(v interface{}) byte { + switch r := v.(type) { + case KRegister: + return 0 + case XMMRegister: + return 0 + case YMMRegister: + return 0 + case ZMMRegister: + return 0 + case RegisterMask: + return toZcodeRegM(r) + case MaskedRegister: + return toZcodeRegM(r.Mask) + case *MemoryOperand: + return toZcodeMem(r) + default: + panic("v is not a maskable operand") + } +} + +func lcode(v interface{}) byte { + switch r := v.(type) { + case Register8: + return byte(r & 0x07) + case Register16: + return byte(r & 0x07) + case Register32: + return byte(r & 0x07) + case Register64: + return byte(r & 0x07) + case KRegister: + return byte(r & 0x07) + case MMRegister: + return byte(r & 0x07) + case XMMRegister: + return byte(r & 0x07) + case YMMRegister: + return byte(r & 0x07) + case ZMMRegister: + return byte(r & 0x07) + case MaskedRegister: + return lcode(r.Reg) + default: + panic("v is not a register") + } +} + +func hcode(v interface{}) byte { + switch r := v.(type) { + case Register8: + return byte(r>>3) & 1 + case Register16: + return byte(r>>3) & 1 + case Register32: + return byte(r>>3) & 1 + case Register64: + return byte(r>>3) & 1 + case KRegister: + return byte(r>>3) & 1 + case MMRegister: + return byte(r>>3) & 1 + case XMMRegister: + return byte(r>>3) & 1 + case YMMRegister: + return byte(r>>3) & 1 + case ZMMRegister: + return byte(r>>3) & 1 + case MaskedRegister: + return hcode(r.Reg) + default: + panic("v is not a register") + } +} + +func ecode(v interface{}) byte { + switch r := v.(type) { + case Register8: + return byte(r>>4) & 1 + case Register16: + return byte(r>>4) & 1 + case Register32: + return byte(r>>4) & 1 + case Register64: + return byte(r>>4) & 1 + case KRegister: + return byte(r>>4) & 1 + case MMRegister: + return byte(r>>4) & 1 + case XMMRegister: + return byte(r>>4) & 1 + case YMMRegister: + return byte(r>>4) & 1 + case ZMMRegister: + return byte(r>>4) & 1 + case MaskedRegister: + return ecode(r.Reg) + default: + panic("v is not a register") + } +} + +func hlcode(v interface{}) byte { + switch r := v.(type) { + case Register8: + return toHLcodeReg8(r) + case Register16: + return byte(r & 0x0f) + case Register32: + return byte(r & 0x0f) + case Register64: + return byte(r & 0x0f) + case KRegister: + return byte(r & 0x0f) + case MMRegister: + return byte(r & 0x0f) + case XMMRegister: + return byte(r & 0x0f) + case YMMRegister: + return byte(r & 0x0f) + case ZMMRegister: + return byte(r & 0x0f) + case MaskedRegister: + return hlcode(r.Reg) + default: + panic("v is not a register") + } +} + +func ehcode(v interface{}) byte { + switch r := v.(type) { + case Register8: + return byte(r>>3) & 0x03 + case Register16: + return byte(r>>3) & 0x03 + case Register32: + return byte(r>>3) & 0x03 + case Register64: + return byte(r>>3) & 0x03 + case KRegister: + return byte(r>>3) & 0x03 + case MMRegister: + return byte(r>>3) & 0x03 + case XMMRegister: + return byte(r>>3) & 0x03 + case YMMRegister: + return byte(r>>3) & 0x03 + case ZMMRegister: + return byte(r>>3) & 0x03 + case MaskedRegister: + return ehcode(r.Reg) + default: + panic("v is not a register") + } +} + +func toImmAny(v interface{}) int64 { + if x, ok := asInt64(v); ok { + return x + } else { + panic("value is not an integer") + } +} + +func toHcodeOpt(v interface{}) byte { + if v == nil { + return 0 + } else { + return hcode(v) + } +} + +func toEcodeVMM(v interface{}, x byte) byte { + switch r := v.(type) { + case XMMRegister: + return ecode(r) + case YMMRegister: + return ecode(r) + case ZMMRegister: + return ecode(r) + default: + return x + } +} + +func toKcodeMem(v *MemoryOperand) byte { + if !v.Masked { + return 0 + } else { + return byte(v.Mask.K) + } +} + +func toZcodeMem(v *MemoryOperand) byte { + if !v.Masked || v.Mask.Z { + return 0 + } else { + return 1 + } +} + +func toZcodeRegM(v RegisterMask) byte { + if v.Z { + return 1 + } else { + return 0 + } +} + +func toHLcodeReg8(v Register8) byte { + switch v { + case AH: + fallthrough + case BH: + fallthrough + case CH: + fallthrough + case DH: + panic("ah/bh/ch/dh registers never use 4-bit encoding") + default: + return byte(v & 0x0f) + } +} + +/** Instruction Encoding Helpers **/ + +const ( + _N_inst = 16 +) + +const ( + _F_rel1 = 1 << iota + _F_rel4 +) + +type _Encoding struct { + len int + flags int + bytes [_N_inst]byte + encoder func(m *_Encoding, v []interface{}) +} + +// buf ensures len + n <= len(bytes). +func (self *_Encoding) buf(n int) []byte { + if i := self.len; i+n > _N_inst { + panic("instruction too long") + } else { + return self.bytes[i:] + } +} + +// emit encodes a single byte. +func (self *_Encoding) emit(v byte) { + self.buf(1)[0] = v + self.len++ +} + +// imm1 encodes a single byte immediate value. +func (self *_Encoding) imm1(v int64) { + self.emit(byte(v)) +} + +// imm2 encodes a two-byte immediate value in little-endian. +func (self *_Encoding) imm2(v int64) { + binary.LittleEndian.PutUint16(self.buf(2), uint16(v)) + self.len += 2 +} + +// imm4 encodes a 4-byte immediate value in little-endian. +func (self *_Encoding) imm4(v int64) { + binary.LittleEndian.PutUint32(self.buf(4), uint32(v)) + self.len += 4 +} + +// imm8 encodes an 8-byte immediate value in little-endian. +func (self *_Encoding) imm8(v int64) { + binary.LittleEndian.PutUint64(self.buf(8), uint64(v)) + self.len += 8 +} + +// vex2 encodes a 2-byte or 3-byte VEX prefix. +// +// 2-byte VEX prefix: +// +// Requires: VEX.W = 0, VEX.mmmmm = 0b00001 and VEX.B = VEX.X = 0 +// +// +----------------+ +// +// Byte 0: | Bits 0-7: 0xc5 | +// +// +----------------+ +// +// +-----------+----------------+----------+--------------+ +// +// Byte 1: | Bit 7: ~R | Bits 3-6 ~vvvv | Bit 2: L | Bits 0-1: pp | +// +// +-----------+----------------+----------+--------------+ +// +// 3-byte VEX prefix: +// +----------------+ +// +// Byte 0: | Bits 0-7: 0xc4 | +// +// +----------------+ +// +// +-----------+-----------+-----------+-------------------+ +// +// Byte 1: | Bit 7: ~R | Bit 6: ~X | Bit 5: ~B | Bits 0-4: 0b00001 | +// +// +-----------+-----------+-----------+-------------------+ +// +// +----------+-----------------+----------+--------------+ +// +// Byte 2: | Bit 7: 0 | Bits 3-6: ~vvvv | Bit 2: L | Bits 0-1: pp | +// +// +----------+-----------------+----------+--------------+ +func (self *_Encoding) vex2(lpp byte, r byte, rm interface{}, vvvv byte) { + var b byte + var x byte + + /* VEX.R must be a single-bit mask */ + if r > 1 { + panic("VEX.R must be a 1-bit mask") + } + + /* VEX.Lpp must be a 3-bit mask */ + if lpp&^0b111 != 0 { + panic("VEX.Lpp must be a 3-bit mask") + } + + /* VEX.vvvv must be a 4-bit mask */ + if vvvv&^0b1111 != 0 { + panic("VEX.vvvv must be a 4-bit mask") + } + + /* encode the RM bits if any */ + if rm != nil { + switch v := rm.(type) { + case *Label: + break + case Register: + b = hcode(v) + case MemoryAddress: + b, x = toHcodeOpt(v.Base), toHcodeOpt(v.Index) + case RelativeOffset: + break + default: + panic("rm is expected to be a register or a memory address") + } + } + + /* if VEX.B and VEX.X are zeroes, 2-byte VEX prefix can be used */ + if x == 0 && b == 0 { + self.emit(0xc5) + self.emit(0xf8 ^ (r << 7) ^ (vvvv << 3) ^ lpp) + } else { + self.emit(0xc4) + self.emit(0xe1 ^ (r << 7) ^ (x << 6) ^ (b << 5)) + self.emit(0x78 ^ (vvvv << 3) ^ lpp) + } +} + +// vex3 encodes a 3-byte VEX or XOP prefix. +// +// 3-byte VEX/XOP prefix +// +-----------------------------------+ +// +// Byte 0: | Bits 0-7: 0xc4 (VEX) / 0x8f (XOP) | +// +// +-----------------------------------+ +// +// +-----------+-----------+-----------+-----------------+ +// +// Byte 1: | Bit 7: ~R | Bit 6: ~X | Bit 5: ~B | Bits 0-4: mmmmm | +// +// +-----------+-----------+-----------+-----------------+ +// +// +----------+-----------------+----------+--------------+ +// +// Byte 2: | Bit 7: W | Bits 3-6: ~vvvv | Bit 2: L | Bits 0-1: pp | +// +// +----------+-----------------+----------+--------------+ +func (self *_Encoding) vex3(esc byte, mmmmm byte, wlpp byte, r byte, rm interface{}, vvvv byte) { + var b byte + var x byte + + /* VEX.R must be a single-bit mask */ + if r > 1 { + panic("VEX.R must be a 1-bit mask") + } + + /* VEX.vvvv must be a 4-bit mask */ + if vvvv&^0b1111 != 0 { + panic("VEX.vvvv must be a 4-bit mask") + } + + /* escape must be a 3-byte VEX (0xc4) or XOP (0x8f) prefix */ + if esc != 0xc4 && esc != 0x8f { + panic("escape must be a 3-byte VEX (0xc4) or XOP (0x8f) prefix") + } + + /* VEX.W____Lpp is expected to have no bits set except 0, 1, 2 and 7 */ + if wlpp&^0b10000111 != 0 { + panic("VEX.W____Lpp is expected to have no bits set except 0, 1, 2 and 7") + } + + /* VEX.m-mmmm is expected to be a 5-bit mask */ + if mmmmm&^0b11111 != 0 { + panic("VEX.m-mmmm is expected to be a 5-bit mask") + } + + /* encode the RM bits */ + switch v := rm.(type) { + case *Label: + break + case MemoryAddress: + b, x = toHcodeOpt(v.Base), toHcodeOpt(v.Index) + case RelativeOffset: + break + default: + panic("rm is expected to be a register or a memory address") + } + + /* encode the 3-byte VEX or XOP prefix */ + self.emit(esc) + self.emit(0xe0 ^ (r << 7) ^ (x << 6) ^ (b << 5) ^ mmmmm) + self.emit(0x78 ^ (vvvv << 3) ^ wlpp) +} + +// evex encodes a 4-byte EVEX prefix. +func (self *_Encoding) evex(mm byte, w1pp byte, ll byte, rr byte, rm interface{}, vvvvv byte, aaa byte, zz byte, bb byte) { + var b byte + var x byte + + /* EVEX.b must be a single-bit mask */ + if bb > 1 { + panic("EVEX.b must be a 1-bit mask") + } + + /* EVEX.z must be a single-bit mask */ + if zz > 1 { + panic("EVEX.z must be a 1-bit mask") + } + + /* EVEX.mm must be a 2-bit mask */ + if mm&^0b11 != 0 { + panic("EVEX.mm must be a 2-bit mask") + } + + /* EVEX.L'L must be a 2-bit mask */ + if ll&^0b11 != 0 { + panic("EVEX.L'L must be a 2-bit mask") + } + + /* EVEX.R'R must be a 2-bit mask */ + if rr&^0b11 != 0 { + panic("EVEX.R'R must be a 2-bit mask") + } + + /* EVEX.aaa must be a 3-bit mask */ + if aaa&^0b111 != 0 { + panic("EVEX.aaa must be a 3-bit mask") + } + + /* EVEX.v'vvvv must be a 5-bit mask */ + if vvvvv&^0b11111 != 0 { + panic("EVEX.v'vvvv must be a 5-bit mask") + } + + /* EVEX.W____1pp is expected to have no bits set except 0, 1, 2, and 7 */ + if w1pp&^0b10000011 != 0b100 { + panic("EVEX.W____1pp is expected to have no bits set except 0, 1, 2, and 7") + } + + /* extract bits from EVEX.R'R and EVEX.v'vvvv */ + r1, r0 := rr>>1, rr&1 + v1, v0 := vvvvv>>4, vvvvv&0b1111 + + /* encode the RM bits if any */ + if rm != nil { + switch m := rm.(type) { + case *Label: + break + case Register: + b, x = hcode(m), ecode(m) + case MemoryAddress: + b, x, v1 = toHcodeOpt(m.Base), toHcodeOpt(m.Index), toEcodeVMM(m.Index, v1) + case RelativeOffset: + break + default: + panic("rm is expected to be a register or a memory address") + } + } + + /* EVEX prefix bytes */ + p0 := (r0 << 7) | (x << 6) | (b << 5) | (r1 << 4) | mm + p1 := (v0 << 3) | w1pp + p2 := (zz << 7) | (ll << 5) | (b << 4) | (v1 << 3) | aaa + + /* p0: invert RXBR' (bits 4-7) + * p1: invert vvvv (bits 3-6) + * p2: invert V' (bit 3) */ + self.emit(0x62) + self.emit(p0 ^ 0xf0) + self.emit(p1 ^ 0x78) + self.emit(p2 ^ 0x08) +} + +// rexm encodes a mandatory REX prefix. +func (self *_Encoding) rexm(w byte, r byte, rm interface{}) { + var b byte + var x byte + + /* REX.R must be 0 or 1 */ + if r != 0 && r != 1 { + panic("REX.R must be 0 or 1") + } + + /* REX.W must be 0 or 1 */ + if w != 0 && w != 1 { + panic("REX.W must be 0 or 1") + } + + /* encode the RM bits */ + switch v := rm.(type) { + case *Label: + break + case MemoryAddress: + b, x = toHcodeOpt(v.Base), toHcodeOpt(v.Index) + case RelativeOffset: + break + default: + panic("rm is expected to be a register or a memory address") + } + + /* encode the REX prefix */ + self.emit(0x40 | (w << 3) | (r << 2) | (x << 1) | b) +} + +// rexo encodes an optional REX prefix. +func (self *_Encoding) rexo(r byte, rm interface{}, force bool) { + var b byte + var x byte + + /* REX.R must be 0 or 1 */ + if r != 0 && r != 1 { + panic("REX.R must be 0 or 1") + } + + /* encode the RM bits */ + switch v := rm.(type) { + case *Label: + break + case Register: + b = hcode(v) + case MemoryAddress: + b, x = toHcodeOpt(v.Base), toHcodeOpt(v.Index) + case RelativeOffset: + break + default: + panic("rm is expected to be a register or a memory address") + } + + /* if REX.R, REX.X, and REX.B are all zeroes, REX prefix can be omitted */ + if force || r != 0 || x != 0 || b != 0 { + self.emit(0x40 | (r << 2) | (x << 1) | b) + } +} + +// mrsd encodes ModR/M, SIB and Displacement. +// +// ModR/M byte +// +// +----------------+---------------+---------------+ +// | Bits 6-7: Mode | Bits 3-5: Reg | Bits 0-2: R/M | +// +----------------+---------------+---------------+ +// +// SIB byte +// +// +-----------------+-----------------+----------------+ +// | Bits 6-7: Scale | Bits 3-5: Index | Bits 0-2: Base | +// +-----------------+-----------------+----------------+ +func (self *_Encoding) mrsd(reg byte, rm interface{}, disp8v int32) { + var ok bool + var mm MemoryAddress + var ro RelativeOffset + + /* ModRM encodes the lower 3-bit of the register */ + if reg > 7 { + panic("invalid register bits") + } + + /* check the displacement scale */ + switch disp8v { + case 1: + break + case 2: + break + case 4: + break + case 8: + break + case 16: + break + case 32: + break + case 64: + break + default: + panic("invalid displacement size") + } + + /* special case: unresolved labels, assuming a zero offset */ + if _, ok = rm.(*Label); ok { + self.emit(0x05 | (reg << 3)) + self.imm4(0) + return + } + + /* special case: RIP-relative offset + * ModRM.Mode == 0 and ModeRM.R/M == 5 indicates (rip + disp32) addressing */ + if ro, ok = rm.(RelativeOffset); ok { + self.emit(0x05 | (reg << 3)) + self.imm4(int64(ro)) + return + } + + /* must be a generic memory address */ + if mm, ok = rm.(MemoryAddress); !ok { + panic("rm must be a memory address") + } + + /* absolute addressing, encoded as disp(%rbp,%rsp,1) */ + if mm.Base == nil && mm.Index == nil { + self.emit(0x04 | (reg << 3)) + self.emit(0x25) + self.imm4(int64(mm.Displacement)) + return + } + + /* no SIB byte */ + if mm.Index == nil && lcode(mm.Base) != 0b100 { + cc := lcode(mm.Base) + dv := mm.Displacement + + /* ModRM.Mode == 0 (no displacement) */ + if dv == 0 && mm.Base != RBP && mm.Base != R13 { + if cc == 0b101 { + panic("rbp/r13 is not encodable as a base register (interpreted as disp32 address)") + } else { + self.emit((reg << 3) | cc) + return + } + } + + /* ModRM.Mode == 1 (8-bit displacement) */ + if dq := dv / disp8v; dq >= math.MinInt8 && dq <= math.MaxInt8 && dv%disp8v == 0 { + self.emit(0x40 | (reg << 3) | cc) + self.imm1(int64(dq)) + return + } + + /* ModRM.Mode == 2 (32-bit displacement) */ + self.emit(0x80 | (reg << 3) | cc) + self.imm4(int64(mm.Displacement)) + return + } + + /* all encodings below use ModRM.R/M = 4 (0b100) to indicate the presence of SIB */ + if mm.Index == RSP { + panic("rsp is not encodable as an index register (interpreted as no index)") + } + + /* index = 4 (0b100) denotes no-index encoding */ + var scale byte + var index byte = 0x04 + + /* encode the scale byte */ + if mm.Scale != 0 { + switch mm.Scale { + case 1: + scale = 0 + case 2: + scale = 1 + case 4: + scale = 2 + case 8: + scale = 3 + default: + panic("invalid scale value") + } + } + + /* encode the index byte */ + if mm.Index != nil { + index = lcode(mm.Index) + } + + /* SIB.Base = 5 (0b101) and ModRM.Mode = 0 indicates no-base encoding with disp32 */ + if mm.Base == nil { + self.emit((reg << 3) | 0b100) + self.emit((scale << 6) | (index << 3) | 0b101) + self.imm4(int64(mm.Displacement)) + return + } + + /* base L-code & displacement value */ + cc := lcode(mm.Base) + dv := mm.Displacement + + /* ModRM.Mode == 0 (no displacement) */ + if dv == 0 && cc != 0b101 { + self.emit((reg << 3) | 0b100) + self.emit((scale << 6) | (index << 3) | cc) + return + } + + /* ModRM.Mode == 1 (8-bit displacement) */ + if dq := dv / disp8v; dq >= math.MinInt8 && dq <= math.MaxInt8 && dv%disp8v == 0 { + self.emit(0x44 | (reg << 3)) + self.emit((scale << 6) | (index << 3) | cc) + self.imm1(int64(dq)) + return + } + + /* ModRM.Mode == 2 (32-bit displacement) */ + self.emit(0x84 | (reg << 3)) + self.emit((scale << 6) | (index << 3) | cc) + self.imm4(int64(mm.Displacement)) +} + +// encode invokes the encoder to encode this instruction. +func (self *_Encoding) encode(v []interface{}) int { + self.len = 0 + self.encoder(self, v) + return self.len +} diff --git a/loader/internal/iasm/x86_64/instructions.go b/loader/internal/iasm/x86_64/instructions.go new file mode 100644 index 000000000..836e1807f --- /dev/null +++ b/loader/internal/iasm/x86_64/instructions.go @@ -0,0 +1,1077 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Code generated by "mkasm_amd64.py", DO NOT EDIT. + +package x86_64 + +// ADDQ performs "Add". +// +// Mnemonic : ADD +// Supported forms : (8 forms) +// +// - ADDQ imm32, rax +// - ADDQ imm8, r64 +// - ADDQ imm32, r64 +// - ADDQ r64, r64 +// - ADDQ m64, r64 +// - ADDQ imm8, m64 +// - ADDQ imm32, m64 +// - ADDQ r64, m64 +func (self *Program) ADDQ(v0 interface{}, v1 interface{}) *Instruction { + p := self.alloc("ADDQ", 2, Operands{v0, v1}) + // ADDQ imm32, rax + if isImm32(v0) && v1 == RAX { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48) + m.emit(0x05) + m.imm4(toImmAny(v[0])) + }) + } + // ADDQ imm8, r64 + if isImm8Ext(v0, 8) && isReg64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48 | hcode(v[1])) + m.emit(0x83) + m.emit(0xc0 | lcode(v[1])) + m.imm1(toImmAny(v[0])) + }) + } + // ADDQ imm32, r64 + if isImm32Ext(v0, 8) && isReg64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48 | hcode(v[1])) + m.emit(0x81) + m.emit(0xc0 | lcode(v[1])) + m.imm4(toImmAny(v[0])) + }) + } + // ADDQ r64, r64 + if isReg64(v0) && isReg64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48 | hcode(v[0])<<2 | hcode(v[1])) + m.emit(0x01) + m.emit(0xc0 | lcode(v[0])<<3 | lcode(v[1])) + }) + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48 | hcode(v[1])<<2 | hcode(v[0])) + m.emit(0x03) + m.emit(0xc0 | lcode(v[1])<<3 | lcode(v[0])) + }) + } + // ADDQ m64, r64 + if isM64(v0) && isReg64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexm(1, hcode(v[1]), addr(v[0])) + m.emit(0x03) + m.mrsd(lcode(v[1]), addr(v[0]), 1) + }) + } + // ADDQ imm8, m64 + if isImm8Ext(v0, 8) && isM64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexm(1, 0, addr(v[1])) + m.emit(0x83) + m.mrsd(0, addr(v[1]), 1) + m.imm1(toImmAny(v[0])) + }) + } + // ADDQ imm32, m64 + if isImm32Ext(v0, 8) && isM64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexm(1, 0, addr(v[1])) + m.emit(0x81) + m.mrsd(0, addr(v[1]), 1) + m.imm4(toImmAny(v[0])) + }) + } + // ADDQ r64, m64 + if isReg64(v0) && isM64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexm(1, hcode(v[0]), addr(v[1])) + m.emit(0x01) + m.mrsd(lcode(v[0]), addr(v[1]), 1) + }) + } + if p.len == 0 { + panic("invalid operands for ADDQ") + } + return p +} + +// CALLQ performs "Call Procedure". +// +// Mnemonic : CALL +// Supported forms : (2 forms) +// +// - CALLQ r64 +// - CALLQ m64 +func (self *Program) CALLQ(v0 interface{}) *Instruction { + p := self.alloc("CALLQ", 1, Operands{v0}) + // CALLQ r64 + if isReg64(v0) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexo(0, v[0], false) + m.emit(0xff) + m.emit(0xd0 | lcode(v[0])) + }) + } + // CALLQ m64 + if isM64(v0) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexo(0, addr(v[0]), false) + m.emit(0xff) + m.mrsd(2, addr(v[0]), 1) + }) + } + if p.len == 0 { + panic("invalid operands for CALLQ") + } + return p +} + +// CMPQ performs "Compare Two Operands". +// +// Mnemonic : CMP +// Supported forms : (8 forms) +// +// - CMPQ imm32, rax +// - CMPQ imm8, r64 +// - CMPQ imm32, r64 +// - CMPQ r64, r64 +// - CMPQ m64, r64 +// - CMPQ imm8, m64 +// - CMPQ imm32, m64 +// - CMPQ r64, m64 +func (self *Program) CMPQ(v0 interface{}, v1 interface{}) *Instruction { + p := self.alloc("CMPQ", 2, Operands{v0, v1}) + // CMPQ imm32, rax + if isImm32(v0) && v1 == RAX { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48) + m.emit(0x3d) + m.imm4(toImmAny(v[0])) + }) + } + // CMPQ imm8, r64 + if isImm8Ext(v0, 8) && isReg64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48 | hcode(v[1])) + m.emit(0x83) + m.emit(0xf8 | lcode(v[1])) + m.imm1(toImmAny(v[0])) + }) + } + // CMPQ imm32, r64 + if isImm32Ext(v0, 8) && isReg64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48 | hcode(v[1])) + m.emit(0x81) + m.emit(0xf8 | lcode(v[1])) + m.imm4(toImmAny(v[0])) + }) + } + // CMPQ r64, r64 + if isReg64(v0) && isReg64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48 | hcode(v[0])<<2 | hcode(v[1])) + m.emit(0x39) + m.emit(0xc0 | lcode(v[0])<<3 | lcode(v[1])) + }) + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48 | hcode(v[1])<<2 | hcode(v[0])) + m.emit(0x3b) + m.emit(0xc0 | lcode(v[1])<<3 | lcode(v[0])) + }) + } + // CMPQ m64, r64 + if isM64(v0) && isReg64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexm(1, hcode(v[1]), addr(v[0])) + m.emit(0x3b) + m.mrsd(lcode(v[1]), addr(v[0]), 1) + }) + } + // CMPQ imm8, m64 + if isImm8Ext(v0, 8) && isM64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexm(1, 0, addr(v[1])) + m.emit(0x83) + m.mrsd(7, addr(v[1]), 1) + m.imm1(toImmAny(v[0])) + }) + } + // CMPQ imm32, m64 + if isImm32Ext(v0, 8) && isM64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexm(1, 0, addr(v[1])) + m.emit(0x81) + m.mrsd(7, addr(v[1]), 1) + m.imm4(toImmAny(v[0])) + }) + } + // CMPQ r64, m64 + if isReg64(v0) && isM64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexm(1, hcode(v[0]), addr(v[1])) + m.emit(0x39) + m.mrsd(lcode(v[0]), addr(v[1]), 1) + }) + } + if p.len == 0 { + panic("invalid operands for CMPQ") + } + return p +} + +// JBE performs "Jump if below or equal (CF == 1 or ZF == 1)". +// +// Mnemonic : JBE +// Supported forms : (2 forms) +// +// - JBE rel8 +// - JBE rel32 +func (self *Program) JBE(v0 interface{}) *Instruction { + p := self.alloc("JBE", 1, Operands{v0}) + p.branch = _B_conditional + // JBE rel8 + if isRel8(v0) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x76) + m.imm1(relv(v[0])) + }) + } + // JBE rel32 + if isRel32(v0) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x0f) + m.emit(0x86) + m.imm4(relv(v[0])) + }) + } + // JBE label + if isLabel(v0) { + p.add(_F_rel1, func(m *_Encoding, v []interface{}) { + m.emit(0x76) + m.imm1(relv(v[0])) + }) + p.add(_F_rel4, func(m *_Encoding, v []interface{}) { + m.emit(0x0f) + m.emit(0x86) + m.imm4(relv(v[0])) + }) + } + if p.len == 0 { + panic("invalid operands for JBE") + } + return p +} + +// JMP performs "Jump Unconditionally". +// +// Mnemonic : JMP +// Supported forms : (2 forms) +// +// - JMP rel8 +// - JMP rel32 +func (self *Program) JMP(v0 interface{}) *Instruction { + p := self.alloc("JMP", 1, Operands{v0}) + p.branch = _B_unconditional + // JMP rel8 + if isRel8(v0) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0xeb) + m.imm1(relv(v[0])) + }) + } + // JMP rel32 + if isRel32(v0) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0xe9) + m.imm4(relv(v[0])) + }) + } + // JMP label + if isLabel(v0) { + p.add(_F_rel1, func(m *_Encoding, v []interface{}) { + m.emit(0xeb) + m.imm1(relv(v[0])) + }) + p.add(_F_rel4, func(m *_Encoding, v []interface{}) { + m.emit(0xe9) + m.imm4(relv(v[0])) + }) + } + if p.len == 0 { + panic("invalid operands for JMP") + } + return p +} + +// JMPQ performs "Jump Unconditionally". +// +// Mnemonic : JMP +// Supported forms : (2 forms) +// +// - JMPQ r64 +// - JMPQ m64 +func (self *Program) JMPQ(v0 interface{}) *Instruction { + p := self.alloc("JMPQ", 1, Operands{v0}) + // JMPQ r64 + if isReg64(v0) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexo(0, v[0], false) + m.emit(0xff) + m.emit(0xe0 | lcode(v[0])) + }) + } + // JMPQ m64 + if isM64(v0) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexo(0, addr(v[0]), false) + m.emit(0xff) + m.mrsd(4, addr(v[0]), 1) + }) + } + if p.len == 0 { + panic("invalid operands for JMPQ") + } + return p +} + +// LEAQ performs "Load Effective Address". +// +// Mnemonic : LEA +// Supported forms : (1 form) +// +// - LEAQ m, r64 +func (self *Program) LEAQ(v0 interface{}, v1 interface{}) *Instruction { + p := self.alloc("LEAQ", 2, Operands{v0, v1}) + // LEAQ m, r64 + if isM(v0) && isReg64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexm(1, hcode(v[1]), addr(v[0])) + m.emit(0x8d) + m.mrsd(lcode(v[1]), addr(v[0]), 1) + }) + } + if p.len == 0 { + panic("invalid operands for LEAQ") + } + return p +} + +// MOVQ performs "Move". +// +// Mnemonic : MOV +// Supported forms : (16 forms) +// +// - MOVQ imm32, r64 +// - MOVQ imm64, r64 +// - MOVQ r64, r64 +// - MOVQ m64, r64 +// - MOVQ imm32, m64 +// - MOVQ r64, m64 +// - MOVQ mm, r64 [MMX] +// - MOVQ r64, mm [MMX] +// - MOVQ mm, mm [MMX] +// - MOVQ m64, mm [MMX] +// - MOVQ mm, m64 [MMX] +// - MOVQ xmm, r64 [SSE2] +// - MOVQ r64, xmm [SSE2] +// - MOVQ xmm, xmm [SSE2] +// - MOVQ m64, xmm [SSE2] +// - MOVQ xmm, m64 [SSE2] +func (self *Program) MOVQ(v0 interface{}, v1 interface{}) *Instruction { + p := self.alloc("MOVQ", 2, Operands{v0, v1}) + // MOVQ imm32, r64 + if isImm32Ext(v0, 8) && isReg64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48 | hcode(v[1])) + m.emit(0xc7) + m.emit(0xc0 | lcode(v[1])) + m.imm4(toImmAny(v[0])) + }) + } + // MOVQ imm64, r64 + if isImm64(v0) && isReg64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48 | hcode(v[1])) + m.emit(0xb8 | lcode(v[1])) + m.imm8(toImmAny(v[0])) + }) + } + // MOVQ r64, r64 + if isReg64(v0) && isReg64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48 | hcode(v[0])<<2 | hcode(v[1])) + m.emit(0x89) + m.emit(0xc0 | lcode(v[0])<<3 | lcode(v[1])) + }) + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48 | hcode(v[1])<<2 | hcode(v[0])) + m.emit(0x8b) + m.emit(0xc0 | lcode(v[1])<<3 | lcode(v[0])) + }) + } + // MOVQ m64, r64 + if isM64(v0) && isReg64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexm(1, hcode(v[1]), addr(v[0])) + m.emit(0x8b) + m.mrsd(lcode(v[1]), addr(v[0]), 1) + }) + } + // MOVQ imm32, m64 + if isImm32Ext(v0, 8) && isM64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexm(1, 0, addr(v[1])) + m.emit(0xc7) + m.mrsd(0, addr(v[1]), 1) + m.imm4(toImmAny(v[0])) + }) + } + // MOVQ r64, m64 + if isReg64(v0) && isM64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexm(1, hcode(v[0]), addr(v[1])) + m.emit(0x89) + m.mrsd(lcode(v[0]), addr(v[1]), 1) + }) + } + // MOVQ mm, r64 + if isMM(v0) && isReg64(v1) { + self.require(ISA_MMX) + p.domain = DomainMMXSSE + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48 | hcode(v[0])<<2 | hcode(v[1])) + m.emit(0x0f) + m.emit(0x7e) + m.emit(0xc0 | lcode(v[0])<<3 | lcode(v[1])) + }) + } + // MOVQ r64, mm + if isReg64(v0) && isMM(v1) { + self.require(ISA_MMX) + p.domain = DomainMMXSSE + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48 | hcode(v[1])<<2 | hcode(v[0])) + m.emit(0x0f) + m.emit(0x6e) + m.emit(0xc0 | lcode(v[1])<<3 | lcode(v[0])) + }) + } + // MOVQ mm, mm + if isMM(v0) && isMM(v1) { + self.require(ISA_MMX) + p.domain = DomainMMXSSE + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexo(hcode(v[1]), v[0], false) + m.emit(0x0f) + m.emit(0x6f) + m.emit(0xc0 | lcode(v[1])<<3 | lcode(v[0])) + }) + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexo(hcode(v[0]), v[1], false) + m.emit(0x0f) + m.emit(0x7f) + m.emit(0xc0 | lcode(v[0])<<3 | lcode(v[1])) + }) + } + // MOVQ m64, mm + if isM64(v0) && isMM(v1) { + self.require(ISA_MMX) + p.domain = DomainMMXSSE + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexo(hcode(v[1]), addr(v[0]), false) + m.emit(0x0f) + m.emit(0x6f) + m.mrsd(lcode(v[1]), addr(v[0]), 1) + }) + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexm(1, hcode(v[1]), addr(v[0])) + m.emit(0x0f) + m.emit(0x6e) + m.mrsd(lcode(v[1]), addr(v[0]), 1) + }) + } + // MOVQ mm, m64 + if isMM(v0) && isM64(v1) { + self.require(ISA_MMX) + p.domain = DomainMMXSSE + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexo(hcode(v[0]), addr(v[1]), false) + m.emit(0x0f) + m.emit(0x7f) + m.mrsd(lcode(v[0]), addr(v[1]), 1) + }) + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexm(1, hcode(v[0]), addr(v[1])) + m.emit(0x0f) + m.emit(0x7e) + m.mrsd(lcode(v[0]), addr(v[1]), 1) + }) + } + // MOVQ xmm, r64 + if isXMM(v0) && isReg64(v1) { + self.require(ISA_SSE2) + p.domain = DomainMMXSSE + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x66) + m.emit(0x48 | hcode(v[0])<<2 | hcode(v[1])) + m.emit(0x0f) + m.emit(0x7e) + m.emit(0xc0 | lcode(v[0])<<3 | lcode(v[1])) + }) + } + // MOVQ r64, xmm + if isReg64(v0) && isXMM(v1) { + self.require(ISA_SSE2) + p.domain = DomainMMXSSE + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x66) + m.emit(0x48 | hcode(v[1])<<2 | hcode(v[0])) + m.emit(0x0f) + m.emit(0x6e) + m.emit(0xc0 | lcode(v[1])<<3 | lcode(v[0])) + }) + } + // MOVQ xmm, xmm + if isXMM(v0) && isXMM(v1) { + self.require(ISA_SSE2) + p.domain = DomainMMXSSE + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0xf3) + m.rexo(hcode(v[1]), v[0], false) + m.emit(0x0f) + m.emit(0x7e) + m.emit(0xc0 | lcode(v[1])<<3 | lcode(v[0])) + }) + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x66) + m.rexo(hcode(v[0]), v[1], false) + m.emit(0x0f) + m.emit(0xd6) + m.emit(0xc0 | lcode(v[0])<<3 | lcode(v[1])) + }) + } + // MOVQ m64, xmm + if isM64(v0) && isXMM(v1) { + self.require(ISA_SSE2) + p.domain = DomainMMXSSE + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0xf3) + m.rexo(hcode(v[1]), addr(v[0]), false) + m.emit(0x0f) + m.emit(0x7e) + m.mrsd(lcode(v[1]), addr(v[0]), 1) + }) + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x66) + m.rexm(1, hcode(v[1]), addr(v[0])) + m.emit(0x0f) + m.emit(0x6e) + m.mrsd(lcode(v[1]), addr(v[0]), 1) + }) + } + // MOVQ xmm, m64 + if isXMM(v0) && isM64(v1) { + self.require(ISA_SSE2) + p.domain = DomainMMXSSE + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x66) + m.rexo(hcode(v[0]), addr(v[1]), false) + m.emit(0x0f) + m.emit(0xd6) + m.mrsd(lcode(v[0]), addr(v[1]), 1) + }) + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x66) + m.rexm(1, hcode(v[0]), addr(v[1])) + m.emit(0x0f) + m.emit(0x7e) + m.mrsd(lcode(v[0]), addr(v[1]), 1) + }) + } + if p.len == 0 { + panic("invalid operands for MOVQ") + } + return p +} + +// MOVSD performs "Move Scalar Double-Precision Floating-Point Value". +// +// Mnemonic : MOVSD +// Supported forms : (3 forms) +// +// - MOVSD xmm, xmm [SSE2] +// - MOVSD m64, xmm [SSE2] +// - MOVSD xmm, m64 [SSE2] +func (self *Program) MOVSD(v0 interface{}, v1 interface{}) *Instruction { + p := self.alloc("MOVSD", 2, Operands{v0, v1}) + // MOVSD xmm, xmm + if isXMM(v0) && isXMM(v1) { + self.require(ISA_SSE2) + p.domain = DomainMMXSSE + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0xf2) + m.rexo(hcode(v[1]), v[0], false) + m.emit(0x0f) + m.emit(0x10) + m.emit(0xc0 | lcode(v[1])<<3 | lcode(v[0])) + }) + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0xf2) + m.rexo(hcode(v[0]), v[1], false) + m.emit(0x0f) + m.emit(0x11) + m.emit(0xc0 | lcode(v[0])<<3 | lcode(v[1])) + }) + } + // MOVSD m64, xmm + if isM64(v0) && isXMM(v1) { + self.require(ISA_SSE2) + p.domain = DomainMMXSSE + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0xf2) + m.rexo(hcode(v[1]), addr(v[0]), false) + m.emit(0x0f) + m.emit(0x10) + m.mrsd(lcode(v[1]), addr(v[0]), 1) + }) + } + // MOVSD xmm, m64 + if isXMM(v0) && isM64(v1) { + self.require(ISA_SSE2) + p.domain = DomainMMXSSE + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0xf2) + m.rexo(hcode(v[0]), addr(v[1]), false) + m.emit(0x0f) + m.emit(0x11) + m.mrsd(lcode(v[0]), addr(v[1]), 1) + }) + } + if p.len == 0 { + panic("invalid operands for MOVSD") + } + return p +} + +// MOVSLQ performs "Move Doubleword to Quadword with Sign-Extension". +// +// Mnemonic : MOVSXD +// Supported forms : (2 forms) +// +// - MOVSLQ r32, r64 +// - MOVSLQ m32, r64 +func (self *Program) MOVSLQ(v0 interface{}, v1 interface{}) *Instruction { + p := self.alloc("MOVSLQ", 2, Operands{v0, v1}) + // MOVSLQ r32, r64 + if isReg32(v0) && isReg64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48 | hcode(v[1])<<2 | hcode(v[0])) + m.emit(0x63) + m.emit(0xc0 | lcode(v[1])<<3 | lcode(v[0])) + }) + } + // MOVSLQ m32, r64 + if isM32(v0) && isReg64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexm(1, hcode(v[1]), addr(v[0])) + m.emit(0x63) + m.mrsd(lcode(v[1]), addr(v[0]), 1) + }) + } + if p.len == 0 { + panic("invalid operands for MOVSLQ") + } + return p +} + +// MOVSS performs "Move Scalar Single-Precision Floating-Point Values". +// +// Mnemonic : MOVSS +// Supported forms : (3 forms) +// +// - MOVSS xmm, xmm [SSE] +// - MOVSS m32, xmm [SSE] +// - MOVSS xmm, m32 [SSE] +func (self *Program) MOVSS(v0 interface{}, v1 interface{}) *Instruction { + p := self.alloc("MOVSS", 2, Operands{v0, v1}) + // MOVSS xmm, xmm + if isXMM(v0) && isXMM(v1) { + self.require(ISA_SSE) + p.domain = DomainMMXSSE + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0xf3) + m.rexo(hcode(v[1]), v[0], false) + m.emit(0x0f) + m.emit(0x10) + m.emit(0xc0 | lcode(v[1])<<3 | lcode(v[0])) + }) + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0xf3) + m.rexo(hcode(v[0]), v[1], false) + m.emit(0x0f) + m.emit(0x11) + m.emit(0xc0 | lcode(v[0])<<3 | lcode(v[1])) + }) + } + // MOVSS m32, xmm + if isM32(v0) && isXMM(v1) { + self.require(ISA_SSE) + p.domain = DomainMMXSSE + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0xf3) + m.rexo(hcode(v[1]), addr(v[0]), false) + m.emit(0x0f) + m.emit(0x10) + m.mrsd(lcode(v[1]), addr(v[0]), 1) + }) + } + // MOVSS xmm, m32 + if isXMM(v0) && isM32(v1) { + self.require(ISA_SSE) + p.domain = DomainMMXSSE + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0xf3) + m.rexo(hcode(v[0]), addr(v[1]), false) + m.emit(0x0f) + m.emit(0x11) + m.mrsd(lcode(v[0]), addr(v[1]), 1) + }) + } + if p.len == 0 { + panic("invalid operands for MOVSS") + } + return p +} + +// RET performs "Return from Procedure". +// +// Mnemonic : RET +// Supported forms : (2 forms) +// +// - RET +// - RET imm16 +func (self *Program) RET(vv ...interface{}) *Instruction { + var p *Instruction + switch len(vv) { + case 0: + p = self.alloc("RET", 0, Operands{}) + case 1: + p = self.alloc("RET", 1, Operands{vv[0]}) + default: + panic("instruction RET takes 0 or 1 operands") + } + // RET + if len(vv) == 0 { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0xc3) + }) + } + // RET imm16 + if len(vv) == 1 && isImm16(vv[0]) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0xc2) + m.imm2(toImmAny(v[0])) + }) + } + if p.len == 0 { + panic("invalid operands for RET") + } + return p +} + +// SUBQ performs "Subtract". +// +// Mnemonic : SUB +// Supported forms : (8 forms) +// +// - SUBQ imm32, rax +// - SUBQ imm8, r64 +// - SUBQ imm32, r64 +// - SUBQ r64, r64 +// - SUBQ m64, r64 +// - SUBQ imm8, m64 +// - SUBQ imm32, m64 +// - SUBQ r64, m64 +func (self *Program) SUBQ(v0 interface{}, v1 interface{}) *Instruction { + p := self.alloc("SUBQ", 2, Operands{v0, v1}) + // SUBQ imm32, rax + if isImm32(v0) && v1 == RAX { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48) + m.emit(0x2d) + m.imm4(toImmAny(v[0])) + }) + } + // SUBQ imm8, r64 + if isImm8Ext(v0, 8) && isReg64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48 | hcode(v[1])) + m.emit(0x83) + m.emit(0xe8 | lcode(v[1])) + m.imm1(toImmAny(v[0])) + }) + } + // SUBQ imm32, r64 + if isImm32Ext(v0, 8) && isReg64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48 | hcode(v[1])) + m.emit(0x81) + m.emit(0xe8 | lcode(v[1])) + m.imm4(toImmAny(v[0])) + }) + } + // SUBQ r64, r64 + if isReg64(v0) && isReg64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48 | hcode(v[0])<<2 | hcode(v[1])) + m.emit(0x29) + m.emit(0xc0 | lcode(v[0])<<3 | lcode(v[1])) + }) + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48 | hcode(v[1])<<2 | hcode(v[0])) + m.emit(0x2b) + m.emit(0xc0 | lcode(v[1])<<3 | lcode(v[0])) + }) + } + // SUBQ m64, r64 + if isM64(v0) && isReg64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexm(1, hcode(v[1]), addr(v[0])) + m.emit(0x2b) + m.mrsd(lcode(v[1]), addr(v[0]), 1) + }) + } + // SUBQ imm8, m64 + if isImm8Ext(v0, 8) && isM64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexm(1, 0, addr(v[1])) + m.emit(0x83) + m.mrsd(5, addr(v[1]), 1) + m.imm1(toImmAny(v[0])) + }) + } + // SUBQ imm32, m64 + if isImm32Ext(v0, 8) && isM64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexm(1, 0, addr(v[1])) + m.emit(0x81) + m.mrsd(5, addr(v[1]), 1) + m.imm4(toImmAny(v[0])) + }) + } + // SUBQ r64, m64 + if isReg64(v0) && isM64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexm(1, hcode(v[0]), addr(v[1])) + m.emit(0x29) + m.mrsd(lcode(v[0]), addr(v[1]), 1) + }) + } + if p.len == 0 { + panic("invalid operands for SUBQ") + } + return p +} + +// VPERMIL2PD performs "Permute Two-Source Double-Precision Floating-Point Vectors". +// +// Mnemonic : VPERMIL2PD +// Supported forms : (6 forms) +// +// - VPERMIL2PD imm4, xmm, xmm, xmm, xmm [XOP] +// - VPERMIL2PD imm4, m128, xmm, xmm, xmm [XOP] +// - VPERMIL2PD imm4, xmm, m128, xmm, xmm [XOP] +// - VPERMIL2PD imm4, ymm, ymm, ymm, ymm [XOP] +// - VPERMIL2PD imm4, m256, ymm, ymm, ymm [XOP] +// - VPERMIL2PD imm4, ymm, m256, ymm, ymm [XOP] +func (self *Program) VPERMIL2PD(v0 interface{}, v1 interface{}, v2 interface{}, v3 interface{}, v4 interface{}) *Instruction { + p := self.alloc("VPERMIL2PD", 5, Operands{v0, v1, v2, v3, v4}) + // VPERMIL2PD imm4, xmm, xmm, xmm, xmm + if isImm4(v0) && isXMM(v1) && isXMM(v2) && isXMM(v3) && isXMM(v4) { + self.require(ISA_XOP) + p.domain = DomainAMDSpecific + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0xc4) + m.emit(0xe3 ^ (hcode(v[4]) << 7) ^ (hcode(v[2]) << 5)) + m.emit(0x79 ^ (hlcode(v[3]) << 3)) + m.emit(0x49) + m.emit(0xc0 | lcode(v[4])<<3 | lcode(v[2])) + m.emit((hlcode(v[1]) << 4) | imml(v[0])) + }) + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0xc4) + m.emit(0xe3 ^ (hcode(v[4]) << 7) ^ (hcode(v[1]) << 5)) + m.emit(0xf9 ^ (hlcode(v[3]) << 3)) + m.emit(0x49) + m.emit(0xc0 | lcode(v[4])<<3 | lcode(v[1])) + m.emit((hlcode(v[2]) << 4) | imml(v[0])) + }) + } + // VPERMIL2PD imm4, m128, xmm, xmm, xmm + if isImm4(v0) && isM128(v1) && isXMM(v2) && isXMM(v3) && isXMM(v4) { + self.require(ISA_XOP) + p.domain = DomainAMDSpecific + p.add(0, func(m *_Encoding, v []interface{}) { + m.vex3(0xc4, 0b11, 0x81, hcode(v[4]), addr(v[1]), hlcode(v[3])) + m.emit(0x49) + m.mrsd(lcode(v[4]), addr(v[1]), 1) + m.emit((hlcode(v[2]) << 4) | imml(v[0])) + }) + } + // VPERMIL2PD imm4, xmm, m128, xmm, xmm + if isImm4(v0) && isXMM(v1) && isM128(v2) && isXMM(v3) && isXMM(v4) { + self.require(ISA_XOP) + p.domain = DomainAMDSpecific + p.add(0, func(m *_Encoding, v []interface{}) { + m.vex3(0xc4, 0b11, 0x01, hcode(v[4]), addr(v[2]), hlcode(v[3])) + m.emit(0x49) + m.mrsd(lcode(v[4]), addr(v[2]), 1) + m.emit((hlcode(v[1]) << 4) | imml(v[0])) + }) + } + // VPERMIL2PD imm4, ymm, ymm, ymm, ymm + if isImm4(v0) && isYMM(v1) && isYMM(v2) && isYMM(v3) && isYMM(v4) { + self.require(ISA_XOP) + p.domain = DomainAMDSpecific + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0xc4) + m.emit(0xe3 ^ (hcode(v[4]) << 7) ^ (hcode(v[2]) << 5)) + m.emit(0x7d ^ (hlcode(v[3]) << 3)) + m.emit(0x49) + m.emit(0xc0 | lcode(v[4])<<3 | lcode(v[2])) + m.emit((hlcode(v[1]) << 4) | imml(v[0])) + }) + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0xc4) + m.emit(0xe3 ^ (hcode(v[4]) << 7) ^ (hcode(v[1]) << 5)) + m.emit(0xfd ^ (hlcode(v[3]) << 3)) + m.emit(0x49) + m.emit(0xc0 | lcode(v[4])<<3 | lcode(v[1])) + m.emit((hlcode(v[2]) << 4) | imml(v[0])) + }) + } + // VPERMIL2PD imm4, m256, ymm, ymm, ymm + if isImm4(v0) && isM256(v1) && isYMM(v2) && isYMM(v3) && isYMM(v4) { + self.require(ISA_XOP) + p.domain = DomainAMDSpecific + p.add(0, func(m *_Encoding, v []interface{}) { + m.vex3(0xc4, 0b11, 0x85, hcode(v[4]), addr(v[1]), hlcode(v[3])) + m.emit(0x49) + m.mrsd(lcode(v[4]), addr(v[1]), 1) + m.emit((hlcode(v[2]) << 4) | imml(v[0])) + }) + } + // VPERMIL2PD imm4, ymm, m256, ymm, ymm + if isImm4(v0) && isYMM(v1) && isM256(v2) && isYMM(v3) && isYMM(v4) { + self.require(ISA_XOP) + p.domain = DomainAMDSpecific + p.add(0, func(m *_Encoding, v []interface{}) { + m.vex3(0xc4, 0b11, 0x05, hcode(v[4]), addr(v[2]), hlcode(v[3])) + m.emit(0x49) + m.mrsd(lcode(v[4]), addr(v[2]), 1) + m.emit((hlcode(v[1]) << 4) | imml(v[0])) + }) + } + if p.len == 0 { + panic("invalid operands for VPERMIL2PD") + } + return p +} + +// XORPS performs "Bitwise Logical XOR for Single-Precision Floating-Point Values". +// +// Mnemonic : XORPS +// Supported forms : (2 forms) +// +// - XORPS xmm, xmm [SSE] +// - XORPS m128, xmm [SSE] +func (self *Program) XORPS(v0 interface{}, v1 interface{}) *Instruction { + p := self.alloc("XORPS", 2, Operands{v0, v1}) + // XORPS xmm, xmm + if isXMM(v0) && isXMM(v1) { + self.require(ISA_SSE) + p.domain = DomainMMXSSE + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexo(hcode(v[1]), v[0], false) + m.emit(0x0f) + m.emit(0x57) + m.emit(0xc0 | lcode(v[1])<<3 | lcode(v[0])) + }) + } + // XORPS m128, xmm + if isM128(v0) && isXMM(v1) { + self.require(ISA_SSE) + p.domain = DomainMMXSSE + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexo(hcode(v[1]), addr(v[0]), false) + m.emit(0x0f) + m.emit(0x57) + m.mrsd(lcode(v[1]), addr(v[0]), 1) + }) + } + if p.len == 0 { + panic("invalid operands for XORPS") + } + return p +} diff --git a/loader/internal/iasm/x86_64/instructions_table.go b/loader/internal/iasm/x86_64/instructions_table.go new file mode 100644 index 000000000..2becd9d00 --- /dev/null +++ b/loader/internal/iasm/x86_64/instructions_table.go @@ -0,0 +1,24 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Code generated by "mkasm_amd64.py", DO NOT EDIT. + +package x86_64 + +const ( + _N_args = 5 + _N_forms = 23 +) diff --git a/loader/internal/iasm/x86_64/instructions_test.go b/loader/internal/iasm/x86_64/instructions_test.go new file mode 100644 index 000000000..ed7f8876f --- /dev/null +++ b/loader/internal/iasm/x86_64/instructions_test.go @@ -0,0 +1,55 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package x86_64 + +import ( + "testing" + + "github.com/davecgh/go-spew/spew" +) + +func TestInstr_Encode(t *testing.T) { + m := []byte(nil) + a := CreateArch() + p := a.CreateProgram() + p.VPERMIL2PD(7, Sib(R8, R9, 1, 12345), YMM1, YMM2, YMM3).encode(&m) + spew.Dump(m) +} + +func TestInstr_EncodeSegment(t *testing.T) { + m := []byte(nil) + a := CreateArch() + p := a.CreateProgram() + p.MOVQ(Abs(0x30), RCX).GS().encode(&m) + spew.Dump(m) +} + +func BenchmarkInstr_Encode(b *testing.B) { + a := CreateArch() + m := make([]byte, 0, 16) + p := a.CreateProgram() + p.VPERMIL2PD(7, Sib(R8, R9, 1, 12345), YMM1, YMM2, YMM3).encode(&m) + p.Free() + b.SetBytes(int64(len(m))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + m = m[:0] + p = a.CreateProgram() + p.VPERMIL2PD(7, Sib(R8, R9, 1, 12345), YMM1, YMM2, YMM3).encode(&m) + p.Free() + } +} diff --git a/loader/internal/iasm/x86_64/operands.go b/loader/internal/iasm/x86_64/operands.go new file mode 100644 index 000000000..61adbf83b --- /dev/null +++ b/loader/internal/iasm/x86_64/operands.go @@ -0,0 +1,665 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package x86_64 + +import ( + "errors" + "fmt" + "math" + "reflect" + "strconv" + "strings" + "sync/atomic" +) + +// RelativeOffset represents an RIP-relative offset. +type RelativeOffset int32 + +// String implements the fmt.Stringer interface. +func (self RelativeOffset) String() string { + if self == 0 { + return "(%rip)" + } else { + return fmt.Sprintf("%d(%%rip)", self) + } +} + +// RoundingControl represents a floating-point rounding option. +type RoundingControl uint8 + +const ( + // RN_SAE represents "Round Nearest", which is the default rounding option. + RN_SAE RoundingControl = iota + + // RD_SAE represents "Round Down". + RD_SAE + + // RU_SAE represents "Round Up". + RU_SAE + + // RZ_SAE represents "Round towards Zero". + RZ_SAE +) + +var _RC_NAMES = map[RoundingControl]string{ + RN_SAE: "rn-sae", + RD_SAE: "rd-sae", + RU_SAE: "ru-sae", + RZ_SAE: "rz-sae", +} + +func (self RoundingControl) String() string { + if v, ok := _RC_NAMES[self]; ok { + return v + } else { + panic("invalid RoundingControl value") + } +} + +// ExceptionControl represents the "Suppress All Exceptions" flag. +type ExceptionControl uint8 + +const ( + // SAE represents the flag "Suppress All Exceptions" for floating point operations. + SAE ExceptionControl = iota +) + +func (ExceptionControl) String() string { + return "sae" +} + +// AddressType indicates which kind of value that an Addressable object contains. +type AddressType uint + +const ( + // None indicates the Addressable does not contain any addressable value. + None AddressType = iota + + // Memory indicates the Addressable contains a memory address. + Memory + + // Offset indicates the Addressable contains an RIP-relative offset. + Offset + + // Reference indicates the Addressable contains a label reference. + Reference +) + +// Disposable is a type of object that can be Free'd manually. +type Disposable interface { + Free() +} + +// Label represents a location within the program. +type Label struct { + refs int64 + Name string + Dest *Instruction +} + +func (self *Label) offset(p uintptr, n int) RelativeOffset { + if self.Dest == nil { + panic("unresolved label: " + self.Name) + } else { + return RelativeOffset(self.Dest.pc - p - uintptr(n)) + } +} + +// Free decreases the reference count of a Label, if the +// refcount drops to 0, the Label will be recycled. +func (self *Label) Free() { + if atomic.AddInt64(&self.refs, -1) == 0 { + //freeLabel(self) + } +} + +// String implements the fmt.Stringer interface. +func (self *Label) String() string { + if self.Dest == nil { + return fmt.Sprintf("%s(%%rip)", self.Name) + } else { + return fmt.Sprintf("%s(%%rip)@%#x", self.Name, self.Dest.pc) + } +} + +// Retain increases the reference count of a Label. +func (self *Label) Retain() *Label { + atomic.AddInt64(&self.refs, 1) + return self +} + +// Evaluate implements the interface expr.Term. +func (self *Label) Evaluate() (int64, error) { + if self.Dest != nil { + return int64(self.Dest.pc), nil + } else { + return 0, errors.New("unresolved label: " + self.Name) + } +} + +// Addressable is a union to represent an addressable operand. +type Addressable struct { + Type AddressType + Memory MemoryAddress + Offset RelativeOffset + Reference *Label +} + +// String implements the fmt.Stringer interface. +func (self *Addressable) String() string { + switch self.Type { + case None: + return "(not addressable)" + case Memory: + return self.Memory.String() + case Offset: + return self.Offset.String() + case Reference: + return self.Reference.String() + default: + return "(invalid addressable)" + } +} + +// MemoryOperand represents a memory operand for an instruction. +type MemoryOperand struct { + refs int64 + Size int + Addr Addressable + Mask RegisterMask + Masked bool + Broadcast uint8 +} + +const ( + _Sizes = 0b10000000100010111 // bit-mask for valid sizes (0, 1, 2, 4, 8, 16) +) + +func (self *MemoryOperand) isVMX(evex bool) bool { + return self.Addr.Type == Memory && self.Addr.Memory.isVMX(evex) +} + +func (self *MemoryOperand) isVMY(evex bool) bool { + return self.Addr.Type == Memory && self.Addr.Memory.isVMY(evex) +} + +func (self *MemoryOperand) isVMZ() bool { + return self.Addr.Type == Memory && self.Addr.Memory.isVMZ() +} + +func (self *MemoryOperand) isMem() bool { + if (_Sizes & (1 << self.Broadcast)) == 0 { + return false + } else if self.Addr.Type == Memory { + return self.Addr.Memory.isMem() + } else if self.Addr.Type == Offset { + return true + } else if self.Addr.Type == Reference { + return true + } else { + return false + } +} + +func (self *MemoryOperand) isSize(n int) bool { + return self.Size == 0 || self.Size == n +} + +func (self *MemoryOperand) isBroadcast(n int, b uint8) bool { + return self.Size == n && self.Broadcast == b +} + +func (self *MemoryOperand) formatMask() string { + if !self.Masked { + return "" + } else { + return self.Mask.String() + } +} + +func (self *MemoryOperand) formatBroadcast() string { + if self.Broadcast == 0 { + return "" + } else { + return fmt.Sprintf("{1to%d}", self.Broadcast) + } +} + +func (self *MemoryOperand) ensureAddrValid() { + switch self.Addr.Type { + case None: + break + case Memory: + self.Addr.Memory.EnsureValid() + case Offset: + break + case Reference: + break + default: + panic("invalid address type") + } +} + +func (self *MemoryOperand) ensureSizeValid() { + if (_Sizes & (1 << self.Size)) == 0 { + panic("invalid memory operand size") + } +} + +func (self *MemoryOperand) ensureBroadcastValid() { + if (_Sizes & (1 << self.Broadcast)) == 0 { + panic("invalid memory operand broadcast") + } +} + +// Free decreases the reference count of a MemoryOperand, if the +// refcount drops to 0, the Label will be recycled. +func (self *MemoryOperand) Free() { + if atomic.AddInt64(&self.refs, -1) == 0 { + //freeMemoryOperand(self) + } +} + +// String implements the fmt.Stringer interface. +func (self *MemoryOperand) String() string { + return self.Addr.String() + self.formatMask() + self.formatBroadcast() +} + +// Retain increases the reference count of a MemoryOperand. +func (self *MemoryOperand) Retain() *MemoryOperand { + atomic.AddInt64(&self.refs, 1) + return self +} + +// EnsureValid checks if the memory operand is valid, if not, it panics. +func (self *MemoryOperand) EnsureValid() { + self.ensureAddrValid() + self.ensureSizeValid() + self.ensureBroadcastValid() +} + +// MemoryAddress represents a memory address. +type MemoryAddress struct { + Base Register + Index Register + Scale uint8 + Displacement int32 +} + +const ( + _Scales = 0b100010111 // bit-mask for valid scales (0, 1, 2, 4, 8) +) + +func (self *MemoryAddress) isVMX(evex bool) bool { + return self.isMemBase() && (self.Index == nil || isXMM(self.Index) || (evex && isEVEXXMM(self.Index))) +} + +func (self *MemoryAddress) isVMY(evex bool) bool { + return self.isMemBase() && (self.Index == nil || isYMM(self.Index) || (evex && isEVEXYMM(self.Index))) +} + +func (self *MemoryAddress) isVMZ() bool { + return self.isMemBase() && (self.Index == nil || isZMM(self.Index)) +} + +func (self *MemoryAddress) isMem() bool { + return self.isMemBase() && (self.Index == nil || isReg64(self.Index)) +} + +func (self *MemoryAddress) isMemBase() bool { + return (self.Base == nil || isReg64(self.Base)) && // `Base` must be 64-bit if present + (self.Scale == 0) == (self.Index == nil) && // `Scale` and `Index` depends on each other + (_Scales&(1<= 2 { + sb.WriteByte(',') + sb.WriteString(strconv.Itoa(int(self.Scale))) + } + } + + /* close the bracket */ + sb.WriteByte(')') + return sb.String() +} + +// EnsureValid checks if the memory address is valid, if not, it panics. +func (self *MemoryAddress) EnsureValid() { + if !self.isMemBase() || (self.Index != nil && !isIndexable(self.Index)) { + panic("not a valid memory address") + } +} + +// Ref constructs a memory reference to a label. +func Ref(ref *Label) (v *MemoryOperand) { + v = CreateMemoryOperand() + v.Addr.Type = Reference + v.Addr.Reference = ref + return +} + +// Abs construct a simple memory address that represents absolute addressing. +func Abs(disp int32) *MemoryOperand { + return Sib(nil, nil, 0, disp) +} + +// Ptr constructs a simple memory operand with base and displacement. +func Ptr(base Register, disp int32) *MemoryOperand { + return Sib(base, nil, 0, disp) +} + +// Sib constructs a simple memory operand that represents a complete memory address. +func Sib(base Register, index Register, scale uint8, disp int32) (v *MemoryOperand) { + v = CreateMemoryOperand() + v.Addr.Type = Memory + v.Addr.Memory.Base = base + v.Addr.Memory.Index = index + v.Addr.Memory.Scale = scale + v.Addr.Memory.Displacement = disp + v.EnsureValid() + return +} + +/** Operand Matching Helpers **/ + +const _IntMask = (1 << reflect.Int) | + (1 << reflect.Int8) | + (1 << reflect.Int16) | + (1 << reflect.Int32) | + (1 << reflect.Int64) | + (1 << reflect.Uint) | + (1 << reflect.Uint8) | + (1 << reflect.Uint16) | + (1 << reflect.Uint32) | + (1 << reflect.Uint64) | + (1 << reflect.Uintptr) + +func isInt(k reflect.Kind) bool { + return (_IntMask & (1 << k)) != 0 +} + +func asInt64(v interface{}) (int64, bool) { + if isSpecial(v) { + return 0, false + } else if x := efaceOf(v); isInt(x.kind()) { + return x.toInt64(), true + } else { + return 0, false + } +} + +func inRange(v interface{}, low int64, high int64) bool { + x, ok := asInt64(v) + return ok && x >= low && x <= high +} + +func isSpecial(v interface{}) bool { + switch v.(type) { + case Register8: + return true + case Register16: + return true + case Register32: + return true + case Register64: + return true + case KRegister: + return true + case MMRegister: + return true + case XMMRegister: + return true + case YMMRegister: + return true + case ZMMRegister: + return true + case RelativeOffset: + return true + case RoundingControl: + return true + case ExceptionControl: + return true + default: + return false + } +} + +func isIndexable(v interface{}) bool { + return isZMM(v) || isReg64(v) || isEVEXXMM(v) || isEVEXYMM(v) +} + +func isImm4(v interface{}) bool { return inRange(v, 0, 15) } +func isImm8(v interface{}) bool { return inRange(v, math.MinInt8, math.MaxUint8) } +func isImm16(v interface{}) bool { return inRange(v, math.MinInt16, math.MaxUint16) } +func isImm32(v interface{}) bool { return inRange(v, math.MinInt32, math.MaxUint32) } +func isImm64(v interface{}) bool { _, r := asInt64(v); return r } +func isConst1(v interface{}) bool { x, r := asInt64(v); return r && x == 1 } +func isConst3(v interface{}) bool { x, r := asInt64(v); return r && x == 3 } +func isRel8(v interface{}) bool { + x, r := v.(RelativeOffset) + return r && x >= math.MinInt8 && x <= math.MaxInt8 +} +func isRel32(v interface{}) bool { _, r := v.(RelativeOffset); return r } +func isLabel(v interface{}) bool { _, r := v.(*Label); return r } +func isReg8(v interface{}) bool { _, r := v.(Register8); return r } +func isReg8REX(v interface{}) bool { + x, r := v.(Register8) + return r && (x&0x80) == 0 && x >= SPL +} +func isReg16(v interface{}) bool { _, r := v.(Register16); return r } +func isReg32(v interface{}) bool { _, r := v.(Register32); return r } +func isReg64(v interface{}) bool { _, r := v.(Register64); return r } +func isMM(v interface{}) bool { _, r := v.(MMRegister); return r } +func isXMM(v interface{}) bool { x, r := v.(XMMRegister); return r && x <= XMM15 } +func isEVEXXMM(v interface{}) bool { _, r := v.(XMMRegister); return r } +func isXMMk(v interface{}) bool { + x, r := v.(MaskedRegister) + return isXMM(v) || (r && isXMM(x.Reg) && !x.Mask.Z) +} +func isXMMkz(v interface{}) bool { + x, r := v.(MaskedRegister) + return isXMM(v) || (r && isXMM(x.Reg)) +} +func isYMM(v interface{}) bool { x, r := v.(YMMRegister); return r && x <= YMM15 } +func isEVEXYMM(v interface{}) bool { _, r := v.(YMMRegister); return r } +func isYMMk(v interface{}) bool { + x, r := v.(MaskedRegister) + return isYMM(v) || (r && isYMM(x.Reg) && !x.Mask.Z) +} +func isYMMkz(v interface{}) bool { + x, r := v.(MaskedRegister) + return isYMM(v) || (r && isYMM(x.Reg)) +} +func isZMM(v interface{}) bool { _, r := v.(ZMMRegister); return r } +func isZMMk(v interface{}) bool { + x, r := v.(MaskedRegister) + return isZMM(v) || (r && isZMM(x.Reg) && !x.Mask.Z) +} +func isZMMkz(v interface{}) bool { + x, r := v.(MaskedRegister) + return isZMM(v) || (r && isZMM(x.Reg)) +} +func isK(v interface{}) bool { _, r := v.(KRegister); return r } +func isKk(v interface{}) bool { + x, r := v.(MaskedRegister) + return isK(v) || (r && isK(x.Reg) && !x.Mask.Z) +} +func isM(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && x.isMem() && x.Broadcast == 0 && !x.Masked +} +func isMk(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && x.isMem() && x.Broadcast == 0 && !(x.Masked && x.Mask.Z) +} +func isMkz(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && x.isMem() && x.Broadcast == 0 +} +func isM8(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && isM(v) && x.isSize(1) +} +func isM16(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && isM(v) && x.isSize(2) +} +func isM16kz(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && isMkz(v) && x.isSize(2) +} +func isM32(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && isM(v) && x.isSize(4) +} +func isM32k(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && isMk(v) && x.isSize(4) +} +func isM32kz(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && isMkz(v) && x.isSize(4) +} +func isM64(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && isM(v) && x.isSize(8) +} +func isM64k(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && isMk(v) && x.isSize(8) +} +func isM64kz(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && isMkz(v) && x.isSize(8) +} +func isM128(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && isM(v) && x.isSize(16) +} +func isM128kz(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && isMkz(v) && x.isSize(16) +} +func isM256(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && isM(v) && x.isSize(32) +} +func isM256kz(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && isMkz(v) && x.isSize(32) +} +func isM512(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && isM(v) && x.isSize(64) +} +func isM512kz(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && isMkz(v) && x.isSize(64) +} +func isM64M32bcst(v interface{}) bool { + x, r := v.(*MemoryOperand) + return isM64(v) || (r && x.isBroadcast(4, 2)) +} +func isM128M32bcst(v interface{}) bool { + x, r := v.(*MemoryOperand) + return isM128(v) || (r && x.isBroadcast(4, 4)) +} +func isM256M32bcst(v interface{}) bool { + x, r := v.(*MemoryOperand) + return isM256(v) || (r && x.isBroadcast(4, 8)) +} +func isM512M32bcst(v interface{}) bool { + x, r := v.(*MemoryOperand) + return isM512(v) || (r && x.isBroadcast(4, 16)) +} +func isM128M64bcst(v interface{}) bool { + x, r := v.(*MemoryOperand) + return isM128(v) || (r && x.isBroadcast(8, 2)) +} +func isM256M64bcst(v interface{}) bool { + x, r := v.(*MemoryOperand) + return isM256(v) || (r && x.isBroadcast(8, 4)) +} +func isM512M64bcst(v interface{}) bool { + x, r := v.(*MemoryOperand) + return isM512(v) || (r && x.isBroadcast(8, 8)) +} +func isVMX(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && x.isVMX(false) && !x.Masked +} +func isEVEXVMX(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && x.isVMX(true) && !x.Masked +} +func isVMXk(v interface{}) bool { x, r := v.(*MemoryOperand); return r && x.isVMX(true) } +func isVMY(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && x.isVMY(false) && !x.Masked +} +func isEVEXVMY(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && x.isVMY(true) && !x.Masked +} +func isVMYk(v interface{}) bool { x, r := v.(*MemoryOperand); return r && x.isVMY(true) } +func isVMZ(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && x.isVMZ() && !x.Masked +} +func isVMZk(v interface{}) bool { x, r := v.(*MemoryOperand); return r && x.isVMZ() } +func isSAE(v interface{}) bool { _, r := v.(ExceptionControl); return r } +func isER(v interface{}) bool { _, r := v.(RoundingControl); return r } + +func isImmExt(v interface{}, ext int, min int64, max int64) bool { + if x, ok := asInt64(v); !ok { + return false + } else if m := int64(1) << (8 * ext); x < m && x >= m+min { + return true + } else { + return x <= max && x >= min + } +} + +func isImm8Ext(v interface{}, ext int) bool { + return isImmExt(v, ext, math.MinInt8, math.MaxInt8) +} + +func isImm32Ext(v interface{}, ext int) bool { + return isImmExt(v, ext, math.MinInt32, math.MaxInt32) +} diff --git a/loader/internal/iasm/x86_64/pools.go b/loader/internal/iasm/x86_64/pools.go new file mode 100644 index 000000000..690db5b7a --- /dev/null +++ b/loader/internal/iasm/x86_64/pools.go @@ -0,0 +1,54 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package x86_64 + +// CreateLabel creates a new Label, it may allocate a new one or grab one from a pool. +func CreateLabel(name string) *Label { + p := new(Label) + + /* initialize the label */ + p.refs = 1 + p.Name = name + return p +} + +func newProgram(arch *Arch) *Program { + p := new(Program) + + /* initialize the program */ + p.arch = arch + return p +} + +func newInstruction(name string, argc int, argv Operands) *Instruction { + p := new(Instruction) + + /* initialize the instruction */ + p.name = name + p.argc = argc + p.argv = argv + return p +} + +// CreateMemoryOperand creates a new MemoryOperand, it may allocate a new one or grab one from a pool. +func CreateMemoryOperand() *MemoryOperand { + p := new(MemoryOperand) + + /* initialize the memory operand */ + p.refs = 1 + return p +} diff --git a/loader/internal/iasm/x86_64/program.go b/loader/internal/iasm/x86_64/program.go new file mode 100644 index 000000000..bf7d3a1dc --- /dev/null +++ b/loader/internal/iasm/x86_64/program.go @@ -0,0 +1,584 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package x86_64 + +import ( + "fmt" + "math" + "math/bits" + + "github.com/bytedance/sonic/loader/internal/iasm/expr" +) + +type ( + _PseudoType int + _InstructionEncoder func(*Program, ...interface{}) *Instruction +) + +const ( + _PseudoNop _PseudoType = iota + 1 + _PseudoByte + _PseudoWord + _PseudoLong + _PseudoQuad + _PseudoData + _PseudoAlign +) + +func (self _PseudoType) String() string { + switch self { + case _PseudoNop: + return ".nop" + case _PseudoByte: + return ".byte" + case _PseudoWord: + return ".word" + case _PseudoLong: + return ".long" + case _PseudoQuad: + return ".quad" + case _PseudoData: + return ".data" + case _PseudoAlign: + return ".align" + default: + panic("unreachable") + } +} + +type _Pseudo struct { + kind _PseudoType + data []byte + uint uint64 + expr *expr.Expr +} + +func (self *_Pseudo) free() { + if self.expr != nil { + self.expr.Free() + } +} + +func (self *_Pseudo) encode(m *[]byte, pc uintptr) int { + switch self.kind { + case _PseudoNop: + return 0 + case _PseudoByte: + self.encodeByte(m) + return 1 + case _PseudoWord: + self.encodeWord(m) + return 2 + case _PseudoLong: + self.encodeLong(m) + return 4 + case _PseudoQuad: + self.encodeQuad(m) + return 8 + case _PseudoData: + self.encodeData(m) + return len(self.data) + case _PseudoAlign: + self.encodeAlign(m, pc) + return self.alignSize(pc) + default: + panic("invalid pseudo instruction") + } +} + +func (self *_Pseudo) evalExpr(low int64, high int64) int64 { + if v, err := self.expr.Evaluate(); err != nil { + panic(err) + } else if v < low || v > high { + panic(fmt.Sprintf("expression out of range [%d, %d]: %d", low, high, v)) + } else { + return v + } +} + +func (self *_Pseudo) alignSize(pc uintptr) int { + if !ispow2(self.uint) { + panic(fmt.Sprintf("aligment should be a power of 2, not %d", self.uint)) + } else { + return align(int(pc), bits.TrailingZeros64(self.uint)) - int(pc) + } +} + +func (self *_Pseudo) encodeData(m *[]byte) { + if m != nil { + *m = append(*m, self.data...) + } +} + +func (self *_Pseudo) encodeByte(m *[]byte) { + if m != nil { + append8(m, byte(self.evalExpr(math.MinInt8, math.MaxUint8))) + } +} + +func (self *_Pseudo) encodeWord(m *[]byte) { + if m != nil { + append16(m, uint16(self.evalExpr(math.MinInt16, math.MaxUint16))) + } +} + +func (self *_Pseudo) encodeLong(m *[]byte) { + if m != nil { + append32(m, uint32(self.evalExpr(math.MinInt32, math.MaxUint32))) + } +} + +func (self *_Pseudo) encodeQuad(m *[]byte) { + if m != nil { + if v, err := self.expr.Evaluate(); err != nil { + panic(err) + } else { + append64(m, uint64(v)) + } + } +} + +func (self *_Pseudo) encodeAlign(m *[]byte, pc uintptr) { + if m != nil { + if self.expr == nil { + expandmm(m, self.alignSize(pc), 0) + } else { + expandmm(m, self.alignSize(pc), byte(self.evalExpr(math.MinInt8, math.MaxUint8))) + } + } +} + +// Operands represents a sequence of operand required by an instruction. +type Operands [_N_args]interface{} + +// InstructionDomain represents the domain of an instruction. +type InstructionDomain uint8 + +const ( + DomainGeneric InstructionDomain = iota + DomainMMXSSE + DomainAVX + DomainFMA + DomainCrypto + DomainMask + DomainAMDSpecific + DomainMisc + DomainPseudo +) + +type ( + _BranchType uint8 +) + +const ( + _B_none _BranchType = iota + _B_conditional + _B_unconditional +) + +// Instruction represents an unencoded instruction. +type Instruction struct { + next *Instruction + pc uintptr + nb int + len int + argc int + name string + argv Operands + forms [_N_forms]_Encoding + pseudo _Pseudo + branch _BranchType + domain InstructionDomain + prefix []byte +} + +func (self *Instruction) add(flags int, encoder func(m *_Encoding, v []interface{})) { + self.forms[self.len].flags = flags + self.forms[self.len].encoder = encoder + self.len++ +} + +func (self *Instruction) free() { + self.clear() + self.pseudo.free() + //freeInstruction(self) +} + +func (self *Instruction) clear() { + for i := 0; i < self.argc; i++ { + if v, ok := self.argv[i].(Disposable); ok { + v.Free() + } + } +} + +func (self *Instruction) check(e *_Encoding) bool { + if (e.flags & _F_rel1) != 0 { + return isRel8(self.argv[0]) + } else if (e.flags & _F_rel4) != 0 { + return isRel32(self.argv[0]) || isLabel(self.argv[0]) + } else { + return true + } +} + +func (self *Instruction) encode(m *[]byte) int { + n := math.MaxInt64 + p := (*_Encoding)(nil) + + /* encode prefixes if any */ + if self.nb = len(self.prefix); m != nil { + *m = append(*m, self.prefix...) + } + + /* check for pseudo-instructions */ + if self.pseudo.kind != 0 { + self.nb += self.pseudo.encode(m, self.pc) + return self.nb + } + + /* find the shortest encoding */ + for i := 0; i < self.len; i++ { + if e := &self.forms[i]; self.check(e) { + if v := e.encode(self.argv[:self.argc]); v < n { + n = v + p = e + } + } + } + + /* add to buffer if needed */ + if m != nil { + *m = append(*m, p.bytes[:n]...) + } + + /* update the instruction length */ + self.nb += n + return self.nb +} + +/** Instruction Prefixes **/ + +const ( + _P_cs = 0x2e + _P_ds = 0x3e + _P_es = 0x26 + _P_fs = 0x64 + _P_gs = 0x65 + _P_ss = 0x36 + _P_lock = 0xf0 +) + +// CS overrides the memory operation of this instruction to CS. +func (self *Instruction) CS() *Instruction { + self.prefix = append(self.prefix, _P_cs) + return self +} + +// DS overrides the memory operation of this instruction to DS, +// this is the default section for most instructions if not specified. +func (self *Instruction) DS() *Instruction { + self.prefix = append(self.prefix, _P_ds) + return self +} + +// ES overrides the memory operation of this instruction to ES. +func (self *Instruction) ES() *Instruction { + self.prefix = append(self.prefix, _P_es) + return self +} + +// FS overrides the memory operation of this instruction to FS. +func (self *Instruction) FS() *Instruction { + self.prefix = append(self.prefix, _P_fs) + return self +} + +// GS overrides the memory operation of this instruction to GS. +func (self *Instruction) GS() *Instruction { + self.prefix = append(self.prefix, _P_gs) + return self +} + +// SS overrides the memory operation of this instruction to SS. +func (self *Instruction) SS() *Instruction { + self.prefix = append(self.prefix, _P_ss) + return self +} + +// LOCK causes the processor's LOCK# signal to be asserted during execution of +// the accompanying instruction (turns the instruction into an atomic instruction). +// In a multiprocessor environment, the LOCK# signal insures that the processor +// has exclusive use of any shared memory while the signal is asserted. +func (self *Instruction) LOCK() *Instruction { + self.prefix = append(self.prefix, _P_lock) + return self +} + +/** Basic Instruction Properties **/ + +// Name returns the instruction name. +func (self *Instruction) Name() string { + return self.name +} + +// Domain returns the domain of this instruction. +func (self *Instruction) Domain() InstructionDomain { + return self.domain +} + +// Operands returns the operands of this instruction. +func (self *Instruction) Operands() []interface{} { + return self.argv[:self.argc] +} + +// Program represents a sequence of instructions. +type Program struct { + arch *Arch + head *Instruction + tail *Instruction +} + +const ( + _N_near = 2 // near-branch (-128 ~ +127) takes 2 bytes to encode + _N_far_cond = 6 // conditional far-branch takes 6 bytes to encode + _N_far_uncond = 5 // unconditional far-branch takes 5 bytes to encode +) + +func (self *Program) clear() { + for p, q := self.head, self.head; p != nil; p = q { + q = p.next + p.free() + } +} + +func (self *Program) alloc(name string, argc int, argv Operands) *Instruction { + p := self.tail + q := newInstruction(name, argc, argv) + + /* attach to tail if any */ + if p != nil { + p.next = q + } else { + self.head = q + } + + /* set the new tail */ + self.tail = q + return q +} + +func (self *Program) pseudo(kind _PseudoType) (p *Instruction) { + p = self.alloc(kind.String(), 0, Operands{}) + p.domain = DomainPseudo + p.pseudo.kind = kind + return +} + +func (self *Program) require(isa ISA) { + if !self.arch.HasISA(isa) { + panic("ISA '" + isa.String() + "' was not enabled") + } +} + +func (self *Program) branchSize(p *Instruction) int { + switch p.branch { + case _B_none: + panic("p is not a branch") + case _B_conditional: + return _N_far_cond + case _B_unconditional: + return _N_far_uncond + default: + panic("invalid instruction") + } +} + +/** Pseudo-Instructions **/ + +// Byte is a pseudo-instruction to add raw byte to the assembled code. +func (self *Program) Byte(v *expr.Expr) (p *Instruction) { + p = self.pseudo(_PseudoByte) + p.pseudo.expr = v + return +} + +// Word is a pseudo-instruction to add raw uint16 as little-endian to the assembled code. +func (self *Program) Word(v *expr.Expr) (p *Instruction) { + p = self.pseudo(_PseudoWord) + p.pseudo.expr = v + return +} + +// Long is a pseudo-instruction to add raw uint32 as little-endian to the assembled code. +func (self *Program) Long(v *expr.Expr) (p *Instruction) { + p = self.pseudo(_PseudoLong) + p.pseudo.expr = v + return +} + +// Quad is a pseudo-instruction to add raw uint64 as little-endian to the assembled code. +func (self *Program) Quad(v *expr.Expr) (p *Instruction) { + p = self.pseudo(_PseudoQuad) + p.pseudo.expr = v + return +} + +// Data is a pseudo-instruction to add raw bytes to the assembled code. +func (self *Program) Data(v []byte) (p *Instruction) { + p = self.pseudo(_PseudoData) + p.pseudo.data = v + return +} + +// Align is a pseudo-instruction to ensure the PC is aligned to a certain value. +func (self *Program) Align(align uint64, padding *expr.Expr) (p *Instruction) { + p = self.pseudo(_PseudoAlign) + p.pseudo.uint = align + p.pseudo.expr = padding + return +} + +/** Program Assembler **/ + +// Free returns the Program object into pool. +// Any operation performed after Free is undefined behavior. +// +// NOTE: This also frees all the instructions, labels, memory +// +// operands and expressions associated with this program. +func (self *Program) Free() { + self.clear() + //freeProgram(self) +} + +// Link pins a label at the current position. +func (self *Program) Link(p *Label) { + if p.Dest != nil { + panic("lable was alreay linked") + } else { + p.Dest = self.pseudo(_PseudoNop) + } +} + +// Assemble assembles and links the entire program into machine code. +func (self *Program) Assemble(pc uintptr) (ret []byte) { + orig := pc + next := true + offs := uintptr(0) + + /* Pass 0: PC-precompute, assume all labeled branches are far-branches. */ + for p := self.head; p != nil; p = p.next { + if p.pc = pc; !isLabel(p.argv[0]) || p.branch == _B_none { + pc += uintptr(p.encode(nil)) + } else { + pc += uintptr(self.branchSize(p)) + } + } + + /* allocate space for the machine code */ + nb := int(pc - orig) + ret = make([]byte, 0, nb) + + /* Pass 1: adjust all the jumps */ + for next { + next = false + offs = uintptr(0) + + /* scan all the branches */ + for p := self.head; p != nil; p = p.next { + var ok bool + var lb *Label + + /* re-calculate the alignment here */ + if nb = p.nb; p.pseudo.kind == _PseudoAlign { + p.pc -= offs + offs += uintptr(nb - p.encode(nil)) + continue + } + + /* adjust the program counter */ + p.pc -= offs + lb, ok = p.argv[0].(*Label) + + /* only care about labeled far-branches */ + if !ok || p.nb == _N_near || p.branch == _B_none { + continue + } + + /* calculate the jump offset */ + size := self.branchSize(p) + diff := lb.offset(p.pc, size) + + /* too far to be a near jump */ + if diff > 127 || diff < -128 { + p.nb = size + continue + } + + /* a far jump becomes a near jump, calculate + * the PC adjustment value and assemble again */ + next = true + p.nb = _N_near + offs += uintptr(size - _N_near) + } + } + + /* Pass 3: link all the cross-references */ + for p := self.head; p != nil; p = p.next { + for i := 0; i < p.argc; i++ { + var ok bool + var lb *Label + var op *MemoryOperand + + /* resolve labels */ + if lb, ok = p.argv[i].(*Label); ok { + p.argv[i] = lb.offset(p.pc, p.nb) + continue + } + + /* check for memory operands */ + if op, ok = p.argv[i].(*MemoryOperand); !ok { + continue + } + + /* check for label references */ + if op.Addr.Type != Reference { + continue + } + + /* replace the label with the real offset */ + op.Addr.Type = Offset + op.Addr.Offset = op.Addr.Reference.offset(p.pc, p.nb) + } + } + + /* Pass 4: actually encode all the instructions */ + for p := self.head; p != nil; p = p.next { + p.encode(&ret) + } + + /* all done */ + return ret +} + +// AssembleAndFree is like Assemble, but it frees the Program after assembling. +func (self *Program) AssembleAndFree(pc uintptr) (ret []byte) { + ret = self.Assemble(pc) + self.Free() + return +} diff --git a/loader/internal/iasm/x86_64/program_test.go b/loader/internal/iasm/x86_64/program_test.go new file mode 100644 index 000000000..c31d87f19 --- /dev/null +++ b/loader/internal/iasm/x86_64/program_test.go @@ -0,0 +1,49 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package x86_64 + +import ( + "bytes" + "testing" + + "github.com/bytedance/sonic/loader/internal/iasm/expr" + "github.com/davecgh/go-spew/spew" +) + +func TestProgram_Assemble(t *testing.T) { + a := CreateArch() + b := CreateLabel("bak") + s := CreateLabel("tab") + j := CreateLabel("jmp") + p := a.CreateProgram() + p.JMP(j) + p.JMP(j) + p.Link(b) + p.Data(bytes.Repeat([]byte{0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00}, 15)) + p.Data([]byte{0x0f, 0x1f, 0x00}) + p.JMP(b) + p.Link(j) + p.LEAQ(Ref(s), RDI) + p.MOVSLQ(Sib(RDI, RAX, 4, -4), RAX) + p.ADDQ(RDI, RAX) + p.JMPQ(RAX) + p.Align(32, expr.Int(0xcc)) + p.Link(s) + p.Long(expr.Ref(s.Retain()).Sub(expr.Ref(j.Retain()))) + p.Long(expr.Ref(s.Retain()).Sub(expr.Ref(b.Retain()))) + spew.Dump(p.AssembleAndFree(0)) +} diff --git a/loader/internal/iasm/x86_64/registers.go b/loader/internal/iasm/x86_64/registers.go new file mode 100644 index 000000000..265575a26 --- /dev/null +++ b/loader/internal/iasm/x86_64/registers.go @@ -0,0 +1,747 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package x86_64 + +import ( + "fmt" +) + +// Register represents a hardware register. +type Register interface { + fmt.Stringer + implRegister() +} + +type ( + Register8 byte + Register16 byte + Register32 byte + Register64 byte +) + +type ( + KRegister byte + MMRegister byte + XMMRegister byte + YMMRegister byte + ZMMRegister byte +) + +// RegisterMask is a KRegister used to mask another register. +type RegisterMask struct { + Z bool + K KRegister +} + +// String implements the fmt.Stringer interface. +func (self RegisterMask) String() string { + if !self.Z { + return fmt.Sprintf("{%%%s}", self.K) + } else { + return fmt.Sprintf("{%%%s}{z}", self.K) + } +} + +// MaskedRegister is a Register masked by a RegisterMask. +type MaskedRegister struct { + Reg Register + Mask RegisterMask +} + +// String implements the fmt.Stringer interface. +func (self MaskedRegister) String() string { + return self.Reg.String() + self.Mask.String() +} + +const ( + AL Register8 = iota + CL + DL + BL + SPL + BPL + SIL + DIL + R8b + R9b + R10b + R11b + R12b + R13b + R14b + R15b +) + +const ( + AH = SPL | 0x80 + CH = BPL | 0x80 + DH = SIL | 0x80 + BH = DIL | 0x80 +) + +const ( + AX Register16 = iota + CX + DX + BX + SP + BP + SI + DI + R8w + R9w + R10w + R11w + R12w + R13w + R14w + R15w +) + +const ( + EAX Register32 = iota + ECX + EDX + EBX + ESP + EBP + ESI + EDI + R8d + R9d + R10d + R11d + R12d + R13d + R14d + R15d +) + +const ( + RAX Register64 = iota + RCX + RDX + RBX + RSP + RBP + RSI + RDI + R8 + R9 + R10 + R11 + R12 + R13 + R14 + R15 +) + +const ( + K0 KRegister = iota + K1 + K2 + K3 + K4 + K5 + K6 + K7 +) + +const ( + MM0 MMRegister = iota + MM1 + MM2 + MM3 + MM4 + MM5 + MM6 + MM7 +) + +const ( + XMM0 XMMRegister = iota + XMM1 + XMM2 + XMM3 + XMM4 + XMM5 + XMM6 + XMM7 + XMM8 + XMM9 + XMM10 + XMM11 + XMM12 + XMM13 + XMM14 + XMM15 + XMM16 + XMM17 + XMM18 + XMM19 + XMM20 + XMM21 + XMM22 + XMM23 + XMM24 + XMM25 + XMM26 + XMM27 + XMM28 + XMM29 + XMM30 + XMM31 +) + +const ( + YMM0 YMMRegister = iota + YMM1 + YMM2 + YMM3 + YMM4 + YMM5 + YMM6 + YMM7 + YMM8 + YMM9 + YMM10 + YMM11 + YMM12 + YMM13 + YMM14 + YMM15 + YMM16 + YMM17 + YMM18 + YMM19 + YMM20 + YMM21 + YMM22 + YMM23 + YMM24 + YMM25 + YMM26 + YMM27 + YMM28 + YMM29 + YMM30 + YMM31 +) + +const ( + ZMM0 ZMMRegister = iota + ZMM1 + ZMM2 + ZMM3 + ZMM4 + ZMM5 + ZMM6 + ZMM7 + ZMM8 + ZMM9 + ZMM10 + ZMM11 + ZMM12 + ZMM13 + ZMM14 + ZMM15 + ZMM16 + ZMM17 + ZMM18 + ZMM19 + ZMM20 + ZMM21 + ZMM22 + ZMM23 + ZMM24 + ZMM25 + ZMM26 + ZMM27 + ZMM28 + ZMM29 + ZMM30 + ZMM31 +) + +func (self Register8) implRegister() {} +func (self Register16) implRegister() {} +func (self Register32) implRegister() {} +func (self Register64) implRegister() {} + +func (self KRegister) implRegister() {} +func (self MMRegister) implRegister() {} +func (self XMMRegister) implRegister() {} +func (self YMMRegister) implRegister() {} +func (self ZMMRegister) implRegister() {} + +func (self Register8) String() string { + if int(self) >= len(r8names) { + return "???" + } else { + return r8names[self] + } +} +func (self Register16) String() string { + if int(self) >= len(r16names) { + return "???" + } else { + return r16names[self] + } +} +func (self Register32) String() string { + if int(self) >= len(r32names) { + return "???" + } else { + return r32names[self] + } +} +func (self Register64) String() string { + if int(self) >= len(r64names) { + return "???" + } else { + return r64names[self] + } +} + +func (self KRegister) String() string { + if int(self) >= len(knames) { + return "???" + } else { + return knames[self] + } +} +func (self MMRegister) String() string { + if int(self) >= len(mmnames) { + return "???" + } else { + return mmnames[self] + } +} +func (self XMMRegister) String() string { + if int(self) >= len(xmmnames) { + return "???" + } else { + return xmmnames[self] + } +} +func (self YMMRegister) String() string { + if int(self) >= len(ymmnames) { + return "???" + } else { + return ymmnames[self] + } +} +func (self ZMMRegister) String() string { + if int(self) >= len(zmmnames) { + return "???" + } else { + return zmmnames[self] + } +} + +// Registers maps register name into Register instances. +var Registers = map[string]Register{ + "al": AL, + "cl": CL, + "dl": DL, + "bl": BL, + "spl": SPL, + "bpl": BPL, + "sil": SIL, + "dil": DIL, + "r8b": R8b, + "r9b": R9b, + "r10b": R10b, + "r11b": R11b, + "r12b": R12b, + "r13b": R13b, + "r14b": R14b, + "r15b": R15b, + "ah": AH, + "ch": CH, + "dh": DH, + "bh": BH, + "ax": AX, + "cx": CX, + "dx": DX, + "bx": BX, + "sp": SP, + "bp": BP, + "si": SI, + "di": DI, + "r8w": R8w, + "r9w": R9w, + "r10w": R10w, + "r11w": R11w, + "r12w": R12w, + "r13w": R13w, + "r14w": R14w, + "r15w": R15w, + "eax": EAX, + "ecx": ECX, + "edx": EDX, + "ebx": EBX, + "esp": ESP, + "ebp": EBP, + "esi": ESI, + "edi": EDI, + "r8d": R8d, + "r9d": R9d, + "r10d": R10d, + "r11d": R11d, + "r12d": R12d, + "r13d": R13d, + "r14d": R14d, + "r15d": R15d, + "rax": RAX, + "rcx": RCX, + "rdx": RDX, + "rbx": RBX, + "rsp": RSP, + "rbp": RBP, + "rsi": RSI, + "rdi": RDI, + "r8": R8, + "r9": R9, + "r10": R10, + "r11": R11, + "r12": R12, + "r13": R13, + "r14": R14, + "r15": R15, + "k0": K0, + "k1": K1, + "k2": K2, + "k3": K3, + "k4": K4, + "k5": K5, + "k6": K6, + "k7": K7, + "mm0": MM0, + "mm1": MM1, + "mm2": MM2, + "mm3": MM3, + "mm4": MM4, + "mm5": MM5, + "mm6": MM6, + "mm7": MM7, + "xmm0": XMM0, + "xmm1": XMM1, + "xmm2": XMM2, + "xmm3": XMM3, + "xmm4": XMM4, + "xmm5": XMM5, + "xmm6": XMM6, + "xmm7": XMM7, + "xmm8": XMM8, + "xmm9": XMM9, + "xmm10": XMM10, + "xmm11": XMM11, + "xmm12": XMM12, + "xmm13": XMM13, + "xmm14": XMM14, + "xmm15": XMM15, + "xmm16": XMM16, + "xmm17": XMM17, + "xmm18": XMM18, + "xmm19": XMM19, + "xmm20": XMM20, + "xmm21": XMM21, + "xmm22": XMM22, + "xmm23": XMM23, + "xmm24": XMM24, + "xmm25": XMM25, + "xmm26": XMM26, + "xmm27": XMM27, + "xmm28": XMM28, + "xmm29": XMM29, + "xmm30": XMM30, + "xmm31": XMM31, + "ymm0": YMM0, + "ymm1": YMM1, + "ymm2": YMM2, + "ymm3": YMM3, + "ymm4": YMM4, + "ymm5": YMM5, + "ymm6": YMM6, + "ymm7": YMM7, + "ymm8": YMM8, + "ymm9": YMM9, + "ymm10": YMM10, + "ymm11": YMM11, + "ymm12": YMM12, + "ymm13": YMM13, + "ymm14": YMM14, + "ymm15": YMM15, + "ymm16": YMM16, + "ymm17": YMM17, + "ymm18": YMM18, + "ymm19": YMM19, + "ymm20": YMM20, + "ymm21": YMM21, + "ymm22": YMM22, + "ymm23": YMM23, + "ymm24": YMM24, + "ymm25": YMM25, + "ymm26": YMM26, + "ymm27": YMM27, + "ymm28": YMM28, + "ymm29": YMM29, + "ymm30": YMM30, + "ymm31": YMM31, + "zmm0": ZMM0, + "zmm1": ZMM1, + "zmm2": ZMM2, + "zmm3": ZMM3, + "zmm4": ZMM4, + "zmm5": ZMM5, + "zmm6": ZMM6, + "zmm7": ZMM7, + "zmm8": ZMM8, + "zmm9": ZMM9, + "zmm10": ZMM10, + "zmm11": ZMM11, + "zmm12": ZMM12, + "zmm13": ZMM13, + "zmm14": ZMM14, + "zmm15": ZMM15, + "zmm16": ZMM16, + "zmm17": ZMM17, + "zmm18": ZMM18, + "zmm19": ZMM19, + "zmm20": ZMM20, + "zmm21": ZMM21, + "zmm22": ZMM22, + "zmm23": ZMM23, + "zmm24": ZMM24, + "zmm25": ZMM25, + "zmm26": ZMM26, + "zmm27": ZMM27, + "zmm28": ZMM28, + "zmm29": ZMM29, + "zmm30": ZMM30, + "zmm31": ZMM31, +} + +/** Register Name Tables **/ + +var r8names = [...]string{ + AL: "al", + CL: "cl", + DL: "dl", + BL: "bl", + SPL: "spl", + BPL: "bpl", + SIL: "sil", + DIL: "dil", + R8b: "r8b", + R9b: "r9b", + R10b: "r10b", + R11b: "r11b", + R12b: "r12b", + R13b: "r13b", + R14b: "r14b", + R15b: "r15b", + AH: "ah", + CH: "ch", + DH: "dh", + BH: "bh", +} + +var r16names = [...]string{ + AX: "ax", + CX: "cx", + DX: "dx", + BX: "bx", + SP: "sp", + BP: "bp", + SI: "si", + DI: "di", + R8w: "r8w", + R9w: "r9w", + R10w: "r10w", + R11w: "r11w", + R12w: "r12w", + R13w: "r13w", + R14w: "r14w", + R15w: "r15w", +} + +var r32names = [...]string{ + EAX: "eax", + ECX: "ecx", + EDX: "edx", + EBX: "ebx", + ESP: "esp", + EBP: "ebp", + ESI: "esi", + EDI: "edi", + R8d: "r8d", + R9d: "r9d", + R10d: "r10d", + R11d: "r11d", + R12d: "r12d", + R13d: "r13d", + R14d: "r14d", + R15d: "r15d", +} + +var r64names = [...]string{ + RAX: "rax", + RCX: "rcx", + RDX: "rdx", + RBX: "rbx", + RSP: "rsp", + RBP: "rbp", + RSI: "rsi", + RDI: "rdi", + R8: "r8", + R9: "r9", + R10: "r10", + R11: "r11", + R12: "r12", + R13: "r13", + R14: "r14", + R15: "r15", +} + +var knames = [...]string{ + K0: "k0", + K1: "k1", + K2: "k2", + K3: "k3", + K4: "k4", + K5: "k5", + K6: "k6", + K7: "k7", +} + +var mmnames = [...]string{ + MM0: "mm0", + MM1: "mm1", + MM2: "mm2", + MM3: "mm3", + MM4: "mm4", + MM5: "mm5", + MM6: "mm6", + MM7: "mm7", +} + +var xmmnames = [...]string{ + XMM0: "xmm0", + XMM1: "xmm1", + XMM2: "xmm2", + XMM3: "xmm3", + XMM4: "xmm4", + XMM5: "xmm5", + XMM6: "xmm6", + XMM7: "xmm7", + XMM8: "xmm8", + XMM9: "xmm9", + XMM10: "xmm10", + XMM11: "xmm11", + XMM12: "xmm12", + XMM13: "xmm13", + XMM14: "xmm14", + XMM15: "xmm15", + XMM16: "xmm16", + XMM17: "xmm17", + XMM18: "xmm18", + XMM19: "xmm19", + XMM20: "xmm20", + XMM21: "xmm21", + XMM22: "xmm22", + XMM23: "xmm23", + XMM24: "xmm24", + XMM25: "xmm25", + XMM26: "xmm26", + XMM27: "xmm27", + XMM28: "xmm28", + XMM29: "xmm29", + XMM30: "xmm30", + XMM31: "xmm31", +} + +var ymmnames = [...]string{ + YMM0: "ymm0", + YMM1: "ymm1", + YMM2: "ymm2", + YMM3: "ymm3", + YMM4: "ymm4", + YMM5: "ymm5", + YMM6: "ymm6", + YMM7: "ymm7", + YMM8: "ymm8", + YMM9: "ymm9", + YMM10: "ymm10", + YMM11: "ymm11", + YMM12: "ymm12", + YMM13: "ymm13", + YMM14: "ymm14", + YMM15: "ymm15", + YMM16: "ymm16", + YMM17: "ymm17", + YMM18: "ymm18", + YMM19: "ymm19", + YMM20: "ymm20", + YMM21: "ymm21", + YMM22: "ymm22", + YMM23: "ymm23", + YMM24: "ymm24", + YMM25: "ymm25", + YMM26: "ymm26", + YMM27: "ymm27", + YMM28: "ymm28", + YMM29: "ymm29", + YMM30: "ymm30", + YMM31: "ymm31", +} + +var zmmnames = [...]string{ + ZMM0: "zmm0", + ZMM1: "zmm1", + ZMM2: "zmm2", + ZMM3: "zmm3", + ZMM4: "zmm4", + ZMM5: "zmm5", + ZMM6: "zmm6", + ZMM7: "zmm7", + ZMM8: "zmm8", + ZMM9: "zmm9", + ZMM10: "zmm10", + ZMM11: "zmm11", + ZMM12: "zmm12", + ZMM13: "zmm13", + ZMM14: "zmm14", + ZMM15: "zmm15", + ZMM16: "zmm16", + ZMM17: "zmm17", + ZMM18: "zmm18", + ZMM19: "zmm19", + ZMM20: "zmm20", + ZMM21: "zmm21", + ZMM22: "zmm22", + ZMM23: "zmm23", + ZMM24: "zmm24", + ZMM25: "zmm25", + ZMM26: "zmm26", + ZMM27: "zmm27", + ZMM28: "zmm28", + ZMM29: "zmm29", + ZMM30: "zmm30", + ZMM31: "zmm31", +} diff --git a/loader/internal/iasm/x86_64/utils.go b/loader/internal/iasm/x86_64/utils.go new file mode 100644 index 000000000..107dfb3cd --- /dev/null +++ b/loader/internal/iasm/x86_64/utils.go @@ -0,0 +1,147 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package x86_64 + +import ( + "encoding/binary" + "errors" + "reflect" + "strconv" + "unicode/utf8" + "unsafe" +) + +const ( + _CC_digit = 1 << iota + _CC_ident + _CC_ident0 + _CC_number +) + +func ispow2(v uint64) bool { + return (v & (v - 1)) == 0 +} + +func isdigit(cc rune) bool { + return '0' <= cc && cc <= '9' +} + +func isalpha(cc rune) bool { + return (cc >= 'a' && cc <= 'z') || (cc >= 'A' && cc <= 'Z') +} + +func isident(cc rune) bool { + return cc == '_' || isalpha(cc) || isdigit(cc) +} + +func isident0(cc rune) bool { + return cc == '_' || isalpha(cc) +} + +func isnumber(cc rune) bool { + return (cc == 'b' || cc == 'B') || + (cc == 'o' || cc == 'O') || + (cc == 'x' || cc == 'X') || + (cc >= '0' && cc <= '9') || + (cc >= 'a' && cc <= 'f') || + (cc >= 'A' && cc <= 'F') +} + +func align(v int, n int) int { + return (((v - 1) >> n) + 1) << n +} + +func append8(m *[]byte, v byte) { + *m = append(*m, v) +} + +func append16(m *[]byte, v uint16) { + p := len(*m) + *m = append(*m, 0, 0) + binary.LittleEndian.PutUint16((*m)[p:], v) +} + +func append32(m *[]byte, v uint32) { + p := len(*m) + *m = append(*m, 0, 0, 0, 0) + binary.LittleEndian.PutUint32((*m)[p:], v) +} + +func append64(m *[]byte, v uint64) { + p := len(*m) + *m = append(*m, 0, 0, 0, 0, 0, 0, 0, 0) + binary.LittleEndian.PutUint64((*m)[p:], v) +} + +func expandmm(m *[]byte, n int, v byte) { + sl := (*_GoSlice)(unsafe.Pointer(m)) + nb := sl.len + n + + /* grow as needed */ + if nb > cap(*m) { + *m = growslice(byteType, *m, nb) + } + + /* fill the new area */ + memset(unsafe.Pointer(uintptr(sl.ptr)+uintptr(sl.len)), v, uintptr(n)) + sl.len = nb +} + +func memset(p unsafe.Pointer, c byte, n uintptr) { + if c != 0 { + memsetv(p, c, n) + } else { + memclrNoHeapPointers(p, n) + } +} + +func memsetv(p unsafe.Pointer, c byte, n uintptr) { + for i := uintptr(0); i < n; i++ { + *(*byte)(unsafe.Pointer(uintptr(p) + i)) = c + } +} + +func literal64(v string) (uint64, error) { + var nb int + var ch rune + var ex error + var mm [12]byte + + /* unquote the runes */ + for v != "" { + if ch, _, v, ex = strconv.UnquoteChar(v, '\''); ex != nil { + return 0, ex + } else if nb += utf8.EncodeRune(mm[nb:], ch); nb > 8 { + return 0, errors.New("multi-char constant too large") + } + } + + /* convert to uint64 */ + return *(*uint64)(unsafe.Pointer(&mm)), nil +} + +var ( + byteWrap = reflect.TypeOf(byte(0)) + byteType = (*_GoType)(efaceOf(byteWrap).ptr) +) + +//go:linkname growslice runtime.growslice +func growslice(_ *_GoType, _ []byte, _ int) []byte + +//go:noescape +//go:linkname memclrNoHeapPointers runtime.memclrNoHeapPointers +func memclrNoHeapPointers(_ unsafe.Pointer, _ uintptr) diff --git a/native/atof_native.h b/native/atof_native.h index 629d0cc35..a6c4a62dd 100644 --- a/native/atof_native.h +++ b/native/atof_native.h @@ -18,7 +18,7 @@ #include "native.h" -/* decimical shift witout overflow, e.g. 9 << 61 overflow */ +/* decimical shift without overflow, e.g. 9 << 61 overflow */ #define MAX_SHIFT 60 /* Decimal represent the integer or float @@ -90,7 +90,7 @@ static always_inline void decimal_set(Decimal *d, const char *s, ssize_t len, ch d->d[d->nd] = s[i]; d->nd++; } else if (s[i] != '0') { - /* truncat the remaining digits */ + /* truncate the remaining digits */ d->trunc = 1; } } else if (s[i] == '.') { @@ -367,7 +367,7 @@ static always_inline int decimal_to_f64(Decimal *d, double *val) { /* Our range is [0.5,1) but floating point range is [1,2) */ exp2 --; - /* Minimum exp2 for doulbe is -1022. + /* Minimum exp2 for double is -1022. * If the exponent is smaller, move it up and * adjust d accordingly. */ diff --git a/native/html_escape.c b/native/html_escape.c index e6925d1b2..5d895471d 100644 --- a/native/html_escape.c +++ b/native/html_escape.c @@ -36,7 +36,7 @@ ssize_t html_escape(const char *sp, ssize_t nb, char *dp, ssize_t *dn) { break; } - /* mark cur postion */ + /* mark cur position */ cur = sp; /* check for \u2028 and \u2029, binary is \xe2\x80\xa8 and \xe2\x80\xa9 */ diff --git a/native/lookup_small_key.c b/native/lookup_small_key.c index fcf41b455..07eff791a 100644 --- a/native/lookup_small_key.c +++ b/native/lookup_small_key.c @@ -130,6 +130,10 @@ long lookup_small_key(GoString* key, GoSlice* table, long lower_off) { } case_sensitve: + if (lower_off == -1) { + return -_NOT_FOUND; + } + offset = lower_off + *(uint32_t*)(meta + 1); p = table->buf + offset; to_lower(lower, (uint8_t*)(key->buf), 32); diff --git a/native/parse_with_padding.c b/native/parse_with_padding.c index 64f685751..4a6d21744 100644 --- a/native/parse_with_padding.c +++ b/native/parse_with_padding.c @@ -37,7 +37,7 @@ static const option _F_USE_NUMBER = 1; // types.B_USE_NUMBER static const option _F_VALIDATE_STR = 5; // types.B_VALIDATE_STRING static const option _F_ALLOW_CTL = 31; // types.B_ALLOW_CONTROL -// NOTE: only enbale these flags in sonic-rs. +// NOTE: only enable these flags in sonic-rs. static const option F_USE_NUMBER = 1 << _F_USE_NUMBER; static const option F_USE_INT64 = 1 << _F_USE_INT64; static const option F_VALIDATE_STRING = 1 << _F_VALIDATE_STR; @@ -636,7 +636,7 @@ static always_inline long parse_string_inplace(uint8_t** cur, bool* has_esc, uin return -SONIC_INVALID_ESCAPED_UTF; } - // check continous escaped char + // check continuous escaped char if (**cur == '\\') { goto escape; } @@ -739,7 +739,7 @@ static always_inline error_code parse_number(uint8_t** cur, parse_num* num, bool } if (**cur == '.') { - // skip continous zeros + // skip continuous zeros *cur += 1; uint8_t* dot_pos = *cur; should_digit(**cur); diff --git a/native/parsing.h b/native/parsing.h index f3f188574..fee32d6c7 100644 --- a/native/parsing.h +++ b/native/parsing.h @@ -423,7 +423,7 @@ static always_inline ssize_t memcchr_p32(const char *s, ssize_t nb, char *p) { _mm256_zeroupper(); #endif - /* initialze with '\\' */ + /* initialize with '\\' */ __m128i x; __m128i y; __m128i a = _mm_set1_epi8('\\'); diff --git a/native/scanning.h b/native/scanning.h index 96047baa1..b399eb45c 100644 --- a/native/scanning.h +++ b/native/scanning.h @@ -349,7 +349,7 @@ static always_inline int _mm256_get_mask(__m256i v, __m256i t) { return _mm256_movemask_epi8(_mm256_cmpeq_epi8(v, t)); } -// contrl char: 0x00 ~ 0x1F +// control char: 0x00 ~ 0x1F static always_inline int _mm256_cchars_mask(__m256i v) { __m256i e1 = _mm256_cmpgt_epi8 (v, _mm256_set1_epi8(-1)); __m256i e2 = _mm256_cmpgt_epi8 (v, _mm256_set1_epi8(31)); @@ -367,7 +367,7 @@ static always_inline int _mm_get_mask(__m128i v, __m128i t) { return _mm_movemask_epi8(_mm_cmpeq_epi8(v, t)); } -// contrl char: 0x00 ~ 0x1F +// control char: 0x00 ~ 0x1F static always_inline int _mm_cchars_mask(__m128i v) { __m128i e1 = _mm_cmpgt_epi8 (v, _mm_set1_epi8(-1)); __m128i e2 = _mm_cmpgt_epi8 (v, _mm_set1_epi8(31)); @@ -800,7 +800,7 @@ static always_inline double atof_fast(uint64_t man, int exp, int sgn, int trunc, static bool always_inline is_overflow(uint64_t man, int sgn, int exp10) { /* the former exp10 != 0 means man has overflowed - * the later euqals to man*sgn < INT64_MIN or > INT64_MAX */ + * the latter equals to man*sgn < INT64_MIN or > INT64_MAX */ return exp10 != 0 || ((man >> 63) == 1 && ((uint64_t)sgn & man) != (1ull << 63)); } diff --git a/native/utf8.h b/native/utf8.h index 73a988ca9..fe58b48fe 100644 --- a/native/utf8.h +++ b/native/utf8.h @@ -131,7 +131,7 @@ static always_inline long write_error(int pos, StateMachine *m, size_t msize) { return 0; } -// scalar code, error position should excesss 4096 +// scalar code, error position should excess 4096 static always_inline long validate_utf8_with_errors(const char *src, long len, long *p, StateMachine *m) { const char* start = src + *p; const char* end = src + len; @@ -437,14 +437,14 @@ static always_inline long validate_utf8_errors(const GoString* s) { return; } - // frist 64 bytes is ascii, next 64 bytes must be utf8 + // first 64 bytes is ascii, next 64 bytes must be utf8 if (likely(is_ascii(reducer1))) { checker->error = _mm256_or_si256(checker->error, checker->prev_incomplete); check64_utf(checker, start + 64); return; } - // frist 64 bytes has utf8, next 64 bytes + // first 64 bytes has utf8, next 64 bytes check64_utf(checker, start); if (unlikely(is_ascii(reducer2))) { checker->error = _mm256_or_si256(checker->error, checker->prev_incomplete); diff --git a/option/option.go b/option/option.go index 4d9965260..1ec6a8528 100644 --- a/option/option.go +++ b/option/option.go @@ -68,7 +68,7 @@ type CompileOption func(o *CompileOptions) // // For deep nested struct (depth exceeds MaxInlineDepth), // try to set more loops to completely compile, -// thus reduce JIT unstability in the first hit. +// thus reduce JIT instability in the first hit. func WithCompileRecursiveDepth(loop int) CompileOption { return func(o *CompileOptions) { if loop < 0 { diff --git a/rawmessage.go b/rawmessage.go new file mode 100644 index 000000000..e90f42291 --- /dev/null +++ b/rawmessage.go @@ -0,0 +1,43 @@ +/* + * Copyright 2024 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sonic + +import ( + "errors" +) + +// NoCopyRawMessage is a NOCOPY raw encoded JSON value. +// It implements [Marshaler] and [Unmarshaler] and can +// be used to delay JSON decoding or precompute a JSON encoding. +type NoCopyRawMessage []byte + +// MarshalJSON returns m as the JSON encoding of m. +func (m NoCopyRawMessage) MarshalJSON() ([]byte, error) { + if m == nil { + return []byte("null"), nil + } + return m, nil +} + +// UnmarshalJSON sets *m to a reference of data. NoCopy here!!! +func (m *NoCopyRawMessage) UnmarshalJSON(data []byte) error { + if m == nil { + return errors.New("sonic.NoCopyRawMessage: UnmarshalJSON on nil pointer") + } + *m = data + return nil +} diff --git a/scripts/bench.py b/scripts/bench.py index 41e1b5a08..206a9dd1a 100755 --- a/scripts/bench.py +++ b/scripts/bench.py @@ -49,7 +49,7 @@ def run_r(cmd): return data.decode("utf-8") def compare(args): - # detech current branch. + # detect current branch. # result = run_r("git branch") current_branch = run_s("git status | head -n1 | sed 's/On branch //'") # for br in result.split('\n'): @@ -58,14 +58,14 @@ def compare(args): # break if not current_branch: - print ("Failed to detech current branch") + print ("Failed to detect current branch") return None # get the current diff (fd, diff) = tempfile.mkstemp() run("git diff > %s"%diff) - # early return if currrent is main branch. + # early return if current is main branch. print ("Current branch: %s"%(current_branch)) if current_branch == "main": print ("Cannot compare at the main branch.Please build a new branch") diff --git a/scripts/build-x86.sh b/scripts/build-x86.sh index 4a0725bdd..030e7a907 100755 --- a/scripts/build-x86.sh +++ b/scripts/build-x86.sh @@ -60,7 +60,7 @@ for arc in "${CPU_ARCS[@]}"; do # should check the output assembly files if ! grep -rq ${CHECK_ARCS[$i]} $out_dir; then - echo "compiled instructions is incorret, pleas check again" + echo "compiled instructions is incorrect, please check again" exit 1 fi diff --git a/tools/asm2arm/arm.py b/tools/asm2arm/arm.py index 14b6164de..46effcd71 100644 --- a/tools/asm2arm/arm.py +++ b/tools/asm2arm/arm.py @@ -477,7 +477,7 @@ def encode(buf: bytes, comments: str = '') -> str: # return '\n\t'.join(r) if (n % 4 != 0): - raise RuntimeError("Unkown instruction which not encoding 4 bytes: %s " % comments, buf) + raise RuntimeError("Unknown instruction which not encoding 4 bytes: %s " % comments, buf) while i < n - 3: r.append('WORD $0x%08x' % int.from_bytes(buf[i:i + 4], 'little')) @@ -567,7 +567,7 @@ def optimize(self): self.out.append((self.pc - self.entry, self.sp)) # sort by pc self.out.sort(key=lambda x: x[0]) - # NOTICE: first pair {1, 0} to be compitable with golang + # NOTICE: first pair {1, 0} to be compatible with golang tmp = [(1, 0)] lpc, lsp = 0, -1 for pc, sp in self.out: @@ -927,7 +927,7 @@ def parse(cls, src: str) -> 'Command': # invalid hexadecimal character elif esc in (ESC_HEX0, ESC_HEX1): - raise SyntaxError('invalid hexdecimal character: ' + repr(nch)) + raise SyntaxError('invalid hexadecimal character: ' + repr(nch)) # octal escape characters elif esc in (ESC_OCT1, ESC_OCT2) and nch.lower() in string.octdigits: @@ -1329,7 +1329,7 @@ def if_all_IntInstr_then_2_RawInstr(self): if instr_size == 8 or instr_size == 4: return - # All instrs are IntInstr, golang asm only suuport WORD and DWORD for arm. We need + # All instrs are IntInstr, golang asm only support WORD and DWORD for arm. We need # combine them as 4-bytes RawInstr and align block nb = [] # new body raw_buf = []; @@ -1374,7 +1374,7 @@ def jump_to(self, block: 'BasicBlock'): block.prevs.append(self) @classmethod - def annonymous(cls) -> 'BasicBlock': + def anonymous(cls) -> 'BasicBlock': return cls('// bb.%d' % Counter.next(), weak = False) CLANG_JUMPTABLE_LABLE = 'LJTI' @@ -1392,7 +1392,7 @@ def __init__(self): self.dead = False self.labels = {} self.export = False - self.blocks = [BasicBlock.annonymous()] + self.blocks = [BasicBlock.anonymous()] self.jmptabs = {} self.funcs = {} self.bsmap_ = {} @@ -1451,7 +1451,7 @@ def _kill(self, name: str): def _split(self, jmp: BasicBlock): self.jump = True - link = BasicBlock.annonymous() + link = BasicBlock.anonymous() self.labels[link.name] = link self.block.link_to(link) self.block.jump_to(jmp) @@ -1505,7 +1505,7 @@ def _alloc_instr(self, instr: Instruction): else: self.block.body.append(BranchInstr(instr)) - # it seems to not be able to specify stack aligment inside the Go ASM so we + # it seems to not be able to specify stack alignment inside the Go ASM so we # need to replace the aligned instructions with unaligned one if either of it's # operand is an RBP relative addressing memory operand @@ -1645,7 +1645,7 @@ def _trace_instructions(self, bb: BasicBlock, pcsp: Pcsp) -> Tuple[int, bool]: elif name == 'str': diff = -self._mk_align(operands[3]) else: - raise RuntimeError("An instruction adjsut sp but bot processed: %s" % ins.instr.asm_code) + raise RuntimeError("An instruction adjust sp but bot processed: %s" % ins.instr.asm_code) cursp += diff @@ -2092,7 +2092,7 @@ def _declare_function(self, name: str, proto: Prototype): # self.out.append('\tMOVD R29, -8(RSP)') # self.out.append('\tSUB $8, RSP, R29') - # intialize all the arguments + # initialize all the arguments for arg in proto.args: offs += arg.size op, reg = REG_MAP[arg.creg.reg] diff --git a/unquote/unquote.go b/unquote/unquote.go index d81406b2a..29b2fcde8 100644 --- a/unquote/unquote.go +++ b/unquote/unquote.go @@ -25,7 +25,7 @@ import ( `github.com/bytedance/sonic/internal/rt` ) -// String unescapes a escaped string (not including `"` at begining and end) +// String unescapes an escaped string (not including `"` at beginning and end) // It validates invalid UTF8 and replace with `\ufffd` func String(s string) (ret string, err types.ParsingError) { mm := make([]byte, 0, len(s)) @@ -43,7 +43,7 @@ func IntoBytes(s string, m *[]byte) types.ParsingError { } } -// String unescapes a escaped string (not including `"` at begining and end) +// String unescapes an escaped string (not including `"` at beginning and end) // - replace enables replacing invalid utf8 escaped char with `\uffd` func _String(s string, replace bool) (ret string, err error) { mm := make([]byte, 0, len(s)) diff --git a/utf8/utf8.go b/utf8/utf8.go index 573fe2f5b..9d8bcc958 100644 --- a/utf8/utf8.go +++ b/utf8/utf8.go @@ -29,7 +29,7 @@ func CorrectWith(dst []byte, src []byte, repl string) []byte { sstr := rt.Mem2Str(src) sidx := 0 - /* state machine records the invalid postions */ + /* state machine records the invalid positions */ m := types.NewStateMachine() m.Sp = 0 // invalid utf8 numbers